1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_file.h> 10 #include <drm/drm_gem.h> 11 #include <drm/drm_gem_shmem_helper.h> 12 #include <drm/drm_print.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 21 #define MAX_HWCTX_ID 255 22 #define MAX_ARG_COUNT 4095 23 24 struct amdxdna_fence { 25 struct dma_fence base; 26 spinlock_t lock; /* for base */ 27 struct amdxdna_hwctx *hwctx; 28 }; 29 30 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence) 31 { 32 return KBUILD_MODNAME; 33 } 34 35 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence) 36 { 37 struct amdxdna_fence *xdna_fence; 38 39 xdna_fence = container_of(fence, struct amdxdna_fence, base); 40 41 return xdna_fence->hwctx->name; 42 } 43 44 static const struct dma_fence_ops fence_ops = { 45 .get_driver_name = amdxdna_fence_get_driver_name, 46 .get_timeline_name = amdxdna_fence_get_timeline_name, 47 }; 48 49 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx) 50 { 51 struct amdxdna_fence *fence; 52 53 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 54 if (!fence) 55 return NULL; 56 57 fence->hwctx = hwctx; 58 spin_lock_init(&fence->lock); 59 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0); 60 return &fence->base; 61 } 62 63 void amdxdna_hwctx_suspend(struct amdxdna_client *client) 64 { 65 struct amdxdna_dev *xdna = client->xdna; 66 struct amdxdna_hwctx *hwctx; 67 unsigned long hwctx_id; 68 69 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 70 mutex_lock(&client->hwctx_lock); 71 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) 72 xdna->dev_info->ops->hwctx_suspend(hwctx); 73 mutex_unlock(&client->hwctx_lock); 74 } 75 76 void amdxdna_hwctx_resume(struct amdxdna_client *client) 77 { 78 struct amdxdna_dev *xdna = client->xdna; 79 struct amdxdna_hwctx *hwctx; 80 unsigned long hwctx_id; 81 82 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 83 mutex_lock(&client->hwctx_lock); 84 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) 85 xdna->dev_info->ops->hwctx_resume(hwctx); 86 mutex_unlock(&client->hwctx_lock); 87 } 88 89 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx, 90 struct srcu_struct *ss) 91 { 92 struct amdxdna_dev *xdna = hwctx->client->xdna; 93 94 synchronize_srcu(ss); 95 96 /* At this point, user is not able to submit new commands */ 97 mutex_lock(&xdna->dev_lock); 98 xdna->dev_info->ops->hwctx_fini(hwctx); 99 mutex_unlock(&xdna->dev_lock); 100 101 kfree(hwctx->name); 102 kfree(hwctx); 103 } 104 105 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) 106 { 107 struct amdxdna_cmd *cmd = abo->mem.kva; 108 u32 num_masks, count; 109 110 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 111 num_masks = 0; 112 else 113 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 114 115 if (size) { 116 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); 117 if (unlikely(count <= num_masks)) { 118 *size = 0; 119 return NULL; 120 } 121 *size = (count - num_masks) * sizeof(u32); 122 } 123 return &cmd->data[num_masks]; 124 } 125 126 int amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) 127 { 128 struct amdxdna_cmd *cmd = abo->mem.kva; 129 u32 num_masks, i; 130 u32 *cu_mask; 131 132 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 133 return -1; 134 135 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 136 cu_mask = cmd->data; 137 for (i = 0; i < num_masks; i++) { 138 if (cu_mask[i]) 139 return ffs(cu_mask[i]) - 1; 140 } 141 142 return -1; 143 } 144 145 /* 146 * This should be called in close() and remove(). DO NOT call in other syscalls. 147 * This guarantee that when hwctx and resources will be released, if user 148 * doesn't call amdxdna_drm_destroy_hwctx_ioctl. 149 */ 150 void amdxdna_hwctx_remove_all(struct amdxdna_client *client) 151 { 152 struct amdxdna_hwctx *hwctx; 153 unsigned long hwctx_id; 154 155 mutex_lock(&client->hwctx_lock); 156 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 157 XDNA_DBG(client->xdna, "PID %d close HW context %d", 158 client->pid, hwctx->id); 159 xa_erase(&client->hwctx_xa, hwctx->id); 160 mutex_unlock(&client->hwctx_lock); 161 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 162 mutex_lock(&client->hwctx_lock); 163 } 164 mutex_unlock(&client->hwctx_lock); 165 } 166 167 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 168 { 169 struct amdxdna_client *client = filp->driver_priv; 170 struct amdxdna_drm_create_hwctx *args = data; 171 struct amdxdna_dev *xdna = to_xdna_dev(dev); 172 struct amdxdna_hwctx *hwctx; 173 int ret, idx; 174 175 if (args->ext || args->ext_flags) 176 return -EINVAL; 177 178 if (!drm_dev_enter(dev, &idx)) 179 return -ENODEV; 180 181 hwctx = kzalloc(sizeof(*hwctx), GFP_KERNEL); 182 if (!hwctx) { 183 ret = -ENOMEM; 184 goto exit; 185 } 186 187 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) { 188 XDNA_ERR(xdna, "Access QoS info failed"); 189 ret = -EFAULT; 190 goto free_hwctx; 191 } 192 193 hwctx->client = client; 194 hwctx->fw_ctx_id = -1; 195 hwctx->num_tiles = args->num_tiles; 196 hwctx->mem_size = args->mem_size; 197 hwctx->max_opc = args->max_opc; 198 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx, 199 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID), 200 &client->next_hwctxid, GFP_KERNEL); 201 if (ret < 0) { 202 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret); 203 goto free_hwctx; 204 } 205 206 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->id); 207 if (!hwctx->name) { 208 ret = -ENOMEM; 209 goto rm_id; 210 } 211 212 mutex_lock(&xdna->dev_lock); 213 ret = xdna->dev_info->ops->hwctx_init(hwctx); 214 if (ret) { 215 mutex_unlock(&xdna->dev_lock); 216 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret); 217 goto free_name; 218 } 219 args->handle = hwctx->id; 220 args->syncobj_handle = hwctx->syncobj_hdl; 221 mutex_unlock(&xdna->dev_lock); 222 223 atomic64_set(&hwctx->job_submit_cnt, 0); 224 atomic64_set(&hwctx->job_free_cnt, 0); 225 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret); 226 drm_dev_exit(idx); 227 return 0; 228 229 free_name: 230 kfree(hwctx->name); 231 rm_id: 232 xa_erase(&client->hwctx_xa, hwctx->id); 233 free_hwctx: 234 kfree(hwctx); 235 exit: 236 drm_dev_exit(idx); 237 return ret; 238 } 239 240 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 241 { 242 struct amdxdna_client *client = filp->driver_priv; 243 struct amdxdna_drm_destroy_hwctx *args = data; 244 struct amdxdna_dev *xdna = to_xdna_dev(dev); 245 struct amdxdna_hwctx *hwctx; 246 int ret = 0, idx; 247 248 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 249 return -EINVAL; 250 251 if (!drm_dev_enter(dev, &idx)) 252 return -ENODEV; 253 254 hwctx = xa_erase(&client->hwctx_xa, args->handle); 255 if (!hwctx) { 256 ret = -EINVAL; 257 XDNA_DBG(xdna, "PID %d HW context %d not exist", 258 client->pid, args->handle); 259 goto out; 260 } 261 262 /* 263 * The pushed jobs are handled by DRM scheduler during destroy. 264 * SRCU to synchronize with exec command ioctls. 265 */ 266 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 267 268 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle); 269 out: 270 drm_dev_exit(idx); 271 return ret; 272 } 273 274 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 275 { 276 struct amdxdna_client *client = filp->driver_priv; 277 struct amdxdna_drm_config_hwctx *args = data; 278 struct amdxdna_dev *xdna = to_xdna_dev(dev); 279 struct amdxdna_hwctx *hwctx; 280 int ret, idx; 281 u32 buf_size; 282 void *buf; 283 u64 val; 284 285 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 286 return -EINVAL; 287 288 if (!xdna->dev_info->ops->hwctx_config) 289 return -EOPNOTSUPP; 290 291 val = args->param_val; 292 buf_size = args->param_val_size; 293 294 switch (args->param_type) { 295 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 296 /* For those types that param_val is pointer */ 297 if (buf_size > PAGE_SIZE) { 298 XDNA_ERR(xdna, "Config CU param buffer too large"); 299 return -E2BIG; 300 } 301 302 /* Hwctx needs to keep buf */ 303 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 304 if (!buf) 305 return -ENOMEM; 306 307 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) { 308 kfree(buf); 309 return -EFAULT; 310 } 311 312 break; 313 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 314 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 315 /* For those types that param_val is a value */ 316 buf = NULL; 317 buf_size = 0; 318 break; 319 default: 320 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type); 321 return -EINVAL; 322 } 323 324 mutex_lock(&xdna->dev_lock); 325 idx = srcu_read_lock(&client->hwctx_srcu); 326 hwctx = xa_load(&client->hwctx_xa, args->handle); 327 if (!hwctx) { 328 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); 329 ret = -EINVAL; 330 goto unlock_srcu; 331 } 332 333 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); 334 335 unlock_srcu: 336 srcu_read_unlock(&client->hwctx_srcu, idx); 337 mutex_unlock(&xdna->dev_lock); 338 kfree(buf); 339 return ret; 340 } 341 342 static void 343 amdxdna_arg_bos_put(struct amdxdna_sched_job *job) 344 { 345 int i; 346 347 for (i = 0; i < job->bo_cnt; i++) { 348 if (!job->bos[i]) 349 break; 350 drm_gem_object_put(job->bos[i]); 351 } 352 } 353 354 static int 355 amdxdna_arg_bos_lookup(struct amdxdna_client *client, 356 struct amdxdna_sched_job *job, 357 u32 *bo_hdls, u32 bo_cnt) 358 { 359 struct drm_gem_object *gobj; 360 int i, ret; 361 362 job->bo_cnt = bo_cnt; 363 for (i = 0; i < job->bo_cnt; i++) { 364 struct amdxdna_gem_obj *abo; 365 366 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]); 367 if (!gobj) { 368 ret = -ENOENT; 369 goto put_shmem_bo; 370 } 371 abo = to_xdna_obj(gobj); 372 373 mutex_lock(&abo->lock); 374 if (abo->pinned) { 375 mutex_unlock(&abo->lock); 376 job->bos[i] = gobj; 377 continue; 378 } 379 380 ret = amdxdna_gem_pin_nolock(abo); 381 if (ret) { 382 mutex_unlock(&abo->lock); 383 drm_gem_object_put(gobj); 384 goto put_shmem_bo; 385 } 386 abo->pinned = true; 387 mutex_unlock(&abo->lock); 388 389 job->bos[i] = gobj; 390 } 391 392 return 0; 393 394 put_shmem_bo: 395 amdxdna_arg_bos_put(job); 396 return ret; 397 } 398 399 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) 400 { 401 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); 402 amdxdna_arg_bos_put(job); 403 amdxdna_gem_put_obj(job->cmd_bo); 404 } 405 406 int amdxdna_cmd_submit(struct amdxdna_client *client, 407 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt, 408 u32 hwctx_hdl, u64 *seq) 409 { 410 struct amdxdna_dev *xdna = client->xdna; 411 struct amdxdna_sched_job *job; 412 struct amdxdna_hwctx *hwctx; 413 int ret, idx; 414 415 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); 416 job = kzalloc(struct_size(job, bos, arg_bo_cnt), GFP_KERNEL); 417 if (!job) 418 return -ENOMEM; 419 420 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) { 421 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD); 422 if (!job->cmd_bo) { 423 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl); 424 ret = -EINVAL; 425 goto free_job; 426 } 427 } else { 428 job->cmd_bo = NULL; 429 } 430 431 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); 432 if (ret) { 433 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret); 434 goto cmd_put; 435 } 436 437 idx = srcu_read_lock(&client->hwctx_srcu); 438 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); 439 if (!hwctx) { 440 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", 441 client->pid, hwctx_hdl); 442 ret = -EINVAL; 443 goto unlock_srcu; 444 } 445 446 if (hwctx->status != HWCTX_STAT_READY) { 447 XDNA_ERR(xdna, "HW Context is not ready"); 448 ret = -EINVAL; 449 goto unlock_srcu; 450 } 451 452 job->hwctx = hwctx; 453 job->mm = current->mm; 454 455 job->fence = amdxdna_fence_create(hwctx); 456 if (!job->fence) { 457 XDNA_ERR(xdna, "Failed to create fence"); 458 ret = -ENOMEM; 459 goto unlock_srcu; 460 } 461 kref_init(&job->refcnt); 462 463 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq); 464 if (ret) 465 goto put_fence; 466 467 /* 468 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated 469 * resource after synchronize_srcu(). The submitted jobs should be 470 * handled by the queue, for example DRM scheduler, in device layer. 471 * For here we can unlock SRCU. 472 */ 473 srcu_read_unlock(&client->hwctx_srcu, idx); 474 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed"); 475 476 return 0; 477 478 put_fence: 479 dma_fence_put(job->fence); 480 unlock_srcu: 481 srcu_read_unlock(&client->hwctx_srcu, idx); 482 amdxdna_arg_bos_put(job); 483 cmd_put: 484 amdxdna_gem_put_obj(job->cmd_bo); 485 free_job: 486 kfree(job); 487 return ret; 488 } 489 490 /* 491 * The submit command ioctl submits a command to firmware. One firmware command 492 * may contain multiple command BOs for processing as a whole. 493 * The command sequence number is returned which can be used for wait command ioctl. 494 */ 495 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client, 496 struct amdxdna_drm_exec_cmd *args) 497 { 498 struct amdxdna_dev *xdna = client->xdna; 499 u32 *arg_bo_hdls; 500 u32 cmd_bo_hdl; 501 int ret; 502 503 if (!args->arg_count || args->arg_count > MAX_ARG_COUNT) { 504 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count); 505 return -EINVAL; 506 } 507 508 /* Only support single command for now. */ 509 if (args->cmd_count != 1) { 510 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count); 511 return -EINVAL; 512 } 513 514 cmd_bo_hdl = (u32)args->cmd_handles; 515 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL); 516 if (!arg_bo_hdls) 517 return -ENOMEM; 518 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args), 519 args->arg_count * sizeof(u32)); 520 if (ret) { 521 ret = -EFAULT; 522 goto free_cmd_bo_hdls; 523 } 524 525 ret = amdxdna_cmd_submit(client, cmd_bo_hdl, arg_bo_hdls, 526 args->arg_count, args->hwctx, &args->seq); 527 if (ret) 528 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret); 529 530 free_cmd_bo_hdls: 531 kfree(arg_bo_hdls); 532 if (!ret) 533 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq); 534 return ret; 535 } 536 537 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 538 { 539 struct amdxdna_client *client = filp->driver_priv; 540 struct amdxdna_drm_exec_cmd *args = data; 541 542 if (args->ext || args->ext_flags) 543 return -EINVAL; 544 545 switch (args->type) { 546 case AMDXDNA_CMD_SUBMIT_EXEC_BUF: 547 return amdxdna_drm_submit_execbuf(client, args); 548 } 549 550 XDNA_ERR(client->xdna, "Invalid command type %d", args->type); 551 return -EINVAL; 552 } 553