1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_file.h>
10 #include <drm/drm_gem.h>
11 #include <drm/drm_gem_shmem_helper.h>
12 #include <drm/drm_print.h>
13 #include <drm/gpu_scheduler.h>
14 #include <linux/xarray.h>
15 #include <trace/events/amdxdna.h>
16
17 #include "amdxdna_ctx.h"
18 #include "amdxdna_gem.h"
19 #include "amdxdna_pci_drv.h"
20 #include "amdxdna_pm.h"
21
22 #define MAX_HWCTX_ID 255
23 #define MAX_ARG_COUNT 4095
24
25 struct amdxdna_fence {
26 struct dma_fence base;
27 spinlock_t lock; /* for base */
28 struct amdxdna_hwctx *hwctx;
29 };
30
amdxdna_fence_get_driver_name(struct dma_fence * fence)31 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence)
32 {
33 return KBUILD_MODNAME;
34 }
35
amdxdna_fence_get_timeline_name(struct dma_fence * fence)36 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence)
37 {
38 struct amdxdna_fence *xdna_fence;
39
40 xdna_fence = container_of(fence, struct amdxdna_fence, base);
41
42 return xdna_fence->hwctx->name;
43 }
44
45 static const struct dma_fence_ops fence_ops = {
46 .get_driver_name = amdxdna_fence_get_driver_name,
47 .get_timeline_name = amdxdna_fence_get_timeline_name,
48 };
49
amdxdna_fence_create(struct amdxdna_hwctx * hwctx)50 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
51 {
52 struct amdxdna_fence *fence;
53
54 fence = kzalloc_obj(*fence);
55 if (!fence)
56 return NULL;
57
58 fence->hwctx = hwctx;
59 spin_lock_init(&fence->lock);
60 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0);
61 return &fence->base;
62 }
63
amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx * hwctx,struct srcu_struct * ss)64 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
65 struct srcu_struct *ss)
66 {
67 struct amdxdna_dev *xdna = hwctx->client->xdna;
68
69 synchronize_srcu(ss);
70
71 /* At this point, user is not able to submit new commands */
72 xdna->dev_info->ops->hwctx_fini(hwctx);
73
74 kfree(hwctx->name);
75 kfree(hwctx);
76 }
77
amdxdna_hwctx_walk(struct amdxdna_client * client,void * arg,int (* walk)(struct amdxdna_hwctx * hwctx,void * arg))78 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg,
79 int (*walk)(struct amdxdna_hwctx *hwctx, void *arg))
80 {
81 struct amdxdna_hwctx *hwctx;
82 unsigned long hwctx_id;
83 int ret = 0, idx;
84
85 idx = srcu_read_lock(&client->hwctx_srcu);
86 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
87 ret = walk(hwctx, arg);
88 if (ret)
89 break;
90 }
91 srcu_read_unlock(&client->hwctx_srcu, idx);
92
93 return ret;
94 }
95
amdxdna_cmd_get_payload(struct amdxdna_gem_obj * abo,u32 * size)96 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size)
97 {
98 struct amdxdna_cmd *cmd = abo->mem.kva;
99 u32 num_masks, count;
100
101 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
102 num_masks = 0;
103 else
104 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
105
106 if (size) {
107 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header);
108 if (unlikely(count <= num_masks ||
109 count * sizeof(u32) +
110 offsetof(struct amdxdna_cmd, data[0]) >
111 abo->mem.size)) {
112 *size = 0;
113 return NULL;
114 }
115 *size = (count - num_masks) * sizeof(u32);
116 }
117 return &cmd->data[num_masks];
118 }
119
amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj * abo)120 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
121 {
122 struct amdxdna_cmd *cmd = abo->mem.kva;
123 u32 num_masks, i;
124 u32 *cu_mask;
125
126 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
127 return INVALID_CU_IDX;
128
129 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
130 cu_mask = cmd->data;
131 for (i = 0; i < num_masks; i++) {
132 if (cu_mask[i])
133 return ffs(cu_mask[i]) - 1;
134 }
135
136 return INVALID_CU_IDX;
137 }
138
amdxdna_cmd_set_error(struct amdxdna_gem_obj * abo,struct amdxdna_sched_job * job,u32 cmd_idx,enum ert_cmd_state error_state)139 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
140 struct amdxdna_sched_job *job, u32 cmd_idx,
141 enum ert_cmd_state error_state)
142 {
143 struct amdxdna_client *client = job->hwctx->client;
144 struct amdxdna_cmd *cmd = abo->mem.kva;
145 struct amdxdna_cmd_chain *cc = NULL;
146
147 cmd->header &= ~AMDXDNA_CMD_STATE;
148 cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state);
149
150 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) {
151 cc = amdxdna_cmd_get_payload(abo, NULL);
152 cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0;
153 abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_CMD);
154 if (!abo)
155 return -EINVAL;
156 cmd = abo->mem.kva;
157 }
158
159 memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd));
160 if (cc)
161 amdxdna_gem_put_obj(abo);
162
163 return 0;
164 }
165
166 /*
167 * This should be called in close() and remove(). DO NOT call in other syscalls.
168 * This guarantee that when hwctx and resources will be released, if user
169 * doesn't call amdxdna_drm_destroy_hwctx_ioctl.
170 */
amdxdna_hwctx_remove_all(struct amdxdna_client * client)171 void amdxdna_hwctx_remove_all(struct amdxdna_client *client)
172 {
173 struct amdxdna_hwctx *hwctx;
174 unsigned long hwctx_id;
175
176 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
177 XDNA_DBG(client->xdna, "PID %d close HW context %d",
178 client->pid, hwctx->id);
179 xa_erase(&client->hwctx_xa, hwctx->id);
180 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
181 }
182 }
183
amdxdna_drm_create_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)184 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
185 {
186 struct amdxdna_client *client = filp->driver_priv;
187 struct amdxdna_drm_create_hwctx *args = data;
188 struct amdxdna_dev *xdna = to_xdna_dev(dev);
189 struct amdxdna_hwctx *hwctx;
190 int ret, idx;
191
192 if (args->ext || args->ext_flags)
193 return -EINVAL;
194
195 hwctx = kzalloc_obj(*hwctx);
196 if (!hwctx)
197 return -ENOMEM;
198
199 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) {
200 XDNA_ERR(xdna, "Access QoS info failed");
201 kfree(hwctx);
202 return -EFAULT;
203 }
204
205 hwctx->client = client;
206 hwctx->fw_ctx_id = -1;
207 hwctx->num_tiles = args->num_tiles;
208 hwctx->mem_size = args->mem_size;
209 hwctx->max_opc = args->max_opc;
210
211 guard(mutex)(&xdna->dev_lock);
212
213 if (!drm_dev_enter(dev, &idx)) {
214 ret = -ENODEV;
215 goto free_hwctx;
216 }
217
218 ret = xdna->dev_info->ops->hwctx_init(hwctx);
219 if (ret) {
220 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
221 goto dev_exit;
222 }
223
224 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id);
225 if (!hwctx->name) {
226 ret = -ENOMEM;
227 goto fini_hwctx;
228 }
229
230 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx,
231 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID),
232 &client->next_hwctxid, GFP_KERNEL);
233 if (ret < 0) {
234 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret);
235 goto free_name;
236 }
237
238 args->handle = hwctx->id;
239 args->syncobj_handle = hwctx->syncobj_hdl;
240
241 atomic64_set(&hwctx->job_submit_cnt, 0);
242 atomic64_set(&hwctx->job_free_cnt, 0);
243 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret);
244 drm_dev_exit(idx);
245 return 0;
246
247 free_name:
248 kfree(hwctx->name);
249 fini_hwctx:
250 xdna->dev_info->ops->hwctx_fini(hwctx);
251 dev_exit:
252 drm_dev_exit(idx);
253 free_hwctx:
254 kfree(hwctx);
255 return ret;
256 }
257
amdxdna_drm_destroy_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)258 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
259 {
260 struct amdxdna_client *client = filp->driver_priv;
261 struct amdxdna_drm_destroy_hwctx *args = data;
262 struct amdxdna_dev *xdna = to_xdna_dev(dev);
263 struct amdxdna_hwctx *hwctx;
264 int ret = 0, idx;
265
266 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
267 return -EINVAL;
268
269 if (!drm_dev_enter(dev, &idx))
270 return -ENODEV;
271
272 mutex_lock(&xdna->dev_lock);
273 hwctx = xa_erase(&client->hwctx_xa, args->handle);
274 if (!hwctx) {
275 ret = -EINVAL;
276 XDNA_DBG(xdna, "PID %d HW context %d not exist",
277 client->pid, args->handle);
278 goto out;
279 }
280
281 /*
282 * The pushed jobs are handled by DRM scheduler during destroy.
283 * SRCU to synchronize with exec command ioctls.
284 */
285 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
286
287 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
288 out:
289 mutex_unlock(&xdna->dev_lock);
290 drm_dev_exit(idx);
291 return ret;
292 }
293
amdxdna_drm_config_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)294 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
295 {
296 struct amdxdna_client *client = filp->driver_priv;
297 struct amdxdna_drm_config_hwctx *args = data;
298 struct amdxdna_dev *xdna = to_xdna_dev(dev);
299 struct amdxdna_hwctx *hwctx;
300 u32 buf_size;
301 void *buf;
302 int ret;
303 u64 val;
304
305 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
306 return -EINVAL;
307
308 if (!xdna->dev_info->ops->hwctx_config)
309 return -EOPNOTSUPP;
310
311 val = args->param_val;
312 buf_size = args->param_val_size;
313
314 switch (args->param_type) {
315 case DRM_AMDXDNA_HWCTX_CONFIG_CU:
316 /* For those types that param_val is pointer */
317 if (buf_size > PAGE_SIZE) {
318 XDNA_ERR(xdna, "Config CU param buffer too large");
319 return -E2BIG;
320 }
321
322 /* Hwctx needs to keep buf */
323 buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
324 if (!buf)
325 return -ENOMEM;
326
327 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) {
328 kfree(buf);
329 return -EFAULT;
330 }
331
332 break;
333 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
334 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
335 /* For those types that param_val is a value */
336 buf = NULL;
337 buf_size = 0;
338 break;
339 default:
340 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type);
341 return -EINVAL;
342 }
343
344 guard(mutex)(&xdna->dev_lock);
345 hwctx = xa_load(&client->hwctx_xa, args->handle);
346 if (!hwctx) {
347 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
348 ret = -EINVAL;
349 goto free_buf;
350 }
351
352 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);
353
354 free_buf:
355 kfree(buf);
356 return ret;
357 }
358
amdxdna_hwctx_sync_debug_bo(struct amdxdna_client * client,u32 debug_bo_hdl)359 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
360 {
361 struct amdxdna_dev *xdna = client->xdna;
362 struct amdxdna_hwctx *hwctx;
363 struct amdxdna_gem_obj *abo;
364 struct drm_gem_object *gobj;
365 int ret;
366
367 if (!xdna->dev_info->ops->hwctx_sync_debug_bo)
368 return -EOPNOTSUPP;
369
370 gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl);
371 if (!gobj)
372 return -EINVAL;
373
374 abo = to_xdna_obj(gobj);
375 guard(mutex)(&xdna->dev_lock);
376 hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx);
377 if (!hwctx) {
378 ret = -EINVAL;
379 goto put_obj;
380 }
381
382 ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl);
383
384 put_obj:
385 drm_gem_object_put(gobj);
386 return ret;
387 }
388
389 static void
amdxdna_arg_bos_put(struct amdxdna_sched_job * job)390 amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
391 {
392 int i;
393
394 for (i = 0; i < job->bo_cnt; i++) {
395 if (!job->bos[i])
396 break;
397 drm_gem_object_put(job->bos[i]);
398 }
399 }
400
401 static int
amdxdna_arg_bos_lookup(struct amdxdna_client * client,struct amdxdna_sched_job * job,u32 * bo_hdls,u32 bo_cnt)402 amdxdna_arg_bos_lookup(struct amdxdna_client *client,
403 struct amdxdna_sched_job *job,
404 u32 *bo_hdls, u32 bo_cnt)
405 {
406 struct drm_gem_object *gobj;
407 int i, ret;
408
409 job->bo_cnt = bo_cnt;
410 for (i = 0; i < job->bo_cnt; i++) {
411 struct amdxdna_gem_obj *abo;
412
413 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]);
414 if (!gobj) {
415 ret = -ENOENT;
416 goto put_shmem_bo;
417 }
418 abo = to_xdna_obj(gobj);
419
420 mutex_lock(&abo->lock);
421 if (abo->pinned) {
422 mutex_unlock(&abo->lock);
423 job->bos[i] = gobj;
424 continue;
425 }
426
427 ret = amdxdna_gem_pin_nolock(abo);
428 if (ret) {
429 mutex_unlock(&abo->lock);
430 drm_gem_object_put(gobj);
431 goto put_shmem_bo;
432 }
433 abo->pinned = true;
434 mutex_unlock(&abo->lock);
435
436 job->bos[i] = gobj;
437 }
438
439 return 0;
440
441 put_shmem_bo:
442 amdxdna_arg_bos_put(job);
443 return ret;
444 }
445
amdxdna_sched_job_cleanup(struct amdxdna_sched_job * job)446 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
447 {
448 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release");
449 amdxdna_pm_suspend_put(job->hwctx->client->xdna);
450 amdxdna_arg_bos_put(job);
451 amdxdna_gem_put_obj(job->cmd_bo);
452 dma_fence_put(job->fence);
453 }
454
amdxdna_cmd_submit(struct amdxdna_client * client,struct amdxdna_drv_cmd * drv_cmd,u32 cmd_bo_hdl,u32 * arg_bo_hdls,u32 arg_bo_cnt,u32 hwctx_hdl,u64 * seq)455 int amdxdna_cmd_submit(struct amdxdna_client *client,
456 struct amdxdna_drv_cmd *drv_cmd,
457 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt,
458 u32 hwctx_hdl, u64 *seq)
459 {
460 struct amdxdna_dev *xdna = client->xdna;
461 struct amdxdna_sched_job *job;
462 struct amdxdna_hwctx *hwctx;
463 int ret, idx;
464
465 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
466 job = kzalloc_flex(*job, bos, arg_bo_cnt);
467 if (!job)
468 return -ENOMEM;
469
470 job->drv_cmd = drv_cmd;
471
472 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) {
473 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD);
474 if (!job->cmd_bo) {
475 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl);
476 ret = -EINVAL;
477 goto free_job;
478 }
479 }
480
481 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
482 if (ret) {
483 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret);
484 goto cmd_put;
485 }
486
487 ret = amdxdna_pm_resume_get(xdna);
488 if (ret) {
489 XDNA_ERR(xdna, "Resume failed, ret %d", ret);
490 goto put_bos;
491 }
492
493 idx = srcu_read_lock(&client->hwctx_srcu);
494 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl);
495 if (!hwctx) {
496 XDNA_DBG(xdna, "PID %d failed to get hwctx %d",
497 client->pid, hwctx_hdl);
498 ret = -EINVAL;
499 goto unlock_srcu;
500 }
501
502
503 job->hwctx = hwctx;
504 job->mm = current->mm;
505
506 job->fence = amdxdna_fence_create(hwctx);
507 if (!job->fence) {
508 XDNA_ERR(xdna, "Failed to create fence");
509 ret = -ENOMEM;
510 goto unlock_srcu;
511 }
512 kref_init(&job->refcnt);
513
514 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq);
515 if (ret)
516 goto put_fence;
517
518 /*
519 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated
520 * resource after synchronize_srcu(). The submitted jobs should be
521 * handled by the queue, for example DRM scheduler, in device layer.
522 * For here we can unlock SRCU.
523 */
524 srcu_read_unlock(&client->hwctx_srcu, idx);
525 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed");
526
527 return 0;
528
529 put_fence:
530 dma_fence_put(job->fence);
531 unlock_srcu:
532 srcu_read_unlock(&client->hwctx_srcu, idx);
533 amdxdna_pm_suspend_put(xdna);
534 put_bos:
535 amdxdna_arg_bos_put(job);
536 cmd_put:
537 amdxdna_gem_put_obj(job->cmd_bo);
538 free_job:
539 kfree(job);
540 return ret;
541 }
542
543 /*
544 * The submit command ioctl submits a command to firmware. One firmware command
545 * may contain multiple command BOs for processing as a whole.
546 * The command sequence number is returned which can be used for wait command ioctl.
547 */
amdxdna_drm_submit_execbuf(struct amdxdna_client * client,struct amdxdna_drm_exec_cmd * args)548 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client,
549 struct amdxdna_drm_exec_cmd *args)
550 {
551 struct amdxdna_dev *xdna = client->xdna;
552 u32 *arg_bo_hdls = NULL;
553 u32 cmd_bo_hdl;
554 int ret;
555
556 if (args->arg_count > MAX_ARG_COUNT) {
557 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count);
558 return -EINVAL;
559 }
560
561 /* Only support single command for now. */
562 if (args->cmd_count != 1) {
563 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count);
564 return -EINVAL;
565 }
566
567 cmd_bo_hdl = (u32)args->cmd_handles;
568 if (args->arg_count) {
569 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL);
570 if (!arg_bo_hdls)
571 return -ENOMEM;
572 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args),
573 args->arg_count * sizeof(u32));
574 if (ret) {
575 ret = -EFAULT;
576 goto free_cmd_bo_hdls;
577 }
578 }
579
580 ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls,
581 args->arg_count, args->hwctx, &args->seq);
582 if (ret)
583 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret);
584
585 free_cmd_bo_hdls:
586 kfree(arg_bo_hdls);
587 if (!ret)
588 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq);
589 return ret;
590 }
591
amdxdna_drm_submit_cmd_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)592 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
593 {
594 struct amdxdna_client *client = filp->driver_priv;
595 struct amdxdna_drm_exec_cmd *args = data;
596
597 if (args->ext || args->ext_flags)
598 return -EINVAL;
599
600 switch (args->type) {
601 case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
602 return amdxdna_drm_submit_execbuf(client, args);
603 }
604
605 XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
606 return -EINVAL;
607 }
608