1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_file.h>
10 #include <drm/drm_gem.h>
11 #include <drm/drm_gem_shmem_helper.h>
12 #include <drm/drm_print.h>
13 #include <drm/gpu_scheduler.h>
14 #include <linux/xarray.h>
15 #include <trace/events/amdxdna.h>
16
17 #include "amdxdna_ctx.h"
18 #include "amdxdna_gem.h"
19 #include "amdxdna_pci_drv.h"
20 #include "amdxdna_pm.h"
21
22 #define MAX_HWCTX_ID 255
23 #define MAX_ARG_COUNT 4095
24
25 struct amdxdna_fence {
26 struct dma_fence base;
27 spinlock_t lock; /* for base */
28 struct amdxdna_hwctx *hwctx;
29 };
30
amdxdna_fence_get_driver_name(struct dma_fence * fence)31 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence)
32 {
33 return KBUILD_MODNAME;
34 }
35
amdxdna_fence_get_timeline_name(struct dma_fence * fence)36 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence)
37 {
38 struct amdxdna_fence *xdna_fence;
39
40 xdna_fence = container_of(fence, struct amdxdna_fence, base);
41
42 return xdna_fence->hwctx->name;
43 }
44
45 static const struct dma_fence_ops fence_ops = {
46 .get_driver_name = amdxdna_fence_get_driver_name,
47 .get_timeline_name = amdxdna_fence_get_timeline_name,
48 };
49
amdxdna_fence_create(struct amdxdna_hwctx * hwctx)50 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
51 {
52 struct amdxdna_fence *fence;
53
54 fence = kzalloc_obj(*fence);
55 if (!fence)
56 return NULL;
57
58 fence->hwctx = hwctx;
59 spin_lock_init(&fence->lock);
60 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0);
61 return &fence->base;
62 }
63
amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx * hwctx)64 static void amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx *hwctx)
65 {
66 struct amdxdna_client *client = hwctx->client;
67 struct amdxdna_gem_obj *heap;
68 unsigned long heap_id;
69
70 mutex_lock(&client->mm_lock);
71 if (hwctx->last_attached_heap) {
72 xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 1,
73 hwctx->last_attached_heap) {
74 amdxdna_gem_unpin(heap);
75 drm_gem_object_put(to_gobj(heap));
76 }
77 }
78 mutex_unlock(&client->mm_lock);
79 }
80
amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx * hwctx,struct srcu_struct * ss)81 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
82 struct srcu_struct *ss)
83 {
84 struct amdxdna_client *client = hwctx->client;
85 struct amdxdna_dev *xdna = client->xdna;
86
87 synchronize_srcu(ss);
88
89 /* At this point, user is not able to submit new commands */
90 xdna->dev_info->ops->hwctx_fini(hwctx);
91
92 amdxdna_hwctx_release_expanded_heap(hwctx);
93 kfree(hwctx->name);
94 kfree(hwctx);
95 }
96
amdxdna_hwctx_walk(struct amdxdna_client * client,void * arg,int (* walk)(struct amdxdna_hwctx * hwctx,void * arg))97 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg,
98 int (*walk)(struct amdxdna_hwctx *hwctx, void *arg))
99 {
100 struct amdxdna_hwctx *hwctx;
101 unsigned long hwctx_id;
102 int ret = 0, idx;
103
104 idx = srcu_read_lock(&client->hwctx_srcu);
105 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
106 ret = walk(hwctx, arg);
107 if (ret)
108 break;
109 }
110 srcu_read_unlock(&client->hwctx_srcu, idx);
111
112 return ret;
113 }
114
amdxdna_cmd_get_payload(struct amdxdna_gem_obj * abo,u32 * size)115 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size)
116 {
117 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
118 u32 num_masks, count;
119
120 if (!cmd)
121 return NULL;
122
123 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
124 num_masks = 0;
125 else
126 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
127
128 if (size) {
129 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header);
130 if (unlikely(count <= num_masks ||
131 count * sizeof(u32) +
132 offsetof(struct amdxdna_cmd, data[0]) >
133 abo->mem.size)) {
134 *size = 0;
135 return NULL;
136 }
137 *size = (count - num_masks) * sizeof(u32);
138 }
139 return &cmd->data[num_masks];
140 }
141
amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj * abo)142 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
143 {
144 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
145 u32 num_masks, i;
146 u32 *cu_mask;
147
148 if (!cmd)
149 return INVALID_CU_IDX;
150
151 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
152 return INVALID_CU_IDX;
153
154 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
155 cu_mask = cmd->data;
156 for (i = 0; i < num_masks; i++) {
157 if (cu_mask[i])
158 return ffs(cu_mask[i]) - 1;
159 }
160
161 return INVALID_CU_IDX;
162 }
163
amdxdna_cmd_set_error(struct amdxdna_gem_obj * abo,struct amdxdna_sched_job * job,u32 cmd_idx,enum ert_cmd_state error_state,void * err_data,size_t size)164 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
165 struct amdxdna_sched_job *job, u32 cmd_idx,
166 enum ert_cmd_state error_state,
167 void *err_data, size_t size)
168 {
169 struct amdxdna_client *client = job->hwctx->client;
170 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
171 struct amdxdna_cmd_chain *cc = NULL;
172
173 if (!cmd)
174 return -ENOMEM;
175
176 cmd->header &= ~AMDXDNA_CMD_STATE;
177 cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state);
178
179 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) {
180 cc = amdxdna_cmd_get_payload(abo, NULL);
181 cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0;
182 abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_SHARE);
183 if (!abo)
184 return -EINVAL;
185 cmd = amdxdna_gem_vmap(abo);
186 if (!cmd) {
187 amdxdna_gem_put_obj(abo);
188 return -ENOMEM;
189 }
190 }
191
192 memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd));
193 if (err_data)
194 memcpy(cmd->data, err_data, min(size, abo->mem.size - sizeof(*cmd)));
195
196 if (cc)
197 amdxdna_gem_put_obj(abo);
198
199 return 0;
200 }
201
202 /*
203 * This should be called in close() and remove(). DO NOT call in other syscalls.
204 * This guarantee that when hwctx and resources will be released, if user
205 * doesn't call amdxdna_drm_destroy_hwctx_ioctl.
206 */
amdxdna_hwctx_remove_all(struct amdxdna_client * client)207 void amdxdna_hwctx_remove_all(struct amdxdna_client *client)
208 {
209 struct amdxdna_hwctx *hwctx;
210 unsigned long hwctx_id;
211
212 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
213 XDNA_DBG(client->xdna, "PID %d close HW context %d",
214 client->pid, hwctx->id);
215 xa_erase(&client->hwctx_xa, hwctx->id);
216 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
217 }
218 }
219
amdxdna_drm_create_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)220 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
221 {
222 struct amdxdna_client *client = filp->driver_priv;
223 struct amdxdna_drm_create_hwctx *args = data;
224 struct amdxdna_dev *xdna = to_xdna_dev(dev);
225 struct amdxdna_hwctx *hwctx;
226 int ret, idx;
227
228 if (args->ext || args->ext_flags)
229 return -EINVAL;
230
231 if (!xdna->dev_info->ops->hwctx_init)
232 return -EOPNOTSUPP;
233
234 hwctx = kzalloc_obj(*hwctx);
235 if (!hwctx)
236 return -ENOMEM;
237
238 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) {
239 XDNA_ERR(xdna, "Access QoS info failed");
240 kfree(hwctx);
241 return -EFAULT;
242 }
243
244 hwctx->client = client;
245 hwctx->fw_ctx_id = -1;
246 hwctx->num_tiles = args->num_tiles;
247 hwctx->umq_bo_hdl = args->umq_bo;
248 hwctx->doorbell_offset = AMDXDNA_INVALID_DOORBELL_OFFSET;
249 hwctx->mem_size = args->mem_size;
250 hwctx->max_opc = args->max_opc;
251
252 guard(mutex)(&xdna->dev_lock);
253
254 if (!drm_dev_enter(dev, &idx)) {
255 ret = -ENODEV;
256 goto free_hwctx;
257 }
258
259 ret = xdna->dev_info->ops->hwctx_init(hwctx);
260 if (ret) {
261 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
262 goto release_expanded_heap;
263 }
264
265 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id);
266 if (!hwctx->name) {
267 ret = -ENOMEM;
268 goto fini_hwctx;
269 }
270
271 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx,
272 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID),
273 &client->next_hwctxid, GFP_KERNEL);
274 if (ret < 0) {
275 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret);
276 goto free_name;
277 }
278
279 args->handle = hwctx->id;
280 args->syncobj_handle = hwctx->syncobj_hdl;
281 args->umq_doorbell = hwctx->doorbell_offset;
282
283 atomic64_set(&hwctx->job_submit_cnt, 0);
284 atomic64_set(&hwctx->job_free_cnt, 0);
285 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret);
286 drm_dev_exit(idx);
287 return 0;
288
289 free_name:
290 kfree(hwctx->name);
291 fini_hwctx:
292 xdna->dev_info->ops->hwctx_fini(hwctx);
293 release_expanded_heap:
294 amdxdna_hwctx_release_expanded_heap(hwctx);
295 drm_dev_exit(idx);
296 free_hwctx:
297 kfree(hwctx);
298 return ret;
299 }
300
amdxdna_drm_destroy_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)301 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
302 {
303 struct amdxdna_client *client = filp->driver_priv;
304 struct amdxdna_drm_destroy_hwctx *args = data;
305 struct amdxdna_dev *xdna = to_xdna_dev(dev);
306 struct amdxdna_hwctx *hwctx;
307 int ret = 0, idx;
308
309 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
310 return -EINVAL;
311
312 if (!drm_dev_enter(dev, &idx))
313 return -ENODEV;
314
315 mutex_lock(&xdna->client_lock);
316 mutex_lock(&xdna->dev_lock);
317 hwctx = xa_erase(&client->hwctx_xa, args->handle);
318 if (!hwctx) {
319 ret = -EINVAL;
320 XDNA_DBG(xdna, "PID %d HW context %d not exist",
321 client->pid, args->handle);
322 goto out;
323 }
324
325 /*
326 * The pushed jobs are handled by DRM scheduler during destroy.
327 * SRCU to synchronize with exec command ioctls.
328 */
329 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
330
331 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
332 out:
333 mutex_unlock(&xdna->dev_lock);
334 mutex_unlock(&xdna->client_lock);
335 drm_dev_exit(idx);
336 return ret;
337 }
338
amdxdna_drm_config_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)339 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
340 {
341 struct amdxdna_client *client = filp->driver_priv;
342 struct amdxdna_drm_config_hwctx *args = data;
343 struct amdxdna_dev *xdna = to_xdna_dev(dev);
344 struct amdxdna_hwctx *hwctx;
345 u32 buf_size;
346 void *buf;
347 int ret;
348 u64 val;
349
350 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
351 return -EINVAL;
352
353 if (!xdna->dev_info->ops->hwctx_config)
354 return -EOPNOTSUPP;
355
356 val = args->param_val;
357 buf_size = args->param_val_size;
358
359 switch (args->param_type) {
360 case DRM_AMDXDNA_HWCTX_CONFIG_CU:
361 /* For those types that param_val is pointer */
362 if (buf_size > PAGE_SIZE) {
363 XDNA_ERR(xdna, "Config CU param buffer too large");
364 return -E2BIG;
365 }
366
367 /* Hwctx needs to keep buf */
368 buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
369 if (!buf)
370 return -ENOMEM;
371
372 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) {
373 kfree(buf);
374 return -EFAULT;
375 }
376
377 break;
378 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
379 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
380 /* For those types that param_val is a value */
381 buf = NULL;
382 buf_size = 0;
383 break;
384 default:
385 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type);
386 return -EINVAL;
387 }
388
389 ret = amdxdna_pm_resume_get(xdna);
390 if (ret) {
391 XDNA_ERR(xdna, "Resume failed, ret %d", ret);
392 goto free_buf;
393 }
394
395 mutex_lock(&xdna->client_lock);
396 mutex_lock(&xdna->dev_lock);
397 hwctx = xa_load(&client->hwctx_xa, args->handle);
398 if (!hwctx) {
399 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
400 ret = -EINVAL;
401 goto unlock;
402 }
403
404 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);
405
406 unlock:
407 mutex_unlock(&xdna->dev_lock);
408 mutex_unlock(&xdna->client_lock);
409 amdxdna_pm_suspend_put(xdna);
410 free_buf:
411 kfree(buf);
412 return ret;
413 }
414
amdxdna_hwctx_sync_debug_bo(struct amdxdna_client * client,u32 debug_bo_hdl)415 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
416 {
417 struct amdxdna_dev *xdna = client->xdna;
418 struct amdxdna_hwctx *hwctx;
419 struct amdxdna_gem_obj *abo;
420 struct drm_gem_object *gobj;
421 int ret;
422
423 if (!xdna->dev_info->ops->hwctx_sync_debug_bo)
424 return -EOPNOTSUPP;
425
426 gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl);
427 if (!gobj)
428 return -EINVAL;
429
430 ret = amdxdna_pm_resume_get(xdna);
431 if (ret) {
432 XDNA_ERR(xdna, "Resume failed, ret %d", ret);
433 goto put_obj;
434 }
435
436 abo = to_xdna_obj(gobj);
437 mutex_lock(&xdna->client_lock);
438 mutex_lock(&xdna->dev_lock);
439 hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx);
440 if (!hwctx) {
441 ret = -EINVAL;
442 goto unlock;
443 }
444
445 ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl);
446
447 unlock:
448 mutex_unlock(&xdna->dev_lock);
449 mutex_unlock(&xdna->client_lock);
450 amdxdna_pm_suspend_put(xdna);
451 put_obj:
452 drm_gem_object_put(gobj);
453 return ret;
454 }
455
amdxdna_hwctx_expand_heap(struct amdxdna_hwctx * hwctx)456 static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx)
457 {
458 struct amdxdna_client *client = hwctx->client;
459 struct amdxdna_dev *xdna = client->xdna;
460 struct amdxdna_gem_obj *heap;
461 unsigned long heap_id, nid;
462 int ret = 0;
463
464 nid = hwctx->last_attached_heap + 1;
465 if (nid == client->dev_heap_nid)
466 goto out;
467
468 xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
469 nid, client->dev_heap_nid) {
470 drm_gem_object_get(to_gobj(heap));
471 ret = amdxdna_gem_pin(heap);
472 if (ret) {
473 drm_gem_object_put(to_gobj(heap));
474 break;
475 }
476
477 ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap);
478 if (ret) {
479 amdxdna_gem_unpin(heap);
480 drm_gem_object_put(to_gobj(heap));
481 break;
482 }
483
484 hwctx->last_attached_heap = heap_id;
485 }
486
487 out:
488 return ret;
489 }
490
amdxdna_update_heap(struct amdxdna_client * client,struct amdxdna_hwctx * hwctx)491 int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx)
492 {
493 unsigned long hwctx_id;
494 int ret;
495
496 ret = amdxdna_pm_resume_get_locked(client->xdna);
497 if (ret)
498 return ret;
499
500 mutex_lock(&client->mm_lock);
501
502 if (hwctx) {
503 ret = amdxdna_hwctx_expand_heap(hwctx);
504 } else {
505 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
506 ret = amdxdna_hwctx_expand_heap(hwctx);
507 if (ret)
508 break;
509 }
510 }
511 mutex_unlock(&client->mm_lock);
512
513 amdxdna_pm_suspend_put(client->xdna);
514
515 return ret;
516 }
517
518 static void
amdxdna_arg_bos_put(struct amdxdna_sched_job * job)519 amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
520 {
521 int i;
522
523 for (i = 0; i < job->bo_cnt; i++) {
524 if (!job->bos[i])
525 break;
526 drm_gem_object_put(job->bos[i]);
527 }
528 }
529
530 static int
amdxdna_arg_bos_lookup(struct amdxdna_client * client,struct amdxdna_sched_job * job,u32 * bo_hdls,u32 bo_cnt)531 amdxdna_arg_bos_lookup(struct amdxdna_client *client,
532 struct amdxdna_sched_job *job,
533 u32 *bo_hdls, u32 bo_cnt)
534 {
535 struct drm_gem_object *gobj;
536 int i, ret;
537
538 job->bo_cnt = bo_cnt;
539 for (i = 0; i < job->bo_cnt; i++) {
540 struct amdxdna_gem_obj *abo;
541
542 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]);
543 if (!gobj) {
544 ret = -ENOENT;
545 goto put_shmem_bo;
546 }
547 abo = to_xdna_obj(gobj);
548
549 mutex_lock(&abo->lock);
550 if (abo->pinned) {
551 mutex_unlock(&abo->lock);
552 job->bos[i] = gobj;
553 continue;
554 }
555
556 ret = amdxdna_gem_pin_nolock(abo);
557 if (ret) {
558 mutex_unlock(&abo->lock);
559 drm_gem_object_put(gobj);
560 goto put_shmem_bo;
561 }
562 abo->pinned = true;
563 mutex_unlock(&abo->lock);
564
565 job->bos[i] = gobj;
566 }
567
568 return 0;
569
570 put_shmem_bo:
571 amdxdna_arg_bos_put(job);
572 return ret;
573 }
574
amdxdna_sched_job_cleanup(struct amdxdna_sched_job * job)575 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
576 {
577 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release");
578 amdxdna_pm_suspend_put(job->hwctx->client->xdna);
579 amdxdna_arg_bos_put(job);
580 amdxdna_gem_put_obj(job->cmd_bo);
581 dma_fence_put(job->fence);
582 mmdrop(job->mm);
583 }
584
amdxdna_cmd_submit(struct amdxdna_client * client,struct amdxdna_drv_cmd * drv_cmd,u32 cmd_bo_hdl,u32 * arg_bo_hdls,u32 arg_bo_cnt,u32 hwctx_hdl,u64 * seq)585 int amdxdna_cmd_submit(struct amdxdna_client *client,
586 struct amdxdna_drv_cmd *drv_cmd,
587 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt,
588 u32 hwctx_hdl, u64 *seq)
589 {
590 struct amdxdna_dev *xdna = client->xdna;
591 struct amdxdna_sched_job *job;
592 struct amdxdna_hwctx *hwctx;
593 int ret, idx;
594
595 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
596
597 if (!xdna->dev_info->ops->cmd_submit)
598 return -EOPNOTSUPP;
599
600 job = kzalloc_flex(*job, bos, arg_bo_cnt);
601 if (!job)
602 return -ENOMEM;
603
604 job->drv_cmd = drv_cmd;
605
606 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) {
607 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_SHARE);
608 if (!job->cmd_bo) {
609 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl);
610 ret = -EINVAL;
611 goto free_job;
612 }
613 } else if (!drv_cmd) {
614 /*
615 * Only internal driver commands (drv_cmd != NULL) may omit a
616 * command BO. A user command submission with the invalid handle
617 * would leave job->cmd_bo NULL and later fault when the scheduler
618 * dereferences it in amdxdna_cmd_set_state().
619 */
620 XDNA_DBG(xdna, "Command BO handle required for user submission");
621 ret = -EINVAL;
622 goto free_job;
623 }
624
625 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
626 if (ret) {
627 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret);
628 goto cmd_put;
629 }
630
631 ret = amdxdna_pm_resume_get(xdna);
632 if (ret) {
633 XDNA_ERR(xdna, "Resume failed, ret %d", ret);
634 goto put_bos;
635 }
636
637 idx = srcu_read_lock(&client->hwctx_srcu);
638 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl);
639 if (!hwctx) {
640 XDNA_DBG(xdna, "PID %d failed to get hwctx %d",
641 client->pid, hwctx_hdl);
642 ret = -EINVAL;
643 goto unlock_srcu;
644 }
645
646 job->hwctx = hwctx;
647 job->mm = current->mm;
648 mmgrab(job->mm);
649
650 job->fence = amdxdna_fence_create(hwctx);
651 if (!job->fence) {
652 XDNA_ERR(xdna, "Failed to create fence");
653 ret = -ENOMEM;
654 goto unlock_srcu;
655 }
656 kref_init(&job->refcnt);
657
658 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq);
659 if (ret)
660 goto put_fence;
661
662 /*
663 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated
664 * resource after synchronize_srcu(). The submitted jobs should be
665 * handled by the queue, for example DRM scheduler, in device layer.
666 * For here we can unlock SRCU.
667 */
668 srcu_read_unlock(&client->hwctx_srcu, idx);
669 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed");
670
671 return 0;
672
673 put_fence:
674 dma_fence_put(job->fence);
675 unlock_srcu:
676 srcu_read_unlock(&client->hwctx_srcu, idx);
677 amdxdna_pm_suspend_put(xdna);
678 put_bos:
679 amdxdna_arg_bos_put(job);
680 cmd_put:
681 amdxdna_gem_put_obj(job->cmd_bo);
682 free_job:
683 if (job->mm)
684 mmdrop(job->mm);
685 kfree(job);
686 return ret;
687 }
688
689 /*
690 * The submit command ioctl submits a command to firmware. One firmware command
691 * may contain multiple command BOs for processing as a whole.
692 * The command sequence number is returned which can be used for wait command ioctl.
693 */
amdxdna_drm_submit_execbuf(struct amdxdna_client * client,struct amdxdna_drm_exec_cmd * args)694 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client,
695 struct amdxdna_drm_exec_cmd *args)
696 {
697 struct amdxdna_dev *xdna = client->xdna;
698 u32 *arg_bo_hdls = NULL;
699 u32 cmd_bo_hdl;
700 int ret;
701
702 if (args->arg_count > MAX_ARG_COUNT) {
703 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count);
704 return -EINVAL;
705 }
706
707 /* Only support single command for now. */
708 if (args->cmd_count != 1) {
709 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count);
710 return -EINVAL;
711 }
712
713 cmd_bo_hdl = (u32)args->cmd_handles;
714 if (args->arg_count) {
715 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL);
716 if (!arg_bo_hdls)
717 return -ENOMEM;
718 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args),
719 args->arg_count * sizeof(u32));
720 if (ret) {
721 ret = -EFAULT;
722 goto free_cmd_bo_hdls;
723 }
724 }
725
726 ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls,
727 args->arg_count, args->hwctx, &args->seq);
728 if (ret)
729 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret);
730
731 free_cmd_bo_hdls:
732 kfree(arg_bo_hdls);
733 if (!ret)
734 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq);
735 return ret;
736 }
737
amdxdna_drm_submit_cmd_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)738 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
739 {
740 struct amdxdna_client *client = filp->driver_priv;
741 struct amdxdna_drm_exec_cmd *args = data;
742
743 if (args->ext || args->ext_flags)
744 return -EINVAL;
745
746 trace_amdxdna_debug_point(current->comm, args->type, "job received");
747
748 switch (args->type) {
749 case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
750 return amdxdna_drm_submit_execbuf(client, args);
751 }
752
753 XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
754 return -EINVAL;
755 }
756
amdxdna_drm_wait_cmd_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)757 int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
758 {
759 struct amdxdna_client *client = filp->driver_priv;
760 struct amdxdna_dev *xdna = to_xdna_dev(dev);
761 struct amdxdna_drm_wait_cmd *args = data;
762 struct amdxdna_hwctx *hwctx;
763 int ret, idx;
764
765 XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu",
766 client->pid, args->hwctx, args->timeout, args->seq);
767
768 if (!xdna->dev_info->ops->cmd_wait)
769 return -EOPNOTSUPP;
770
771 idx = srcu_read_lock(&client->hwctx_srcu);
772 hwctx = xa_load(&client->hwctx_xa, args->hwctx);
773 if (!hwctx) {
774 XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx);
775 ret = -EINVAL;
776 goto unlock_ctx_srcu;
777 }
778
779 ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout);
780
781 XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d",
782 client->pid, args->hwctx, args->seq, ret);
783
784 trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user");
785
786 unlock_ctx_srcu:
787 srcu_read_unlock(&client->hwctx_srcu, idx);
788 return ret;
789 }
790