aie2_ctx.c (ac49797c1815d4f8f04b7053b2998c546283c89e) aie2_ctx.c (aac243092b707bb3018e951d470cc1a9bcbaba6c)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_device.h>
8#include <drm/drm_gem.h>
9#include <drm/drm_gem_shmem_helper.h>
10#include <drm/drm_print.h>
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_device.h>
8#include <drm/drm_gem.h>
9#include <drm/drm_gem_shmem_helper.h>
10#include <drm/drm_print.h>
11#include <drm/drm_syncobj.h>
12#include <linux/hmm.h>
11#include <linux/types.h>
13#include <linux/types.h>
14#include <trace/events/amdxdna.h>
12
15
16#include "aie2_msg_priv.h"
13#include "aie2_pci.h"
14#include "aie2_solver.h"
15#include "amdxdna_ctx.h"
16#include "amdxdna_gem.h"
17#include "amdxdna_mailbox.h"
18#include "amdxdna_pci_drv.h"
19
17#include "aie2_pci.h"
18#include "aie2_solver.h"
19#include "amdxdna_ctx.h"
20#include "amdxdna_gem.h"
21#include "amdxdna_mailbox.h"
22#include "amdxdna_pci_drv.h"
23
24bool force_cmdlist;
25module_param(force_cmdlist, bool, 0600);
26MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)");
27
28#define HWCTX_MAX_TIMEOUT 60000 /* milliseconds */
29
30static void aie2_job_release(struct kref *ref)
31{
32 struct amdxdna_sched_job *job;
33
34 job = container_of(ref, struct amdxdna_sched_job, refcnt);
35 amdxdna_sched_job_cleanup(job);
36 if (job->out_fence)
37 dma_fence_put(job->out_fence);
38 kfree(job);
39}
40
41static void aie2_job_put(struct amdxdna_sched_job *job)
42{
43 kref_put(&job->refcnt, aie2_job_release);
44}
45
46/* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */
47static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx,
48 struct drm_sched_job *bad_job)
49{
50 drm_sched_stop(&hwctx->priv->sched, bad_job);
51 aie2_destroy_context(xdna->dev_handle, hwctx);
52}
53
54static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)
55{
56 struct amdxdna_gem_obj *heap = hwctx->priv->heap;
57 int ret;
58
59 ret = aie2_create_context(xdna->dev_handle, hwctx);
60 if (ret) {
61 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret);
62 goto out;
63 }
64
65 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
66 heap->mem.userptr, heap->mem.size);
67 if (ret) {
68 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret);
69 goto out;
70 }
71
72 if (hwctx->status != HWCTX_STAT_READY) {
73 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status);
74 goto out;
75 }
76
77 ret = aie2_config_cu(hwctx);
78 if (ret) {
79 XDNA_ERR(xdna, "Config cu failed, ret %d", ret);
80 goto out;
81 }
82
83out:
84 drm_sched_start(&hwctx->priv->sched, 0);
85 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret);
86 return ret;
87}
88
89void aie2_restart_ctx(struct amdxdna_client *client)
90{
91 struct amdxdna_dev *xdna = client->xdna;
92 struct amdxdna_hwctx *hwctx;
93 int next = 0;
94
95 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
96 mutex_lock(&client->hwctx_lock);
97 idr_for_each_entry_continue(&client->hwctx_idr, hwctx, next) {
98 if (hwctx->status != HWCTX_STAT_STOP)
99 continue;
100
101 hwctx->status = hwctx->old_status;
102 XDNA_DBG(xdna, "Resetting %s", hwctx->name);
103 aie2_hwctx_restart(xdna, hwctx);
104 }
105 mutex_unlock(&client->hwctx_lock);
106}
107
108static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq)
109{
110 struct dma_fence *fence, *out_fence = NULL;
111 int ret;
112
113 fence = drm_syncobj_fence_get(hwctx->priv->syncobj);
114 if (!fence)
115 return NULL;
116
117 ret = dma_fence_chain_find_seqno(&fence, seq);
118 if (ret)
119 goto out;
120
121 out_fence = dma_fence_get(dma_fence_chain_contained(fence));
122
123out:
124 dma_fence_put(fence);
125 return out_fence;
126}
127
128static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx)
129{
130 struct dma_fence *fence;
131
132 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1);
133 if (!fence)
134 return;
135
136 dma_fence_wait(fence, false);
137 dma_fence_put(fence);
138}
139
140static void
141aie2_sched_notify(struct amdxdna_sched_job *job)
142{
143 struct dma_fence *fence = job->fence;
144
145 trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq);
146 job->hwctx->priv->completed++;
147 dma_fence_signal(fence);
148
149 up(&job->hwctx->priv->job_sem);
150 job->job_done = true;
151 dma_fence_put(fence);
152 mmput(job->mm);
153 aie2_job_put(job);
154}
155
156static int
157aie2_sched_resp_handler(void *handle, const u32 *data, size_t size)
158{
159 struct amdxdna_sched_job *job = handle;
160 struct amdxdna_gem_obj *cmd_abo;
161 u32 ret = 0;
162 u32 status;
163
164 cmd_abo = job->cmd_bo;
165
166 if (unlikely(!data))
167 goto out;
168
169 if (unlikely(size != sizeof(u32))) {
170 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);
171 ret = -EINVAL;
172 goto out;
173 }
174
175 status = *data;
176 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status);
177 if (status == AIE2_STATUS_SUCCESS)
178 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED);
179 else
180 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR);
181
182out:
183 aie2_sched_notify(job);
184 return ret;
185}
186
187static int
188aie2_sched_nocmd_resp_handler(void *handle, const u32 *data, size_t size)
189{
190 struct amdxdna_sched_job *job = handle;
191 u32 ret = 0;
192 u32 status;
193
194 if (unlikely(!data))
195 goto out;
196
197 if (unlikely(size != sizeof(u32))) {
198 ret = -EINVAL;
199 goto out;
200 }
201
202 status = *data;
203 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status);
204
205out:
206 aie2_sched_notify(job);
207 return ret;
208}
209
210static int
211aie2_sched_cmdlist_resp_handler(void *handle, const u32 *data, size_t size)
212{
213 struct amdxdna_sched_job *job = handle;
214 struct amdxdna_gem_obj *cmd_abo;
215 struct cmd_chain_resp *resp;
216 struct amdxdna_dev *xdna;
217 u32 fail_cmd_status;
218 u32 fail_cmd_idx;
219 u32 ret = 0;
220
221 cmd_abo = job->cmd_bo;
222 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) {
223 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);
224 ret = -EINVAL;
225 goto out;
226 }
227
228 resp = (struct cmd_chain_resp *)data;
229 xdna = job->hwctx->client->xdna;
230 XDNA_DBG(xdna, "Status 0x%x", resp->status);
231 if (resp->status == AIE2_STATUS_SUCCESS) {
232 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED);
233 goto out;
234 }
235
236 /* Slow path to handle error, read from ringbuf on BAR */
237 fail_cmd_idx = resp->fail_cmd_idx;
238 fail_cmd_status = resp->fail_cmd_status;
239 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x",
240 fail_cmd_idx, fail_cmd_status);
241
242 if (fail_cmd_status == AIE2_STATUS_SUCCESS) {
243 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);
244 ret = -EINVAL;
245 goto out;
246 }
247 amdxdna_cmd_set_state(cmd_abo, fail_cmd_status);
248
249 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) {
250 struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL);
251
252 cc->error_index = fail_cmd_idx;
253 if (cc->error_index >= cc->command_count)
254 cc->error_index = 0;
255 }
256out:
257 aie2_sched_notify(job);
258 return ret;
259}
260
261static struct dma_fence *
262aie2_sched_job_run(struct drm_sched_job *sched_job)
263{
264 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
265 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
266 struct amdxdna_hwctx *hwctx = job->hwctx;
267 struct dma_fence *fence;
268 int ret;
269
270 if (!mmget_not_zero(job->mm))
271 return ERR_PTR(-ESRCH);
272
273 kref_get(&job->refcnt);
274 fence = dma_fence_get(job->fence);
275
276 if (unlikely(!cmd_abo)) {
277 ret = aie2_sync_bo(hwctx, job, aie2_sched_nocmd_resp_handler);
278 goto out;
279 }
280
281 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW);
282
283 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN)
284 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler);
285 else if (force_cmdlist)
286 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler);
287 else
288 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler);
289
290out:
291 if (ret) {
292 dma_fence_put(job->fence);
293 aie2_job_put(job);
294 mmput(job->mm);
295 fence = ERR_PTR(ret);
296 }
297 trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq);
298
299 return fence;
300}
301
302static void aie2_sched_job_free(struct drm_sched_job *sched_job)
303{
304 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
305 struct amdxdna_hwctx *hwctx = job->hwctx;
306
307 trace_xdna_job(sched_job, hwctx->name, "job free", job->seq);
308 if (!job->job_done)
309 up(&hwctx->priv->job_sem);
310
311 drm_sched_job_cleanup(sched_job);
312 aie2_job_put(job);
313}
314
315static enum drm_gpu_sched_stat
316aie2_sched_job_timedout(struct drm_sched_job *sched_job)
317{
318 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
319 struct amdxdna_hwctx *hwctx = job->hwctx;
320 struct amdxdna_dev *xdna;
321
322 xdna = hwctx->client->xdna;
323 trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq);
324 mutex_lock(&xdna->dev_lock);
325 aie2_hwctx_stop(xdna, hwctx, sched_job);
326
327 aie2_hwctx_restart(xdna, hwctx);
328 mutex_unlock(&xdna->dev_lock);
329
330 return DRM_GPU_SCHED_STAT_NOMINAL;
331}
332
333const struct drm_sched_backend_ops sched_ops = {
334 .run_job = aie2_sched_job_run,
335 .free_job = aie2_sched_job_free,
336 .timedout_job = aie2_sched_job_timedout,
337};
338
20static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx)
21{
22 struct amdxdna_dev *xdna = hwctx->client->xdna;
23 struct amdxdna_dev_hdl *ndev;
24 int start, end, first, last;
25 u32 width = 1, entries = 0;
26 int i;
27

--- 93 unchanged lines hidden (view full) ---

121 struct amdxdna_dev *xdna = hwctx->client->xdna;
122 int ret;
123
124 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
125 if (ret)
126 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
127}
128
339static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx)
340{
341 struct amdxdna_dev *xdna = hwctx->client->xdna;
342 struct amdxdna_dev_hdl *ndev;
343 int start, end, first, last;
344 u32 width = 1, entries = 0;
345 int i;
346

--- 93 unchanged lines hidden (view full) ---

440 struct amdxdna_dev *xdna = hwctx->client->xdna;
441 int ret;
442
443 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
444 if (ret)
445 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
446}
447
448static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx)
449{
450 struct amdxdna_dev *xdna = hwctx->client->xdna;
451 struct drm_file *filp = hwctx->client->filp;
452 struct drm_syncobj *syncobj;
453 u32 hdl;
454 int ret;
455
456 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE;
457
458 ret = drm_syncobj_create(&syncobj, 0, NULL);
459 if (ret) {
460 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret);
461 return ret;
462 }
463 ret = drm_syncobj_get_handle(filp, syncobj, &hdl);
464 if (ret) {
465 drm_syncobj_put(syncobj);
466 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret);
467 return ret;
468 }
469 hwctx->priv->syncobj = syncobj;
470 hwctx->syncobj_hdl = hdl;
471
472 return 0;
473}
474
475static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx)
476{
477 /*
478 * The syncobj_hdl is owned by user space and will be cleaned up
479 * separately.
480 */
481 drm_syncobj_put(hwctx->priv->syncobj);
482}
483
129int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
130{
131 struct amdxdna_client *client = hwctx->client;
132 struct amdxdna_dev *xdna = client->xdna;
484int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
485{
486 struct amdxdna_client *client = hwctx->client;
487 struct amdxdna_dev *xdna = client->xdna;
488 struct drm_gpu_scheduler *sched;
133 struct amdxdna_hwctx_priv *priv;
134 struct amdxdna_gem_obj *heap;
489 struct amdxdna_hwctx_priv *priv;
490 struct amdxdna_gem_obj *heap;
135 int ret;
491 int i, ret;
136
137 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL);
138 if (!priv)
139 return -ENOMEM;
140 hwctx->priv = priv;
141
142 mutex_lock(&client->mm_lock);
143 heap = client->dev_heap;
144 if (!heap) {
145 XDNA_ERR(xdna, "The client dev heap object not exist");
146 mutex_unlock(&client->mm_lock);
147 ret = -ENOENT;
148 goto free_priv;
149 }
150 drm_gem_object_get(to_gobj(heap));
151 mutex_unlock(&client->mm_lock);
152 priv->heap = heap;
492
493 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL);
494 if (!priv)
495 return -ENOMEM;
496 hwctx->priv = priv;
497
498 mutex_lock(&client->mm_lock);
499 heap = client->dev_heap;
500 if (!heap) {
501 XDNA_ERR(xdna, "The client dev heap object not exist");
502 mutex_unlock(&client->mm_lock);
503 ret = -ENOENT;
504 goto free_priv;
505 }
506 drm_gem_object_get(to_gobj(heap));
507 mutex_unlock(&client->mm_lock);
508 priv->heap = heap;
509 sema_init(&priv->job_sem, HWCTX_MAX_CMDS);
153
154 ret = amdxdna_gem_pin(heap);
155 if (ret) {
156 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret);
157 goto put_heap;
158 }
159
510
511 ret = amdxdna_gem_pin(heap);
512 if (ret) {
513 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret);
514 goto put_heap;
515 }
516
517 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
518 struct amdxdna_gem_obj *abo;
519 struct amdxdna_drm_create_bo args = {
520 .flags = 0,
521 .type = AMDXDNA_BO_DEV,
522 .vaddr = 0,
523 .size = MAX_CHAIN_CMDBUF_SIZE,
524 };
525
526 abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp, true);
527 if (IS_ERR(abo)) {
528 ret = PTR_ERR(abo);
529 goto free_cmd_bufs;
530 }
531
532 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx",
533 i, abo->mem.dev_addr, abo->mem.size);
534 priv->cmd_buf[i] = abo;
535 }
536
537 sched = &priv->sched;
538 mutex_init(&priv->io_lock);
539
540 fs_reclaim_acquire(GFP_KERNEL);
541 might_lock(&priv->io_lock);
542 fs_reclaim_release(GFP_KERNEL);
543
544 ret = drm_sched_init(sched, &sched_ops, NULL, DRM_SCHED_PRIORITY_COUNT,
545 HWCTX_MAX_CMDS, 0, msecs_to_jiffies(HWCTX_MAX_TIMEOUT),
546 NULL, NULL, hwctx->name, xdna->ddev.dev);
547 if (ret) {
548 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
549 goto free_cmd_bufs;
550 }
551
552 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,
553 &sched, 1, NULL);
554 if (ret) {
555 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret);
556 goto free_sched;
557 }
558
160 ret = aie2_hwctx_col_list(hwctx);
161 if (ret) {
162 XDNA_ERR(xdna, "Create col list failed, ret %d", ret);
559 ret = aie2_hwctx_col_list(hwctx);
560 if (ret) {
561 XDNA_ERR(xdna, "Create col list failed, ret %d", ret);
163 goto unpin;
562 goto free_entity;
164 }
165
166 ret = aie2_alloc_resource(hwctx);
167 if (ret) {
168 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret);
169 goto free_col_list;
170 }
171
172 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
173 heap->mem.userptr, heap->mem.size);
174 if (ret) {
175 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret);
176 goto release_resource;
177 }
563 }
564
565 ret = aie2_alloc_resource(hwctx);
566 if (ret) {
567 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret);
568 goto free_col_list;
569 }
570
571 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
572 heap->mem.userptr, heap->mem.size);
573 if (ret) {
574 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret);
575 goto release_resource;
576 }
577
578 ret = aie2_ctx_syncobj_create(hwctx);
579 if (ret) {
580 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret);
581 goto release_resource;
582 }
583
178 hwctx->status = HWCTX_STAT_INIT;
179
180 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name);
181
182 return 0;
183
184release_resource:
185 aie2_release_resource(hwctx);
186free_col_list:
187 kfree(hwctx->col_list);
584 hwctx->status = HWCTX_STAT_INIT;
585
586 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name);
587
588 return 0;
589
590release_resource:
591 aie2_release_resource(hwctx);
592free_col_list:
593 kfree(hwctx->col_list);
188unpin:
594free_entity:
595 drm_sched_entity_destroy(&priv->entity);
596free_sched:
597 drm_sched_fini(&priv->sched);
598free_cmd_bufs:
599 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
600 if (!priv->cmd_buf[i])
601 continue;
602 drm_gem_object_put(to_gobj(priv->cmd_buf[i]));
603 }
189 amdxdna_gem_unpin(heap);
190put_heap:
191 drm_gem_object_put(to_gobj(heap));
192free_priv:
193 kfree(priv);
194 return ret;
195}
196
197void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
198{
604 amdxdna_gem_unpin(heap);
605put_heap:
606 drm_gem_object_put(to_gobj(heap));
607free_priv:
608 kfree(priv);
609 return ret;
610}
611
612void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
613{
614 struct amdxdna_dev *xdna;
615 int idx;
616
617 xdna = hwctx->client->xdna;
618 drm_sched_wqueue_stop(&hwctx->priv->sched);
619
620 /* Now, scheduler will not send command to device. */
199 aie2_release_resource(hwctx);
200
621 aie2_release_resource(hwctx);
622
623 /*
624 * All submitted commands are aborted.
625 * Restart scheduler queues to cleanup jobs. The amdxdna_sched_job_run()
626 * will return NODEV if it is called.
627 */
628 drm_sched_wqueue_start(&hwctx->priv->sched);
629
630 aie2_hwctx_wait_for_idle(hwctx);
631 drm_sched_entity_destroy(&hwctx->priv->entity);
632 drm_sched_fini(&hwctx->priv->sched);
633 aie2_ctx_syncobj_destroy(hwctx);
634
635 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq);
636
637 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++)
638 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx]));
201 amdxdna_gem_unpin(hwctx->priv->heap);
202 drm_gem_object_put(to_gobj(hwctx->priv->heap));
203
639 amdxdna_gem_unpin(hwctx->priv->heap);
640 drm_gem_object_put(to_gobj(hwctx->priv->heap));
641
642 mutex_destroy(&hwctx->priv->io_lock);
204 kfree(hwctx->col_list);
205 kfree(hwctx->priv);
206 kfree(hwctx->cus);
207}
208
209static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size)
210{
211 struct amdxdna_hwctx_param_config_cu *config = buf;

--- 50 unchanged lines hidden (view full) ---

262 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
263 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
264 return -EOPNOTSUPP;
265 default:
266 XDNA_DBG(xdna, "Not supported type %d", type);
267 return -EOPNOTSUPP;
268 }
269}
643 kfree(hwctx->col_list);
644 kfree(hwctx->priv);
645 kfree(hwctx->cus);
646}
647
648static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size)
649{
650 struct amdxdna_hwctx_param_config_cu *config = buf;

--- 50 unchanged lines hidden (view full) ---

701 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
702 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
703 return -EOPNOTSUPP;
704 default:
705 XDNA_DBG(xdna, "Not supported type %d", type);
706 return -EOPNOTSUPP;
707 }
708}
709
710static int aie2_populate_range(struct amdxdna_gem_obj *abo)
711{
712 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
713 struct mm_struct *mm = abo->mem.notifier.mm;
714 struct hmm_range range = { 0 };
715 unsigned long timeout;
716 int ret;
717
718 XDNA_INFO_ONCE(xdna, "populate memory range %llx size %lx",
719 abo->mem.userptr, abo->mem.size);
720 range.notifier = &abo->mem.notifier;
721 range.start = abo->mem.userptr;
722 range.end = abo->mem.userptr + abo->mem.size;
723 range.hmm_pfns = abo->mem.pfns;
724 range.default_flags = HMM_PFN_REQ_FAULT;
725
726 if (!mmget_not_zero(mm))
727 return -EFAULT;
728
729 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
730again:
731 range.notifier_seq = mmu_interval_read_begin(&abo->mem.notifier);
732 mmap_read_lock(mm);
733 ret = hmm_range_fault(&range);
734 mmap_read_unlock(mm);
735 if (ret) {
736 if (time_after(jiffies, timeout)) {
737 ret = -ETIME;
738 goto put_mm;
739 }
740
741 if (ret == -EBUSY)
742 goto again;
743
744 goto put_mm;
745 }
746
747 down_read(&xdna->notifier_lock);
748 if (mmu_interval_read_retry(&abo->mem.notifier, range.notifier_seq)) {
749 up_read(&xdna->notifier_lock);
750 goto again;
751 }
752 abo->mem.map_invalid = false;
753 up_read(&xdna->notifier_lock);
754
755put_mm:
756 mmput(mm);
757 return ret;
758}
759
760int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq)
761{
762 struct amdxdna_dev *xdna = hwctx->client->xdna;
763 struct ww_acquire_ctx acquire_ctx;
764 struct dma_fence_chain *chain;
765 struct amdxdna_gem_obj *abo;
766 unsigned long timeout = 0;
767 int ret, i;
768
769 ret = down_interruptible(&hwctx->priv->job_sem);
770 if (ret) {
771 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret);
772 return ret;
773 }
774
775 chain = dma_fence_chain_alloc();
776 if (!chain) {
777 XDNA_ERR(xdna, "Alloc fence chain failed");
778 ret = -ENOMEM;
779 goto up_sem;
780 }
781
782 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx);
783 if (ret) {
784 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret);
785 goto free_chain;
786 }
787
788retry:
789 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx);
790 if (ret) {
791 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret);
792 goto cleanup_job;
793 }
794
795 for (i = 0; i < job->bo_cnt; i++) {
796 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1);
797 if (ret) {
798 XDNA_WARN(xdna, "Failed to reserve fences %d", ret);
799 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);
800 goto cleanup_job;
801 }
802 }
803
804 down_read(&xdna->notifier_lock);
805 for (i = 0; i < job->bo_cnt; i++) {
806 abo = to_xdna_obj(job->bos[i]);
807 if (abo->mem.map_invalid) {
808 up_read(&xdna->notifier_lock);
809 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);
810 if (!timeout) {
811 timeout = jiffies +
812 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
813 } else if (time_after(jiffies, timeout)) {
814 ret = -ETIME;
815 goto cleanup_job;
816 }
817
818 ret = aie2_populate_range(abo);
819 if (ret)
820 goto cleanup_job;
821 goto retry;
822 }
823 }
824
825 mutex_lock(&hwctx->priv->io_lock);
826 drm_sched_job_arm(&job->base);
827 job->out_fence = dma_fence_get(&job->base.s_fence->finished);
828 for (i = 0; i < job->bo_cnt; i++)
829 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE);
830 job->seq = hwctx->priv->seq++;
831 kref_get(&job->refcnt);
832 drm_sched_entity_push_job(&job->base);
833
834 *seq = job->seq;
835 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq);
836 mutex_unlock(&hwctx->priv->io_lock);
837
838 up_read(&xdna->notifier_lock);
839 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);
840
841 aie2_job_put(job);
842
843 return 0;
844
845cleanup_job:
846 drm_sched_job_cleanup(&job->base);
847free_chain:
848 dma_fence_chain_free(chain);
849up_sem:
850 up(&hwctx->priv->job_sem);
851 job->job_done = true;
852 return ret;
853}
854
855void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo,
856 unsigned long cur_seq)
857{
858 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
859 struct drm_gem_object *gobj = to_gobj(abo);
860 long ret;
861
862 down_write(&xdna->notifier_lock);
863 abo->mem.map_invalid = true;
864 mmu_interval_set_seq(&abo->mem.notifier, cur_seq);
865 up_write(&xdna->notifier_lock);
866 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP,
867 true, MAX_SCHEDULE_TIMEOUT);
868 if (!ret || ret == -ERESTARTSYS)
869 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret);
870}