1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_cache.h>
8 #include <drm/drm_device.h>
9 #include <drm/drm_gem.h>
10 #include <drm/drm_gem_shmem_helper.h>
11 #include <drm/drm_print.h>
12 #include <drm/gpu_scheduler.h>
13 #include <linux/bitfield.h>
14 #include <linux/errno.h>
15 #include <linux/pci.h>
16 #include <linux/types.h>
17 #include <linux/xarray.h>
18
19 #include "aie2_msg_priv.h"
20 #include "aie2_pci.h"
21 #include "amdxdna_ctx.h"
22 #include "amdxdna_gem.h"
23 #include "amdxdna_mailbox.h"
24 #include "amdxdna_mailbox_helper.h"
25 #include "amdxdna_pci_drv.h"
26
27 #define DECLARE_AIE2_MSG(name, op) \
28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE)
29
30 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops)
31
aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl * ndev,struct xdna_mailbox_msg * msg)32 static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
33 struct xdna_mailbox_msg *msg)
34 {
35 struct amdxdna_dev *xdna = ndev->xdna;
36 struct xdna_notify *hdl = msg->handle;
37 int ret;
38
39 if (!ndev->mgmt_chann)
40 return -ENODEV;
41
42 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
43 if (ret == -ETIME)
44 aie2_destroy_mgmt_chann(ndev);
45
46 if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) {
47 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
48 msg->opcode, *hdl->data);
49 ret = -EINVAL;
50 }
51
52 return ret;
53 }
54
aie2_alloc_msg_buffer(struct amdxdna_dev_hdl * ndev,u32 * size,dma_addr_t * dma_addr)55 void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
56 dma_addr_t *dma_addr)
57 {
58 struct amdxdna_dev *xdna = ndev->xdna;
59 int order;
60
61 *size = max(*size, SZ_8K);
62 order = get_order(*size);
63 if (order > MAX_PAGE_ORDER)
64 return NULL;
65 *size = PAGE_SIZE << order;
66
67 return dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
68 DMA_FROM_DEVICE, GFP_KERNEL);
69 }
70
aie2_suspend_fw(struct amdxdna_dev_hdl * ndev)71 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
72 {
73 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
74 int ret;
75
76 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
77 if (ret) {
78 XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret);
79 return ret;
80 }
81
82 return aie2_psp_waitmode_poll(ndev->psp_hdl);
83 }
84
aie2_resume_fw(struct amdxdna_dev_hdl * ndev)85 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
86 {
87 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME);
88
89 return aie2_send_mgmt_msg_wait(ndev, &msg);
90 }
91
aie2_set_runtime_cfg(struct amdxdna_dev_hdl * ndev,u32 type,u64 value)92 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
93 {
94 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
95 int ret;
96
97 req.type = type;
98 req.value = value;
99
100 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
101 if (ret) {
102 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret);
103 return ret;
104 }
105
106 return 0;
107 }
108
aie2_get_runtime_cfg(struct amdxdna_dev_hdl * ndev,u32 type,u64 * value)109 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
110 {
111 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
112 int ret;
113
114 req.type = type;
115 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
116 if (ret) {
117 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret);
118 return ret;
119 }
120
121 *value = resp.value;
122 return 0;
123 }
124
aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl * ndev,u16 pasid)125 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
126 {
127 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
128
129 req.pasid = pasid;
130
131 return aie2_send_mgmt_msg_wait(ndev, &msg);
132 }
133
aie2_query_aie_version(struct amdxdna_dev_hdl * ndev,struct aie_version * version)134 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version)
135 {
136 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
137 struct amdxdna_dev *xdna = ndev->xdna;
138 int ret;
139
140 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
141 if (ret)
142 return ret;
143
144 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed",
145 resp.major, resp.minor);
146
147 version->major = resp.major;
148 version->minor = resp.minor;
149
150 return 0;
151 }
152
aie2_query_aie_metadata(struct amdxdna_dev_hdl * ndev,struct aie_metadata * metadata)153 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata)
154 {
155 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
156 int ret;
157
158 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
159 if (ret)
160 return ret;
161
162 metadata->size = resp.info.size;
163 metadata->cols = resp.info.cols;
164 metadata->rows = resp.info.rows;
165
166 metadata->version.major = resp.info.major;
167 metadata->version.minor = resp.info.minor;
168
169 metadata->core.row_count = resp.info.core_rows;
170 metadata->core.row_start = resp.info.core_row_start;
171 metadata->core.dma_channel_count = resp.info.core_dma_channels;
172 metadata->core.lock_count = resp.info.core_locks;
173 metadata->core.event_reg_count = resp.info.core_events;
174
175 metadata->mem.row_count = resp.info.mem_rows;
176 metadata->mem.row_start = resp.info.mem_row_start;
177 metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
178 metadata->mem.lock_count = resp.info.mem_locks;
179 metadata->mem.event_reg_count = resp.info.mem_events;
180
181 metadata->shim.row_count = resp.info.shim_rows;
182 metadata->shim.row_start = resp.info.shim_row_start;
183 metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
184 metadata->shim.lock_count = resp.info.shim_locks;
185 metadata->shim.event_reg_count = resp.info.shim_events;
186
187 return 0;
188 }
189
aie2_query_firmware_version(struct amdxdna_dev_hdl * ndev,struct amdxdna_fw_ver * fw_ver)190 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
191 struct amdxdna_fw_ver *fw_ver)
192 {
193 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
194 int ret;
195
196 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
197 if (ret)
198 return ret;
199
200 fw_ver->major = resp.major;
201 fw_ver->minor = resp.minor;
202 fw_ver->sub = resp.sub;
203 fw_ver->build = resp.build;
204
205 return 0;
206 }
207
aie2_destroy_context_req(struct amdxdna_dev_hdl * ndev,u32 id)208 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
209 {
210 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
211 struct amdxdna_dev *xdna = ndev->xdna;
212 int ret;
213
214 req.context_id = id;
215 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
216 if (ret && ret != -ENODEV)
217 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
218 else if (ret == -ENODEV)
219 XDNA_DBG(xdna, "Destroy context: device already stopped");
220
221 return ret;
222 }
223
aie2_get_context_priority(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)224 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
225 struct amdxdna_hwctx *hwctx)
226 {
227 if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT))
228 return PRIORITY_HIGH;
229
230 switch (hwctx->qos.priority) {
231 case AMDXDNA_QOS_REALTIME_PRIORITY:
232 return PRIORITY_REALTIME;
233 case AMDXDNA_QOS_HIGH_PRIORITY:
234 return PRIORITY_HIGH;
235 case AMDXDNA_QOS_NORMAL_PRIORITY:
236 return PRIORITY_NORMAL;
237 case AMDXDNA_QOS_LOW_PRIORITY:
238 return PRIORITY_LOW;
239 default:
240 return PRIORITY_HIGH;
241 }
242 }
243
aie2_create_context(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)244 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
245 {
246 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
247 struct amdxdna_dev *xdna = ndev->xdna;
248 struct xdna_mailbox_chann_res x2i;
249 struct xdna_mailbox_chann_res i2x;
250 struct cq_pair *cq_pair;
251 u32 intr_reg;
252 int ret;
253
254 req.aie_type = 1;
255 req.start_col = hwctx->start_col;
256 req.num_col = hwctx->num_col;
257 req.num_unused_col = hwctx->num_unused_col;
258 req.num_cq_pairs_requested = 1;
259 req.pasid = hwctx->client->pasid;
260 req.context_priority = aie2_get_context_priority(ndev, hwctx);
261
262 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
263 if (ret)
264 return ret;
265
266 hwctx->fw_ctx_id = resp.context_id;
267 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1))
268 return -EINVAL;
269
270 if (ndev->force_preempt_enabled) {
271 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id);
272 if (ret) {
273 XDNA_ERR(xdna, "failed to enable force preempt %d", ret);
274 goto del_ctx_req;
275 }
276 }
277
278 cq_pair = &resp.cq_pair[0];
279 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr);
280 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr);
281 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr);
282 x2i.rb_size = cq_pair->x2i_q.buf_size;
283
284 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr);
285 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr);
286 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr);
287 i2x.rb_size = cq_pair->i2x_q.buf_size;
288
289 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id);
290 if (ret == -EINVAL) {
291 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret);
292 goto del_ctx_req;
293 }
294
295 intr_reg = i2x.mb_head_ptr_reg + 4;
296 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox);
297 if (!hwctx->priv->mbox_chann) {
298 XDNA_ERR(xdna, "Not able to create channel");
299 ret = -EINVAL;
300 goto del_ctx_req;
301 }
302
303 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x,
304 intr_reg, ret);
305 if (ret) {
306 XDNA_ERR(xdna, "Not able to create channel");
307 ret = -EINVAL;
308 goto free_channel;
309 }
310 ndev->hwctx_num++;
311
312 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id);
313 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid);
314
315 return 0;
316
317 free_channel:
318 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
319 del_ctx_req:
320 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
321 return ret;
322 }
323
aie2_destroy_context(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)324 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
325 {
326 struct amdxdna_dev *xdna = ndev->xdna;
327 int ret;
328
329 if (!hwctx->priv->mbox_chann)
330 return 0;
331
332 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann);
333 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
334 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
335 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id);
336 hwctx->priv->mbox_chann = NULL;
337 hwctx->fw_ctx_id = -1;
338 ndev->hwctx_num--;
339
340 return ret;
341 }
342
aie2_map_host_buf(struct amdxdna_dev_hdl * ndev,u32 context_id,u64 addr,u64 size)343 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
344 {
345 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
346 struct amdxdna_dev *xdna = ndev->xdna;
347 int ret;
348
349 req.context_id = context_id;
350 req.buf_addr = addr;
351 req.buf_size = size;
352 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
353 if (ret)
354 return ret;
355
356 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx",
357 context_id, addr, size);
358
359 return 0;
360 }
361
amdxdna_hwctx_col_map(struct amdxdna_hwctx * hwctx,void * arg)362 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
363 {
364 u32 *bitmap = arg;
365
366 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col);
367
368 return 0;
369 }
370
aie2_query_status(struct amdxdna_dev_hdl * ndev,char __user * buf,u32 size,u32 * cols_filled)371 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
372 u32 size, u32 *cols_filled)
373 {
374 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
375 struct amdxdna_dev *xdna = ndev->xdna;
376 u32 buf_sz = size, aie_bitmap = 0;
377 struct amdxdna_client *client;
378 dma_addr_t dma_addr;
379 u8 *buff_addr;
380 int ret;
381
382 buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
383 if (!buff_addr)
384 return -ENOMEM;
385
386 /* Go through each hardware context and mark the AIE columns that are active */
387 list_for_each_entry(client, &xdna->client_list, node)
388 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map);
389
390 *cols_filled = 0;
391 req.dump_buff_addr = dma_addr;
392 req.dump_buff_size = buf_sz;
393 req.num_cols = hweight32(aie_bitmap);
394 req.aie_bitmap = aie_bitmap;
395
396 drm_clflush_virt_range(buff_addr, size); /* device can access */
397 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
398 if (ret) {
399 XDNA_ERR(xdna, "Error during NPU query, status %d", ret);
400 goto fail;
401 }
402
403 XDNA_DBG(xdna, "Query NPU status completed");
404
405 if (size < resp.size) {
406 ret = -EINVAL;
407 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
408 goto fail;
409 }
410
411 if (copy_to_user(buf, buff_addr, resp.size)) {
412 ret = -EFAULT;
413 XDNA_ERR(xdna, "Failed to copy NPU status to user space");
414 goto fail;
415 }
416
417 *cols_filled = aie_bitmap;
418
419 fail:
420 aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr);
421 return ret;
422 }
423
aie2_query_telemetry(struct amdxdna_dev_hdl * ndev,char __user * buf,u32 size,struct amdxdna_drm_query_telemetry_header * header)424 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
425 char __user *buf, u32 size,
426 struct amdxdna_drm_query_telemetry_header *header)
427 {
428 DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
429 struct amdxdna_dev *xdna = ndev->xdna;
430 dma_addr_t dma_addr;
431 u32 buf_sz = size;
432 u8 *addr;
433 int ret;
434
435 if (header->type >= MAX_TELEMETRY_TYPE)
436 return -EINVAL;
437
438 addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
439 if (!addr)
440 return -ENOMEM;
441
442 req.buf_addr = dma_addr;
443 req.buf_size = buf_sz;
444 req.type = header->type;
445
446 drm_clflush_virt_range(addr, size); /* device can access */
447 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
448 if (ret) {
449 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret);
450 goto free_buf;
451 }
452
453 if (size < resp.size) {
454 ret = -EINVAL;
455 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
456 goto free_buf;
457 }
458
459 if (copy_to_user(buf, addr, resp.size)) {
460 ret = -EFAULT;
461 XDNA_ERR(xdna, "Failed to copy telemetry to user space");
462 goto free_buf;
463 }
464
465 header->major = resp.major;
466 header->minor = resp.minor;
467
468 free_buf:
469 aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr);
470 return ret;
471 }
472
aie2_register_asyn_event_msg(struct amdxdna_dev_hdl * ndev,dma_addr_t addr,u32 size,void * handle,int (* cb)(void *,void __iomem *,size_t))473 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size,
474 void *handle, int (*cb)(void*, void __iomem *, size_t))
475 {
476 struct async_event_msg_req req = { 0 };
477 struct xdna_mailbox_msg msg = {
478 .send_data = (u8 *)&req,
479 .send_size = sizeof(req),
480 .handle = handle,
481 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG,
482 .notify_cb = cb,
483 };
484
485 req.buf_addr = addr;
486 req.buf_size = size;
487
488 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size);
489 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT);
490 }
491
aie2_config_cu(struct amdxdna_hwctx * hwctx,int (* notify_cb)(void *,void __iomem *,size_t))492 int aie2_config_cu(struct amdxdna_hwctx *hwctx,
493 int (*notify_cb)(void *, void __iomem *, size_t))
494 {
495 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
496 struct amdxdna_dev *xdna = hwctx->client->xdna;
497 u32 shift = xdna->dev_info->dev_mem_buf_shift;
498 struct config_cu_req req = { 0 };
499 struct xdna_mailbox_msg msg;
500 struct drm_gem_object *gobj;
501 struct amdxdna_gem_obj *abo;
502 int i;
503
504 if (!chann)
505 return -ENODEV;
506
507 if (!hwctx->cus)
508 return 0;
509
510 if (hwctx->cus->num_cus > MAX_NUM_CUS) {
511 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS);
512 return -EINVAL;
513 }
514
515 for (i = 0; i < hwctx->cus->num_cus; i++) {
516 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i];
517
518 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad)))
519 return -EINVAL;
520
521 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo);
522 if (!gobj) {
523 XDNA_ERR(xdna, "Lookup GEM object failed");
524 return -EINVAL;
525 }
526 abo = to_xdna_obj(gobj);
527
528 if (abo->type != AMDXDNA_BO_DEV) {
529 drm_gem_object_put(gobj);
530 XDNA_ERR(xdna, "Invalid BO type");
531 return -EINVAL;
532 }
533
534 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR,
535 abo->mem.dev_addr >> shift);
536 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func);
537 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i,
538 abo->mem.dev_addr, req.cfgs[i]);
539 drm_gem_object_put(gobj);
540 }
541 req.num_cus = hwctx->cus->num_cus;
542
543 msg.send_data = (u8 *)&req;
544 msg.send_size = sizeof(req);
545 msg.handle = hwctx;
546 msg.opcode = MSG_OP_CONFIG_CU;
547 msg.notify_cb = notify_cb;
548 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
549 }
550
aie2_init_exec_cu_req(struct amdxdna_gem_obj * cmd_bo,void * req,size_t * size,u32 * msg_op)551 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
552 size_t *size, u32 *msg_op)
553 {
554 struct execute_buffer_req *cu_req = req;
555 u32 cmd_len;
556 void *cmd;
557
558 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
559 if (cmd_len > sizeof(cu_req->payload))
560 return -EINVAL;
561
562 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
563 if (cu_req->cu_idx == INVALID_CU_IDX)
564 return -EINVAL;
565
566 memcpy(cu_req->payload, cmd, cmd_len);
567
568 *size = sizeof(*cu_req);
569 *msg_op = MSG_OP_EXECUTE_BUFFER_CF;
570 return 0;
571 }
572
aie2_init_exec_dpu_req(struct amdxdna_gem_obj * cmd_bo,void * req,size_t * size,u32 * msg_op)573 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
574 size_t *size, u32 *msg_op)
575 {
576 struct exec_dpu_req *dpu_req = req;
577 struct amdxdna_cmd_start_npu *sn;
578 u32 cmd_len;
579
580 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
581 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload))
582 return -EINVAL;
583
584 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
585 if (dpu_req->cu_idx == INVALID_CU_IDX)
586 return -EINVAL;
587
588 dpu_req->inst_buf_addr = sn->buffer;
589 dpu_req->inst_size = sn->buffer_size;
590 dpu_req->inst_prop_cnt = sn->prop_count;
591 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn));
592
593 *size = sizeof(*dpu_req);
594 *msg_op = MSG_OP_EXEC_DPU;
595 return 0;
596 }
597
aie2_init_exec_chain_req(void * req,u64 slot_addr,size_t size,u32 cmd_cnt)598 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
599 {
600 struct cmd_chain_req *chain_req = req;
601
602 chain_req->buf_addr = slot_addr;
603 chain_req->buf_size = size;
604 chain_req->count = cmd_cnt;
605 }
606
aie2_init_npu_chain_req(void * req,u64 slot_addr,size_t size,u32 cmd_cnt)607 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
608 {
609 struct cmd_chain_npu_req *npu_chain_req = req;
610
611 npu_chain_req->flags = 0;
612 npu_chain_req->reserved = 0;
613 npu_chain_req->buf_addr = slot_addr;
614 npu_chain_req->buf_size = size;
615 npu_chain_req->count = cmd_cnt;
616 }
617
618 static int
aie2_cmdlist_fill_cf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)619 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
620 {
621 struct cmd_chain_slot_execbuf_cf *cf_slot = slot;
622 u32 cmd_len;
623 void *cmd;
624
625 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
626 if (*size < sizeof(*cf_slot) + cmd_len)
627 return -EINVAL;
628
629 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
630 if (cf_slot->cu_idx == INVALID_CU_IDX)
631 return -EINVAL;
632
633 cf_slot->arg_cnt = cmd_len / sizeof(u32);
634 memcpy(cf_slot->args, cmd, cmd_len);
635 /* Accurate slot size to hint firmware to do necessary copy */
636 *size = sizeof(*cf_slot) + cmd_len;
637 return 0;
638 }
639
640 static int
aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)641 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
642 {
643 struct cmd_chain_slot_dpu *dpu_slot = slot;
644 struct amdxdna_cmd_start_npu *sn;
645 u32 cmd_len;
646 u32 arg_sz;
647
648 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
649 arg_sz = cmd_len - sizeof(*sn);
650 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE)
651 return -EINVAL;
652
653 if (*size < sizeof(*dpu_slot) + arg_sz)
654 return -EINVAL;
655
656 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
657 if (dpu_slot->cu_idx == INVALID_CU_IDX)
658 return -EINVAL;
659
660 dpu_slot->inst_buf_addr = sn->buffer;
661 dpu_slot->inst_size = sn->buffer_size;
662 dpu_slot->inst_prop_cnt = sn->prop_count;
663 dpu_slot->arg_cnt = arg_sz / sizeof(u32);
664 memcpy(dpu_slot->args, sn->prop_args, arg_sz);
665
666 /* Accurate slot size to hint firmware to do necessary copy */
667 *size = sizeof(*dpu_slot) + arg_sz;
668 return 0;
669 }
670
aie2_cmdlist_unsupp(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)671 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
672 {
673 return -EOPNOTSUPP;
674 }
675
aie2_get_chain_msg_op(u32 cmd_op)676 static u32 aie2_get_chain_msg_op(u32 cmd_op)
677 {
678 switch (cmd_op) {
679 case ERT_START_CU:
680 return MSG_OP_CHAIN_EXEC_BUFFER_CF;
681 case ERT_START_NPU:
682 return MSG_OP_CHAIN_EXEC_DPU;
683 default:
684 break;
685 }
686
687 return MSG_OP_MAX_OPCODE;
688 }
689
690 static struct aie2_exec_msg_ops legacy_exec_message_ops = {
691 .init_cu_req = aie2_init_exec_cu_req,
692 .init_dpu_req = aie2_init_exec_dpu_req,
693 .init_chain_req = aie2_init_exec_chain_req,
694 .fill_cf_slot = aie2_cmdlist_fill_cf,
695 .fill_dpu_slot = aie2_cmdlist_fill_dpu,
696 .fill_preempt_slot = aie2_cmdlist_unsupp,
697 .fill_elf_slot = aie2_cmdlist_unsupp,
698 .get_chain_msg_op = aie2_get_chain_msg_op,
699 };
700
701 static int
aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)702 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
703 {
704 struct cmd_chain_slot_npu *npu_slot = slot;
705 u32 cmd_len;
706 void *cmd;
707
708 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
709 if (*size < sizeof(*npu_slot) + cmd_len)
710 return -EINVAL;
711
712 memset(npu_slot, 0, sizeof(*npu_slot));
713 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
714 if (npu_slot->cu_idx == INVALID_CU_IDX)
715 return -EINVAL;
716
717 npu_slot->type = EXEC_NPU_TYPE_NON_ELF;
718 npu_slot->arg_cnt = cmd_len / sizeof(u32);
719 memcpy(npu_slot->args, cmd, cmd_len);
720
721 *size = sizeof(*npu_slot) + cmd_len;
722 return 0;
723 }
724
725 static int
aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)726 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
727 {
728 struct cmd_chain_slot_npu *npu_slot = slot;
729 struct amdxdna_cmd_start_npu *sn;
730 u32 cmd_len;
731 u32 arg_sz;
732
733 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
734 arg_sz = cmd_len - sizeof(*sn);
735 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE)
736 return -EINVAL;
737
738 if (*size < sizeof(*npu_slot) + arg_sz)
739 return -EINVAL;
740
741 memset(npu_slot, 0, sizeof(*npu_slot));
742 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
743 if (npu_slot->cu_idx == INVALID_CU_IDX)
744 return -EINVAL;
745
746 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF;
747 npu_slot->inst_buf_addr = sn->buffer;
748 npu_slot->inst_size = sn->buffer_size;
749 npu_slot->inst_prop_cnt = sn->prop_count;
750 npu_slot->arg_cnt = arg_sz / sizeof(u32);
751 memcpy(npu_slot->args, sn->prop_args, arg_sz);
752
753 *size = sizeof(*npu_slot) + arg_sz;
754 return 0;
755 }
756
757 static int
aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)758 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
759 {
760 struct cmd_chain_slot_npu *npu_slot = slot;
761 struct amdxdna_cmd_preempt_data *pd;
762 u32 cmd_len;
763 u32 arg_sz;
764
765 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
766 arg_sz = cmd_len - sizeof(*pd);
767 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
768 return -EINVAL;
769
770 if (*size < sizeof(*npu_slot) + arg_sz)
771 return -EINVAL;
772
773 memset(npu_slot, 0, sizeof(*npu_slot));
774 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
775 if (npu_slot->cu_idx == INVALID_CU_IDX)
776 return -EINVAL;
777
778 npu_slot->type = EXEC_NPU_TYPE_PREEMPT;
779 npu_slot->inst_buf_addr = pd->inst_buf;
780 npu_slot->save_buf_addr = pd->save_buf;
781 npu_slot->restore_buf_addr = pd->restore_buf;
782 npu_slot->inst_size = pd->inst_size;
783 npu_slot->save_size = pd->save_size;
784 npu_slot->restore_size = pd->restore_size;
785 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
786 npu_slot->arg_cnt = arg_sz / sizeof(u32);
787 memcpy(npu_slot->args, pd->prop_args, arg_sz);
788
789 *size = sizeof(*npu_slot) + arg_sz;
790 return 0;
791 }
792
793 static int
aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)794 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
795 {
796 struct cmd_chain_slot_npu *npu_slot = slot;
797 struct amdxdna_cmd_preempt_data *pd;
798 u32 cmd_len;
799 u32 arg_sz;
800
801 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
802 arg_sz = cmd_len - sizeof(*pd);
803 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
804 return -EINVAL;
805
806 if (*size < sizeof(*npu_slot) + arg_sz)
807 return -EINVAL;
808
809 memset(npu_slot, 0, sizeof(*npu_slot));
810 npu_slot->type = EXEC_NPU_TYPE_ELF;
811 npu_slot->inst_buf_addr = pd->inst_buf;
812 npu_slot->save_buf_addr = pd->save_buf;
813 npu_slot->restore_buf_addr = pd->restore_buf;
814 npu_slot->inst_size = pd->inst_size;
815 npu_slot->save_size = pd->save_size;
816 npu_slot->restore_size = pd->restore_size;
817 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
818 npu_slot->arg_cnt = 1;
819 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN;
820
821 *size = struct_size(npu_slot, args, npu_slot->arg_cnt);
822 return 0;
823 }
824
aie2_get_npu_chain_msg_op(u32 cmd_op)825 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op)
826 {
827 return MSG_OP_CHAIN_EXEC_NPU;
828 }
829
830 static struct aie2_exec_msg_ops npu_exec_message_ops = {
831 .init_cu_req = aie2_init_exec_cu_req,
832 .init_dpu_req = aie2_init_exec_dpu_req,
833 .init_chain_req = aie2_init_npu_chain_req,
834 .fill_cf_slot = aie2_cmdlist_fill_npu_cf,
835 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu,
836 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt,
837 .fill_elf_slot = aie2_cmdlist_fill_npu_elf,
838 .get_chain_msg_op = aie2_get_npu_chain_msg_op,
839 };
840
aie2_init_exec_req(void * req,struct amdxdna_gem_obj * cmd_abo,size_t * size,u32 * msg_op)841 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
842 size_t *size, u32 *msg_op)
843 {
844 struct amdxdna_dev *xdna = cmd_abo->client->xdna;
845 int ret;
846 u32 op;
847
848
849 op = amdxdna_cmd_get_op(cmd_abo);
850 switch (op) {
851 case ERT_START_CU:
852 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op);
853 if (ret) {
854 XDNA_DBG(xdna, "Init CU req failed ret %d", ret);
855 return ret;
856 }
857 break;
858 case ERT_START_NPU:
859 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op);
860 if (ret) {
861 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret);
862 return ret;
863 }
864
865 break;
866 default:
867 XDNA_ERR(xdna, "Unsupported op %d", op);
868 ret = -EOPNOTSUPP;
869 break;
870 }
871
872 return ret;
873 }
874
875 static int
aie2_cmdlist_fill_slot(void * slot,struct amdxdna_gem_obj * cmd_abo,size_t * size,u32 * cmd_op)876 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
877 size_t *size, u32 *cmd_op)
878 {
879 struct amdxdna_dev *xdna = cmd_abo->client->xdna;
880 int ret;
881 u32 op;
882
883 op = amdxdna_cmd_get_op(cmd_abo);
884 if (*cmd_op == ERT_INVALID_CMD)
885 *cmd_op = op;
886 else if (op != *cmd_op)
887 return -EINVAL;
888
889 switch (op) {
890 case ERT_START_CU:
891 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size);
892 break;
893 case ERT_START_NPU:
894 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size);
895 break;
896 case ERT_START_NPU_PREEMPT:
897 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
898 return -EOPNOTSUPP;
899 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size);
900 break;
901 case ERT_START_NPU_PREEMPT_ELF:
902 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
903 return -EOPNOTSUPP;
904 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size);
905 break;
906 default:
907 XDNA_INFO(xdna, "Unsupported op %d", op);
908 ret = -EOPNOTSUPP;
909 break;
910 }
911
912 return ret;
913 }
914
aie2_msg_init(struct amdxdna_dev_hdl * ndev)915 void aie2_msg_init(struct amdxdna_dev_hdl *ndev)
916 {
917 if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND))
918 ndev->exec_msg_ops = &npu_exec_message_ops;
919 else
920 ndev->exec_msg_ops = &legacy_exec_message_ops;
921 }
922
aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl * ndev)923 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev)
924 {
925 struct amdxdna_dev *xdna = ndev->xdna;
926
927 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
928
929 if (!ndev->mgmt_chann)
930 return;
931
932 xdna_mailbox_stop_channel(ndev->mgmt_chann);
933 xdna_mailbox_free_channel(ndev->mgmt_chann);
934 ndev->mgmt_chann = NULL;
935 }
936
937 static inline struct amdxdna_gem_obj *
aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job * job)938 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job)
939 {
940 int idx = get_job_idx(job->seq);
941
942 return job->hwctx->priv->cmd_buf[idx];
943 }
944
aie2_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))945 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
946 int (*notify_cb)(void *, void __iomem *, size_t))
947 {
948 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
949 struct amdxdna_dev *xdna = hwctx->client->xdna;
950 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
951 struct xdna_mailbox_msg msg;
952 union exec_req req;
953 int ret;
954
955 if (!chann)
956 return -ENODEV;
957
958 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode);
959 if (ret)
960 return ret;
961
962 msg.handle = job;
963 msg.notify_cb = notify_cb;
964 msg.send_data = (u8 *)&req;
965 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req,
966 0x40, false);
967
968 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
969 if (ret) {
970 XDNA_ERR(xdna, "Send message failed");
971 return ret;
972 }
973
974 return 0;
975 }
976
aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))977 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
978 struct amdxdna_sched_job *job,
979 int (*notify_cb)(void *, void __iomem *, size_t))
980 {
981 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
982 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
983 struct amdxdna_client *client = hwctx->client;
984 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
985 struct amdxdna_dev *xdna = client->xdna;
986 struct amdxdna_cmd_chain *payload;
987 struct xdna_mailbox_msg msg;
988 union exec_chain_req req;
989 u32 payload_len;
990 u32 offset = 0;
991 size_t size;
992 int ret;
993 u32 op;
994 u32 i;
995
996 op = amdxdna_cmd_get_op(cmd_abo);
997 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len);
998 if (op != ERT_CMD_CHAIN || !payload ||
999 payload_len < struct_size(payload, data, payload->command_count))
1000 return -EINVAL;
1001
1002 op = ERT_INVALID_CMD;
1003 for (i = 0; i < payload->command_count; i++) {
1004 u32 boh = (u32)(payload->data[i]);
1005 struct amdxdna_gem_obj *abo;
1006
1007 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD);
1008 if (!abo) {
1009 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh);
1010 return -ENOENT;
1011 }
1012
1013 size = cmdbuf_abo->mem.size - offset;
1014 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva + offset,
1015 abo, &size, &op);
1016 amdxdna_gem_put_obj(abo);
1017 if (ret)
1018 return ret;
1019
1020 offset += size;
1021 }
1022 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1023 if (msg.opcode == MSG_OP_MAX_OPCODE)
1024 return -EOPNOTSUPP;
1025
1026 /* The offset is the accumulated total size of the cmd buffer */
1027 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr,
1028 offset, payload->command_count);
1029 drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset);
1030
1031 msg.handle = job;
1032 msg.notify_cb = notify_cb;
1033 msg.send_data = (u8 *)&req;
1034 msg.send_size = sizeof(req);
1035 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1036 if (ret) {
1037 XDNA_ERR(xdna, "Send message failed");
1038 return ret;
1039 }
1040
1041 return 0;
1042 }
1043
aie2_cmdlist_single_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1044 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
1045 struct amdxdna_sched_job *job,
1046 int (*notify_cb)(void *, void __iomem *, size_t))
1047 {
1048 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
1049 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1050 struct amdxdna_dev *xdna = hwctx->client->xdna;
1051 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
1052 struct xdna_mailbox_msg msg;
1053 union exec_chain_req req;
1054 u32 op = ERT_INVALID_CMD;
1055 size_t size;
1056 int ret;
1057
1058 size = cmdbuf_abo->mem.size;
1059 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva, cmd_abo, &size, &op);
1060 if (ret)
1061 return ret;
1062
1063 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1064 if (msg.opcode == MSG_OP_MAX_OPCODE)
1065 return -EOPNOTSUPP;
1066
1067 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr,
1068 size, 1);
1069 drm_clflush_virt_range(cmdbuf_abo->mem.kva, size);
1070
1071 msg.handle = job;
1072 msg.notify_cb = notify_cb;
1073 msg.send_data = (u8 *)&req;
1074 msg.send_size = sizeof(req);
1075 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1076 if (ret) {
1077 XDNA_ERR(hwctx->client->xdna, "Send message failed");
1078 return ret;
1079 }
1080
1081 return 0;
1082 }
1083
aie2_sync_bo(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1084 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1085 int (*notify_cb)(void *, void __iomem *, size_t))
1086 {
1087 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1088 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1089 struct amdxdna_dev *xdna = hwctx->client->xdna;
1090 struct xdna_mailbox_msg msg;
1091 struct sync_bo_req req;
1092 int ret = 0;
1093
1094 req.src_addr = 0;
1095 req.dst_addr = amdxdna_dev_bo_offset(abo);
1096 req.size = abo->mem.size;
1097
1098 /* Device to Host */
1099 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) |
1100 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM);
1101
1102 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed",
1103 req.size, req.src_addr, req.dst_addr);
1104
1105 msg.handle = job;
1106 msg.notify_cb = notify_cb;
1107 msg.send_data = (u8 *)&req;
1108 msg.send_size = sizeof(req);
1109 msg.opcode = MSG_OP_SYNC_BO;
1110
1111 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1112 if (ret) {
1113 XDNA_ERR(xdna, "Send message failed");
1114 return ret;
1115 }
1116
1117 return 0;
1118 }
1119
aie2_config_debug_bo(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1120 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1121 int (*notify_cb)(void *, void __iomem *, size_t))
1122 {
1123 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1124 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1125 struct amdxdna_dev *xdna = hwctx->client->xdna;
1126 struct config_debug_bo_req req;
1127 struct xdna_mailbox_msg msg;
1128
1129 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO)
1130 req.config = DEBUG_BO_REGISTER;
1131 else
1132 req.config = DEBUG_BO_UNREGISTER;
1133
1134 req.offset = amdxdna_dev_bo_offset(abo);
1135 req.size = abo->mem.size;
1136
1137 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d",
1138 req.offset, req.size, req.config);
1139
1140 msg.handle = job;
1141 msg.notify_cb = notify_cb;
1142 msg.send_data = (u8 *)&req;
1143 msg.send_size = sizeof(req);
1144 msg.opcode = MSG_OP_CONFIG_DEBUG_BO;
1145
1146 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1147 }
1148