1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_cache.h>
8 #include <drm/drm_device.h>
9 #include <drm/drm_gem.h>
10 #include <drm/drm_gem_shmem_helper.h>
11 #include <drm/drm_print.h>
12 #include <drm/gpu_scheduler.h>
13 #include <linux/bitfield.h>
14 #include <linux/errno.h>
15 #include <linux/pci.h>
16 #include <linux/types.h>
17 #include <linux/xarray.h>
18
19 #include "aie.h"
20 #include "aie2_msg_priv.h"
21 #include "aie2_pci.h"
22 #include "amdxdna_ctx.h"
23 #include "amdxdna_gem.h"
24 #include "amdxdna_mailbox.h"
25 #include "amdxdna_mailbox_helper.h"
26 #include "amdxdna_pci_drv.h"
27
28 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops)
29
aie2_suspend_fw(struct amdxdna_dev_hdl * ndev)30 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
31 {
32 DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND);
33 int ret;
34
35 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
36 if (ret) {
37 XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret);
38 return ret;
39 }
40
41 return aie_psp_waitmode_poll(ndev->aie.psp_hdl);
42 }
43
aie2_resume_fw(struct amdxdna_dev_hdl * ndev)44 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
45 {
46 DECLARE_AIE_MSG(suspend, MSG_OP_RESUME);
47
48 return aie_send_mgmt_msg_wait(&ndev->aie, &msg);
49 }
50
aie2_set_runtime_cfg(struct amdxdna_dev_hdl * ndev,u32 type,u64 value)51 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
52 {
53 DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
54 int ret;
55
56 req.type = type;
57 req.value = value;
58
59 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
60 if (ret) {
61 XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret);
62 return ret;
63 }
64
65 return 0;
66 }
67
aie2_get_runtime_cfg(struct amdxdna_dev_hdl * ndev,u32 type,u64 * value)68 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
69 {
70 DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
71 int ret;
72
73 req.type = type;
74 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
75 if (ret) {
76 XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret);
77 return ret;
78 }
79
80 *value = resp.value;
81 return 0;
82 }
83
aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl * ndev,u16 pasid)84 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
85 {
86 DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
87
88 req.pasid = pasid;
89
90 return aie_send_mgmt_msg_wait(&ndev->aie, &msg);
91 }
92
aie2_query_aie_version(struct amdxdna_dev_hdl * ndev,struct amdxdna_drm_query_aie_version * version)93 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev,
94 struct amdxdna_drm_query_aie_version *version)
95 {
96 DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
97 struct amdxdna_dev *xdna = ndev->aie.xdna;
98 int ret;
99
100 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
101 if (ret)
102 return ret;
103
104 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed",
105 resp.major, resp.minor);
106
107 version->major = resp.major;
108 version->minor = resp.minor;
109
110 return 0;
111 }
112
aie2_query_aie_metadata(struct amdxdna_dev_hdl * ndev,struct amdxdna_drm_query_aie_metadata * metadata)113 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev,
114 struct amdxdna_drm_query_aie_metadata *metadata)
115 {
116 DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
117 int ret;
118
119 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
120 if (ret)
121 return ret;
122
123 metadata->col_size = resp.info.size;
124 metadata->cols = resp.info.cols;
125 metadata->rows = resp.info.rows;
126
127 metadata->version.major = resp.info.major;
128 metadata->version.minor = resp.info.minor;
129
130 metadata->core.row_count = resp.info.core_rows;
131 metadata->core.row_start = resp.info.core_row_start;
132 metadata->core.dma_channel_count = resp.info.core_dma_channels;
133 metadata->core.lock_count = resp.info.core_locks;
134 metadata->core.event_reg_count = resp.info.core_events;
135
136 metadata->mem.row_count = resp.info.mem_rows;
137 metadata->mem.row_start = resp.info.mem_row_start;
138 metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
139 metadata->mem.lock_count = resp.info.mem_locks;
140 metadata->mem.event_reg_count = resp.info.mem_events;
141
142 metadata->shim.row_count = resp.info.shim_rows;
143 metadata->shim.row_start = resp.info.shim_row_start;
144 metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
145 metadata->shim.lock_count = resp.info.shim_locks;
146 metadata->shim.event_reg_count = resp.info.shim_events;
147
148 return 0;
149 }
150
aie2_query_firmware_version(struct amdxdna_dev_hdl * ndev,struct amdxdna_fw_ver * fw_ver)151 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
152 struct amdxdna_fw_ver *fw_ver)
153 {
154 DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
155 int ret;
156
157 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
158 if (ret)
159 return ret;
160
161 fw_ver->major = resp.major;
162 fw_ver->minor = resp.minor;
163 fw_ver->sub = resp.sub;
164 fw_ver->build = resp.build;
165
166 return 0;
167 }
168
aie2_destroy_context_req(struct amdxdna_dev_hdl * ndev,u32 id)169 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
170 {
171 DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
172 struct amdxdna_dev *xdna = ndev->aie.xdna;
173 int ret;
174
175 req.context_id = id;
176 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
177 if (ret && ret != -ENODEV)
178 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
179 else if (ret == -ENODEV)
180 XDNA_DBG(xdna, "Destroy context: device already stopped");
181
182 return ret;
183 }
184
aie2_get_context_priority(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)185 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
186 struct amdxdna_hwctx *hwctx)
187 {
188 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT))
189 return PRIORITY_HIGH;
190
191 switch (hwctx->qos.priority) {
192 case AMDXDNA_QOS_REALTIME_PRIORITY:
193 return PRIORITY_REALTIME;
194 case AMDXDNA_QOS_HIGH_PRIORITY:
195 return PRIORITY_HIGH;
196 case AMDXDNA_QOS_NORMAL_PRIORITY:
197 return PRIORITY_NORMAL;
198 case AMDXDNA_QOS_LOW_PRIORITY:
199 return PRIORITY_LOW;
200 default:
201 return PRIORITY_HIGH;
202 }
203 }
204
aie2_create_context(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)205 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
206 {
207 DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
208 struct amdxdna_dev *xdna = ndev->aie.xdna;
209 struct xdna_mailbox_chann_res x2i;
210 struct xdna_mailbox_chann_res i2x;
211 struct cq_pair *cq_pair;
212 u32 intr_reg;
213 int ret;
214
215 req.aie_type = 1;
216 req.start_col = hwctx->start_col;
217 req.num_col = hwctx->num_col;
218 req.num_unused_col = hwctx->num_unused_col;
219 req.num_cq_pairs_requested = 1;
220 req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0;
221 req.context_priority = aie2_get_context_priority(ndev, hwctx);
222
223 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
224 if (ret)
225 return ret;
226
227 hwctx->fw_ctx_id = resp.context_id;
228 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1))
229 return -EINVAL;
230
231 if (ndev->force_preempt_enabled) {
232 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id);
233 if (ret) {
234 XDNA_ERR(xdna, "failed to enable force preempt %d", ret);
235 goto del_ctx_req;
236 }
237 }
238
239 cq_pair = &resp.cq_pair[0];
240 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr);
241 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr);
242 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr);
243 x2i.rb_size = cq_pair->x2i_q.buf_size;
244
245 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr);
246 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr);
247 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr);
248 i2x.rb_size = cq_pair->i2x_q.buf_size;
249
250 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id);
251 if (ret == -EINVAL) {
252 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret);
253 goto del_ctx_req;
254 }
255
256 intr_reg = i2x.mb_head_ptr_reg + 4;
257 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox);
258 if (!hwctx->priv->mbox_chann) {
259 XDNA_ERR(xdna, "Not able to create channel");
260 ret = -EINVAL;
261 goto del_ctx_req;
262 }
263
264 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x,
265 intr_reg, ret);
266 if (ret) {
267 XDNA_ERR(xdna, "Not able to create channel");
268 ret = -EINVAL;
269 goto free_channel;
270 }
271 ndev->hwctx_num++;
272
273 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id);
274 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid);
275
276 return 0;
277
278 free_channel:
279 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
280 del_ctx_req:
281 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
282 return ret;
283 }
284
aie2_destroy_context(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * hwctx)285 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
286 {
287 struct amdxdna_dev *xdna = ndev->aie.xdna;
288 int ret;
289
290 if (!hwctx->priv->mbox_chann)
291 return 0;
292
293 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann);
294 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
295 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
296 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id);
297 hwctx->priv->mbox_chann = NULL;
298 hwctx->fw_ctx_id = -1;
299 ndev->hwctx_num--;
300
301 return ret;
302 }
303
aie2_send_host_buf_msgs(struct amdxdna_dev_hdl * ndev,u32 context_id,u64 addr,u64 size,u32 initial_opcode)304 static int aie2_send_host_buf_msgs(struct amdxdna_dev_hdl *ndev, u32 context_id,
305 u64 addr, u64 size, u32 initial_opcode)
306 {
307 DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
308 struct amdxdna_dev *xdna = ndev->aie.xdna;
309 size_t chunk_size;
310 int ret;
311
312 chunk_size = xdna->dev_info->dev_mem_size;
313 if (!size || !IS_ALIGNED(size, chunk_size)) {
314 XDNA_ERR(xdna, "Invalid size 0x%llx for chunk 0x%lx",
315 size, chunk_size);
316 return -EINVAL;
317 }
318
319 msg.opcode = initial_opcode;
320 do {
321 req.context_id = context_id;
322 req.buf_addr = addr;
323 req.buf_size = chunk_size;
324 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
325 if (ret) {
326 XDNA_ERR(xdna, "fw ctx %d addr 0x%llx size 0x%lx",
327 context_id, addr, chunk_size);
328 return ret;
329 }
330
331 XDNA_DBG(xdna, "fw ctx %d host buf op 0x%x addr 0x%llx size 0x%lx",
332 context_id, msg.opcode, addr, chunk_size);
333
334 addr += chunk_size;
335 size -= chunk_size;
336 msg.opcode = MSG_OP_ADD_HOST_BUFFER;
337 } while (size);
338
339 return 0;
340 }
341
aie2_map_host_buf(struct amdxdna_dev_hdl * ndev,u32 context_id,u64 addr,u64 size)342 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
343 {
344 return aie2_send_host_buf_msgs(ndev, context_id, addr, size,
345 MSG_OP_MAP_HOST_BUFFER);
346 }
347
aie2_add_host_buf(struct amdxdna_dev_hdl * ndev,u32 context_id,u64 addr,u64 size)348 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
349 {
350 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_ADD_HOST_BUFFER))
351 return -EOPNOTSUPP;
352
353 return aie2_send_host_buf_msgs(ndev, context_id, addr, size,
354 MSG_OP_ADD_HOST_BUFFER);
355 }
356
amdxdna_hwctx_col_map(struct amdxdna_hwctx * hwctx,void * arg)357 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
358 {
359 u32 *bitmap = arg;
360
361 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col);
362
363 return 0;
364 }
365
aie2_query_status(struct amdxdna_dev_hdl * ndev,char __user * buf,u32 size,u32 * cols_filled)366 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
367 u32 size, u32 *cols_filled)
368 {
369 DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
370 struct amdxdna_dev *xdna = ndev->aie.xdna;
371 u32 buf_sz, aie_bitmap = 0;
372 struct amdxdna_client *client;
373 dma_addr_t dma_addr;
374 u8 *buff_addr;
375 int ret;
376
377 buf_sz = ndev->aie.metadata.cols * ndev->aie.metadata.col_size;
378 buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
379 if (IS_ERR(buff_addr))
380 return PTR_ERR(buff_addr);
381
382 /* Go through each hardware context and mark the AIE columns that are active */
383 list_for_each_entry(client, &xdna->client_list, node)
384 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map);
385
386 *cols_filled = 0;
387 req.dump_buff_addr = dma_addr;
388 req.dump_buff_size = buf_sz;
389 req.num_cols = hweight32(aie_bitmap);
390 req.aie_bitmap = aie_bitmap;
391
392 drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */
393 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
394 if (ret) {
395 XDNA_ERR(xdna, "Error during NPU query, status %d", ret);
396 goto fail;
397 }
398
399 XDNA_DBG(xdna, "Query NPU status completed");
400
401 if (buf_sz < resp.size) {
402 ret = -EINVAL;
403 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size);
404 goto fail;
405 }
406
407 size = min(size, resp.size);
408 if (copy_to_user(buf, buff_addr, size)) {
409 ret = -EFAULT;
410 XDNA_ERR(xdna, "Failed to copy NPU status to user space");
411 goto fail;
412 }
413
414 *cols_filled = aie_bitmap;
415
416 fail:
417 amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr);
418 return ret;
419 }
420
aie2_query_telemetry(struct amdxdna_dev_hdl * ndev,char __user * buf,u32 size,struct amdxdna_drm_query_telemetry_header * header)421 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
422 char __user *buf, u32 size,
423 struct amdxdna_drm_query_telemetry_header *header)
424 {
425 DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
426 struct amdxdna_dev *xdna = ndev->aie.xdna;
427 dma_addr_t dma_addr;
428 u32 buf_sz;
429 u8 *addr;
430 int ret;
431
432 if (header->type >= MAX_TELEMETRY_TYPE)
433 return -EINVAL;
434
435 buf_sz = min(size, SZ_4M);
436 addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
437 if (IS_ERR(addr))
438 return PTR_ERR(addr);
439
440 req.buf_addr = dma_addr;
441 req.buf_size = buf_sz;
442 req.type = header->type;
443
444 drm_clflush_virt_range(addr, req.buf_size); /* device can access */
445 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
446 if (ret) {
447 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret);
448 goto free_buf;
449 }
450
451 if (buf_sz < resp.size) {
452 ret = -EINVAL;
453 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size);
454 goto free_buf;
455 }
456
457 size = min(size, resp.size);
458 if (copy_to_user(buf, addr, size)) {
459 ret = -EFAULT;
460 XDNA_ERR(xdna, "Failed to copy telemetry to user space");
461 goto free_buf;
462 }
463
464 header->major = resp.major;
465 header->minor = resp.minor;
466
467 free_buf:
468 amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr);
469 return ret;
470 }
471
aie2_register_asyn_event_msg(struct amdxdna_dev_hdl * ndev,dma_addr_t addr,u32 size,void * handle,int (* cb)(void *,void __iomem *,size_t))472 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size,
473 void *handle, int (*cb)(void*, void __iomem *, size_t))
474 {
475 struct async_event_msg_req req = { 0 };
476 struct xdna_mailbox_msg msg = {
477 .send_data = (u8 *)&req,
478 .send_size = sizeof(req),
479 .handle = handle,
480 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG,
481 .notify_cb = cb,
482 };
483
484 req.buf_addr = addr;
485 req.buf_size = size;
486
487 XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size);
488 return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT);
489 }
490
aie2_config_cu(struct amdxdna_hwctx * hwctx,int (* notify_cb)(void *,void __iomem *,size_t))491 int aie2_config_cu(struct amdxdna_hwctx *hwctx,
492 int (*notify_cb)(void *, void __iomem *, size_t))
493 {
494 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
495 struct amdxdna_dev *xdna = hwctx->client->xdna;
496 u32 shift = xdna->dev_info->dev_mem_buf_shift;
497 struct config_cu_req req = { 0 };
498 struct xdna_mailbox_msg msg;
499 struct drm_gem_object *gobj;
500 struct amdxdna_gem_obj *abo;
501 int i;
502
503 if (!chann)
504 return -ENODEV;
505
506 if (!hwctx->cus)
507 return 0;
508
509 if (hwctx->cus->num_cus > MAX_NUM_CUS) {
510 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS);
511 return -EINVAL;
512 }
513
514 for (i = 0; i < hwctx->cus->num_cus; i++) {
515 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i];
516
517 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad)))
518 return -EINVAL;
519
520 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo);
521 if (!gobj) {
522 XDNA_ERR(xdna, "Lookup GEM object failed");
523 return -EINVAL;
524 }
525 abo = to_xdna_obj(gobj);
526
527 if (abo->type != AMDXDNA_BO_DEV) {
528 drm_gem_object_put(gobj);
529 XDNA_ERR(xdna, "Invalid BO type");
530 return -EINVAL;
531 }
532
533 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR,
534 amdxdna_gem_dev_addr(abo) >> shift);
535 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func);
536 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i,
537 amdxdna_gem_dev_addr(abo), req.cfgs[i]);
538 drm_gem_object_put(gobj);
539 }
540 req.num_cus = hwctx->cus->num_cus;
541
542 msg.send_data = (u8 *)&req;
543 msg.send_size = sizeof(req);
544 msg.handle = hwctx;
545 msg.opcode = MSG_OP_CONFIG_CU;
546 msg.notify_cb = notify_cb;
547 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
548 }
549
aie2_init_exec_cu_req(struct amdxdna_gem_obj * cmd_bo,void * req,size_t * size,u32 * msg_op)550 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
551 size_t *size, u32 *msg_op)
552 {
553 struct execute_buffer_req *cu_req = req;
554 u32 cmd_len;
555 void *cmd;
556
557 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
558 if (cmd_len > sizeof(cu_req->payload))
559 return -EINVAL;
560
561 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
562 if (cu_req->cu_idx == INVALID_CU_IDX)
563 return -EINVAL;
564
565 memcpy(cu_req->payload, cmd, cmd_len);
566
567 *size = sizeof(*cu_req);
568 *msg_op = MSG_OP_EXECUTE_BUFFER_CF;
569 return 0;
570 }
571
aie2_init_exec_dpu_req(struct amdxdna_gem_obj * cmd_bo,void * req,size_t * size,u32 * msg_op)572 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
573 size_t *size, u32 *msg_op)
574 {
575 struct exec_dpu_req *dpu_req = req;
576 struct amdxdna_cmd_start_npu *sn;
577 u32 cmd_len;
578
579 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
580 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload))
581 return -EINVAL;
582
583 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
584 if (dpu_req->cu_idx == INVALID_CU_IDX)
585 return -EINVAL;
586
587 dpu_req->inst_buf_addr = sn->buffer;
588 dpu_req->inst_size = sn->buffer_size;
589 dpu_req->inst_prop_cnt = sn->prop_count;
590 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn));
591
592 *size = sizeof(*dpu_req);
593 *msg_op = MSG_OP_EXEC_DPU;
594 return 0;
595 }
596
aie2_init_exec_chain_req(void * req,u64 slot_addr,size_t size,u32 cmd_cnt)597 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
598 {
599 struct cmd_chain_req *chain_req = req;
600
601 chain_req->buf_addr = slot_addr;
602 chain_req->buf_size = size;
603 chain_req->count = cmd_cnt;
604 }
605
aie2_init_npu_chain_req(void * req,u64 slot_addr,size_t size,u32 cmd_cnt)606 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
607 {
608 struct cmd_chain_npu_req *npu_chain_req = req;
609
610 npu_chain_req->flags = 0;
611 npu_chain_req->reserved = 0;
612 npu_chain_req->buf_addr = slot_addr;
613 npu_chain_req->buf_size = size;
614 npu_chain_req->count = cmd_cnt;
615 }
616
617 static int
aie2_cmdlist_fill_cf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)618 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
619 {
620 struct cmd_chain_slot_execbuf_cf *cf_slot = slot;
621 u32 cmd_len;
622 void *cmd;
623
624 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
625 if (*size < sizeof(*cf_slot) + cmd_len)
626 return -EINVAL;
627
628 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
629 if (cf_slot->cu_idx == INVALID_CU_IDX)
630 return -EINVAL;
631
632 cf_slot->arg_cnt = cmd_len / sizeof(u32);
633 memcpy(cf_slot->args, cmd, cmd_len);
634 /* Accurate slot size to hint firmware to do necessary copy */
635 *size = sizeof(*cf_slot) + cmd_len;
636 return 0;
637 }
638
639 static int
aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)640 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
641 {
642 struct cmd_chain_slot_dpu *dpu_slot = slot;
643 struct amdxdna_cmd_start_npu *sn;
644 u32 cmd_len;
645 u32 arg_sz;
646
647 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
648 arg_sz = cmd_len - sizeof(*sn);
649 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE)
650 return -EINVAL;
651
652 if (*size < sizeof(*dpu_slot) + arg_sz)
653 return -EINVAL;
654
655 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
656 if (dpu_slot->cu_idx == INVALID_CU_IDX)
657 return -EINVAL;
658
659 dpu_slot->inst_buf_addr = sn->buffer;
660 dpu_slot->inst_size = sn->buffer_size;
661 dpu_slot->inst_prop_cnt = sn->prop_count;
662 dpu_slot->arg_cnt = arg_sz / sizeof(u32);
663 memcpy(dpu_slot->args, sn->prop_args, arg_sz);
664
665 /* Accurate slot size to hint firmware to do necessary copy */
666 *size = sizeof(*dpu_slot) + arg_sz;
667 return 0;
668 }
669
aie2_cmdlist_unsupp(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)670 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
671 {
672 return -EOPNOTSUPP;
673 }
674
aie2_get_chain_msg_op(u32 cmd_op)675 static u32 aie2_get_chain_msg_op(u32 cmd_op)
676 {
677 switch (cmd_op) {
678 case ERT_START_CU:
679 return MSG_OP_CHAIN_EXEC_BUFFER_CF;
680 case ERT_START_NPU:
681 return MSG_OP_CHAIN_EXEC_DPU;
682 default:
683 break;
684 }
685
686 return MSG_OP_MAX_OPCODE;
687 }
688
689 static struct aie2_exec_msg_ops legacy_exec_message_ops = {
690 .init_cu_req = aie2_init_exec_cu_req,
691 .init_dpu_req = aie2_init_exec_dpu_req,
692 .init_chain_req = aie2_init_exec_chain_req,
693 .fill_cf_slot = aie2_cmdlist_fill_cf,
694 .fill_dpu_slot = aie2_cmdlist_fill_dpu,
695 .fill_preempt_slot = aie2_cmdlist_unsupp,
696 .fill_elf_slot = aie2_cmdlist_unsupp,
697 .get_chain_msg_op = aie2_get_chain_msg_op,
698 };
699
700 static int
aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)701 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
702 {
703 struct cmd_chain_slot_npu *npu_slot = slot;
704 u32 cmd_len;
705 void *cmd;
706
707 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
708 if (*size < sizeof(*npu_slot) + cmd_len)
709 return -EINVAL;
710
711 memset(npu_slot, 0, sizeof(*npu_slot));
712 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
713 if (npu_slot->cu_idx == INVALID_CU_IDX)
714 return -EINVAL;
715
716 npu_slot->type = EXEC_NPU_TYPE_NON_ELF;
717 npu_slot->arg_cnt = cmd_len / sizeof(u32);
718 memcpy(npu_slot->args, cmd, cmd_len);
719
720 *size = sizeof(*npu_slot) + cmd_len;
721 return 0;
722 }
723
724 static int
aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)725 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
726 {
727 struct cmd_chain_slot_npu *npu_slot = slot;
728 struct amdxdna_cmd_start_npu *sn;
729 u32 cmd_len;
730 u32 arg_sz;
731
732 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
733 arg_sz = cmd_len - sizeof(*sn);
734 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE)
735 return -EINVAL;
736
737 if (*size < sizeof(*npu_slot) + arg_sz)
738 return -EINVAL;
739
740 memset(npu_slot, 0, sizeof(*npu_slot));
741 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
742 if (npu_slot->cu_idx == INVALID_CU_IDX)
743 return -EINVAL;
744
745 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF;
746 npu_slot->inst_buf_addr = sn->buffer;
747 npu_slot->inst_size = sn->buffer_size;
748 npu_slot->inst_prop_cnt = sn->prop_count;
749 npu_slot->arg_cnt = arg_sz / sizeof(u32);
750 memcpy(npu_slot->args, sn->prop_args, arg_sz);
751
752 *size = sizeof(*npu_slot) + arg_sz;
753 return 0;
754 }
755
756 static int
aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)757 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
758 {
759 struct cmd_chain_slot_npu *npu_slot = slot;
760 struct amdxdna_cmd_preempt_data *pd;
761 u32 cmd_len;
762 u32 arg_sz;
763
764 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
765 arg_sz = cmd_len - sizeof(*pd);
766 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
767 return -EINVAL;
768
769 if (*size < sizeof(*npu_slot) + arg_sz)
770 return -EINVAL;
771
772 memset(npu_slot, 0, sizeof(*npu_slot));
773 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
774 if (npu_slot->cu_idx == INVALID_CU_IDX)
775 return -EINVAL;
776
777 npu_slot->type = EXEC_NPU_TYPE_PREEMPT;
778 npu_slot->inst_buf_addr = pd->inst_buf;
779 npu_slot->save_buf_addr = pd->save_buf;
780 npu_slot->restore_buf_addr = pd->restore_buf;
781 npu_slot->inst_size = pd->inst_size;
782 npu_slot->save_size = pd->save_size;
783 npu_slot->restore_size = pd->restore_size;
784 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
785 npu_slot->arg_cnt = arg_sz / sizeof(u32);
786 memcpy(npu_slot->args, pd->prop_args, arg_sz);
787
788 *size = sizeof(*npu_slot) + arg_sz;
789 return 0;
790 }
791
792 static int
aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj * cmd_bo,void * slot,size_t * size)793 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
794 {
795 struct cmd_chain_slot_npu *npu_slot = slot;
796 struct amdxdna_cmd_preempt_data *pd;
797 u32 cmd_len;
798 u32 arg_sz;
799
800 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
801 arg_sz = cmd_len - sizeof(*pd);
802 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
803 return -EINVAL;
804
805 if (*size < sizeof(*npu_slot) + arg_sz)
806 return -EINVAL;
807
808 memset(npu_slot, 0, sizeof(*npu_slot));
809 npu_slot->type = EXEC_NPU_TYPE_ELF;
810 npu_slot->inst_buf_addr = pd->inst_buf;
811 npu_slot->save_buf_addr = pd->save_buf;
812 npu_slot->restore_buf_addr = pd->restore_buf;
813 npu_slot->inst_size = pd->inst_size;
814 npu_slot->save_size = pd->save_size;
815 npu_slot->restore_size = pd->restore_size;
816 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
817 npu_slot->arg_cnt = 1;
818 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN;
819
820 *size = struct_size(npu_slot, args, npu_slot->arg_cnt);
821 return 0;
822 }
823
aie2_get_npu_chain_msg_op(u32 cmd_op)824 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op)
825 {
826 return MSG_OP_CHAIN_EXEC_NPU;
827 }
828
829 static struct aie2_exec_msg_ops npu_exec_message_ops = {
830 .init_cu_req = aie2_init_exec_cu_req,
831 .init_dpu_req = aie2_init_exec_dpu_req,
832 .init_chain_req = aie2_init_npu_chain_req,
833 .fill_cf_slot = aie2_cmdlist_fill_npu_cf,
834 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu,
835 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt,
836 .fill_elf_slot = aie2_cmdlist_fill_npu_elf,
837 .get_chain_msg_op = aie2_get_npu_chain_msg_op,
838 };
839
aie2_init_exec_req(void * req,struct amdxdna_gem_obj * cmd_abo,size_t * size,u32 * msg_op)840 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
841 size_t *size, u32 *msg_op)
842 {
843 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev);
844 int ret;
845 u32 op;
846
847 op = amdxdna_cmd_get_op(cmd_abo);
848 switch (op) {
849 case ERT_START_CU:
850 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op);
851 if (ret) {
852 XDNA_DBG(xdna, "Init CU req failed ret %d", ret);
853 return ret;
854 }
855 break;
856 case ERT_START_NPU:
857 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op);
858 if (ret) {
859 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret);
860 return ret;
861 }
862
863 break;
864 default:
865 XDNA_ERR(xdna, "Unsupported op %d", op);
866 ret = -EOPNOTSUPP;
867 break;
868 }
869
870 return ret;
871 }
872
873 static int
aie2_cmdlist_fill_slot(void * slot,struct amdxdna_gem_obj * cmd_abo,size_t * size,u32 * cmd_op)874 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
875 size_t *size, u32 *cmd_op)
876 {
877 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev);
878 int ret;
879 u32 op;
880
881 op = amdxdna_cmd_get_op(cmd_abo);
882 if (*cmd_op == ERT_INVALID_CMD)
883 *cmd_op = op;
884 else if (op != *cmd_op)
885 return -EINVAL;
886
887 switch (op) {
888 case ERT_START_CU:
889 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size);
890 break;
891 case ERT_START_NPU:
892 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size);
893 break;
894 case ERT_START_NPU_PREEMPT:
895 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT))
896 return -EOPNOTSUPP;
897 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size);
898 break;
899 case ERT_START_NPU_PREEMPT_ELF:
900 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT))
901 return -EOPNOTSUPP;
902 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size);
903 break;
904 default:
905 XDNA_INFO(xdna, "Unsupported op %d", op);
906 ret = -EOPNOTSUPP;
907 break;
908 }
909
910 return ret;
911 }
912
aie2_msg_init(struct amdxdna_dev_hdl * ndev)913 void aie2_msg_init(struct amdxdna_dev_hdl *ndev)
914 {
915 if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND))
916 ndev->exec_msg_ops = &npu_exec_message_ops;
917 else
918 ndev->exec_msg_ops = &legacy_exec_message_ops;
919 }
920
921 static inline struct amdxdna_gem_obj *
aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job * job)922 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job)
923 {
924 int idx = get_job_idx(job->seq);
925
926 return job->hwctx->priv->cmd_buf[idx];
927 }
928
aie2_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))929 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
930 int (*notify_cb)(void *, void __iomem *, size_t))
931 {
932 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
933 struct amdxdna_dev *xdna = hwctx->client->xdna;
934 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
935 struct xdna_mailbox_msg msg;
936 union exec_req req;
937 int ret;
938
939 if (!chann)
940 return -ENODEV;
941
942 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode);
943 if (ret)
944 return ret;
945
946 msg.handle = job;
947 msg.notify_cb = notify_cb;
948 msg.send_data = (u8 *)&req;
949 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req,
950 0x40, false);
951
952 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
953 if (ret) {
954 XDNA_ERR(xdna, "Send message failed");
955 return ret;
956 }
957
958 return 0;
959 }
960
aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))961 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
962 struct amdxdna_sched_job *job,
963 int (*notify_cb)(void *, void __iomem *, size_t))
964 {
965 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
966 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
967 struct amdxdna_client *client = hwctx->client;
968 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
969 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo);
970 struct amdxdna_dev *xdna = client->xdna;
971 struct amdxdna_cmd_chain *payload;
972 struct xdna_mailbox_msg msg;
973 union exec_chain_req req;
974 u32 payload_len, ccnt;
975 u32 offset = 0;
976 size_t size;
977 int ret;
978 u32 op;
979 u32 i;
980
981 if (!cmd_buf)
982 return -ENOMEM;
983
984 op = amdxdna_cmd_get_op(cmd_abo);
985 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len);
986 if (op != ERT_CMD_CHAIN) {
987 XDNA_DBG(xdna, "Invalid op code %d", op);
988 return -EINVAL;
989 }
990
991 if (!payload) {
992 XDNA_DBG(xdna, "Failed to get command payload");
993 return -EINVAL;
994 }
995
996 ccnt = payload->command_count;
997 if (payload_len < struct_size(payload, data, ccnt)) {
998 XDNA_DBG(xdna, "Invalid command count %d", ccnt);
999 return -EINVAL;
1000 }
1001
1002 op = ERT_INVALID_CMD;
1003 for (i = 0; i < ccnt; i++) {
1004 u32 boh = (u32)(payload->data[i]);
1005 struct amdxdna_gem_obj *abo;
1006
1007 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_SHARE);
1008 if (!abo) {
1009 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh);
1010 return -ENOENT;
1011 }
1012
1013 size = cmdbuf_abo->mem.size - offset;
1014 ret = aie2_cmdlist_fill_slot(cmd_buf + offset, abo, &size, &op);
1015 amdxdna_gem_put_obj(abo);
1016 if (ret)
1017 return ret;
1018
1019 offset += size;
1020 }
1021
1022 XDNA_DBG(xdna, "Total %d commands:", ccnt);
1023 print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4,
1024 cmd_buf, offset, false);
1025
1026 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1027 if (msg.opcode == MSG_OP_MAX_OPCODE)
1028 return -EOPNOTSUPP;
1029
1030 /* The offset is the accumulated total size of the cmd buffer */
1031 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo),
1032 offset, ccnt);
1033 drm_clflush_virt_range(cmd_buf, offset);
1034
1035 msg.handle = job;
1036 msg.notify_cb = notify_cb;
1037 msg.send_data = (u8 *)&req;
1038 msg.send_size = sizeof(req);
1039 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
1040 &req, msg.send_size, false);
1041 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1042 if (ret) {
1043 XDNA_ERR(xdna, "Send message failed");
1044 return ret;
1045 }
1046
1047 return 0;
1048 }
1049
aie2_cmdlist_single_execbuf(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1050 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
1051 struct amdxdna_sched_job *job,
1052 int (*notify_cb)(void *, void __iomem *, size_t))
1053 {
1054 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
1055 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1056 struct amdxdna_dev *xdna = hwctx->client->xdna;
1057 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
1058 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo);
1059 struct xdna_mailbox_msg msg;
1060 union exec_chain_req req;
1061 u32 op = ERT_INVALID_CMD;
1062 size_t size;
1063 int ret;
1064
1065 if (!cmd_buf)
1066 return -ENOMEM;
1067
1068 size = cmdbuf_abo->mem.size;
1069 ret = aie2_cmdlist_fill_slot(cmd_buf, cmd_abo, &size, &op);
1070 if (ret)
1071 return ret;
1072
1073 print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, cmd_buf, size, false);
1074
1075 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1076 if (msg.opcode == MSG_OP_MAX_OPCODE)
1077 return -EOPNOTSUPP;
1078
1079 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), size, 1);
1080 drm_clflush_virt_range(cmd_buf, size);
1081
1082 msg.handle = job;
1083 msg.notify_cb = notify_cb;
1084 msg.send_data = (u8 *)&req;
1085 msg.send_size = sizeof(req);
1086 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
1087 &req, msg.send_size, false);
1088 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1089 if (ret) {
1090 XDNA_ERR(hwctx->client->xdna, "Send message failed");
1091 return ret;
1092 }
1093
1094 return 0;
1095 }
1096
aie2_sync_bo(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1097 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1098 int (*notify_cb)(void *, void __iomem *, size_t))
1099 {
1100 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1101 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1102 struct amdxdna_dev *xdna = hwctx->client->xdna;
1103 struct xdna_mailbox_msg msg;
1104 struct sync_bo_req req;
1105 int ret = 0;
1106
1107 req.src_addr = 0;
1108 req.dst_addr = amdxdna_dev_bo_offset(abo);
1109 req.size = abo->mem.size;
1110
1111 /* Device to Host */
1112 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) |
1113 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM);
1114
1115 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed",
1116 req.size, req.src_addr, req.dst_addr);
1117
1118 msg.handle = job;
1119 msg.notify_cb = notify_cb;
1120 msg.send_data = (u8 *)&req;
1121 msg.send_size = sizeof(req);
1122 msg.opcode = MSG_OP_SYNC_BO;
1123
1124 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1125 if (ret) {
1126 XDNA_ERR(xdna, "Send message failed");
1127 return ret;
1128 }
1129
1130 return 0;
1131 }
1132
aie2_config_debug_bo(struct amdxdna_hwctx * hwctx,struct amdxdna_sched_job * job,int (* notify_cb)(void *,void __iomem *,size_t))1133 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1134 int (*notify_cb)(void *, void __iomem *, size_t))
1135 {
1136 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1137 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1138 struct amdxdna_dev *xdna = hwctx->client->xdna;
1139 struct config_debug_bo_req req;
1140 struct xdna_mailbox_msg msg;
1141
1142 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO)
1143 req.config = DEBUG_BO_REGISTER;
1144 else
1145 req.config = DEBUG_BO_UNREGISTER;
1146
1147 req.offset = amdxdna_dev_bo_offset(abo);
1148 req.size = abo->mem.size;
1149
1150 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d",
1151 req.offset, req.size, req.config);
1152
1153 msg.handle = job;
1154 msg.notify_cb = notify_cb;
1155 msg.send_data = (u8 *)&req;
1156 msg.send_size = sizeof(req);
1157 msg.opcode = MSG_OP_CONFIG_DEBUG_BO;
1158
1159 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1160 }
1161
aie2_query_app_health(struct amdxdna_dev_hdl * ndev,u32 context_id,struct app_health_report * report)1162 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
1163 struct app_health_report *report)
1164 {
1165 DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH);
1166 struct amdxdna_dev *xdna = ndev->aie.xdna;
1167 struct app_health_report *buf;
1168 dma_addr_t dma_addr;
1169 u32 buf_size;
1170 int ret;
1171
1172 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) {
1173 XDNA_DBG(xdna, "App health feature not supported");
1174 return -EOPNOTSUPP;
1175 }
1176
1177 buf_size = sizeof(*report);
1178 buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr);
1179 if (IS_ERR(buf)) {
1180 XDNA_ERR(xdna, "Failed to allocate buffer for app health");
1181 return PTR_ERR(buf);
1182 }
1183
1184 req.buf_addr = dma_addr;
1185 req.context_id = context_id;
1186 req.buf_size = buf_size;
1187
1188 drm_clflush_virt_range(buf, req.buf_size);
1189 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
1190 if (ret) {
1191 XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status);
1192 goto free_buf;
1193 }
1194
1195 /* Copy the report to caller's buffer */
1196 memcpy(report, buf, sizeof(*report));
1197
1198 free_buf:
1199 amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr);
1200 return ret;
1201 }
1202
1203 static int
aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl * ndev,struct amdxdna_hwctx * ctx,u32 type,u32 value)1204 aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl *ndev,
1205 struct amdxdna_hwctx *ctx, u32 type, u32 value)
1206 {
1207 DECLARE_AIE_MSG(update_property, MSG_OP_UPDATE_PROPERTY);
1208 struct amdxdna_dev *xdna = ndev->aie.xdna;
1209 int ret;
1210
1211 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_UPDATE_PROPERTY))
1212 return -EOPNOTSUPP;
1213
1214 if (ctx)
1215 req.context_id = ctx->fw_ctx_id;
1216 else
1217 req.context_id = AIE2_UPDATE_PROPERTY_ALL_CTX;
1218
1219 req.time_quota_us = value;
1220 req.type = type;
1221
1222 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
1223 if (ret) {
1224 XDNA_ERR(xdna, "%s update property failed, type %d ret %d",
1225 ctx ? ctx->name : "ctx.all", type, ret);
1226 return ret;
1227 }
1228
1229 return 0;
1230 }
1231
aie2_update_prop_time_quota(struct amdxdna_dev_hdl * ndev,u32 us)1232 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us)
1233 {
1234 struct amdxdna_dev *xdna = ndev->aie.xdna;
1235 int ret;
1236
1237 ret = aie2_runtime_update_ctx_prop(ndev, NULL, UPDATE_PROPERTY_TIME_QUOTA, us);
1238 if (ret == -EOPNOTSUPP) {
1239 XDNA_DBG(xdna, "update time quota not support, skipped");
1240 ret = 0;
1241 } else if (!ret) {
1242 XDNA_DBG(xdna, "Ctx exec time quantum updated to %u us", us);
1243 }
1244 return ret;
1245 }
1246
aie2_get_dev_revision(struct amdxdna_dev_hdl * ndev,enum aie2_dev_revision * rev)1247 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev)
1248 {
1249 DECLARE_AIE_MSG(get_dev_revision, MSG_OP_GET_DEV_REVISION);
1250 struct amdxdna_dev *xdna = ndev->aie.xdna;
1251 int ret;
1252
1253 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_GET_DEV_REVISION))
1254 return -EOPNOTSUPP;
1255
1256 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
1257 if (ret)
1258 return ret;
1259
1260 *rev = resp.rev;
1261
1262 if (*rev < AIE2_DEV_REVISION_STXA || *rev >= AIE2_DEV_REVISION_UNKN) {
1263 XDNA_ERR(xdna, "Unknown device revision: %d (raw fuse: 0x%x)",
1264 *rev, resp.raw_fuse_data);
1265 return -EINVAL;
1266 }
1267
1268 XDNA_DBG(xdna, "Device revision: %d (raw fuse: 0x%x)", *rev, resp.raw_fuse_data);
1269 return 0;
1270 }
1271