1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_gem_shmem_helper.h>
10 #include <drm/drm_managed.h>
11 #include <drm/drm_print.h>
12 #include <drm/gpu_scheduler.h>
13 #include <linux/amd-pmf-io.h>
14 #include <linux/cleanup.h>
15 #include <linux/errno.h>
16 #include <linux/firmware.h>
17 #include <linux/iommu.h>
18 #include <linux/iopoll.h>
19 #include <linux/pci.h>
20 #include <linux/xarray.h>
21 #include <asm/hypervisor.h>
22
23 #include "aie2_msg_priv.h"
24 #include "aie2_pci.h"
25 #include "aie2_solver.h"
26 #include "amdxdna_ctx.h"
27 #include "amdxdna_gem.h"
28 #include "amdxdna_mailbox.h"
29 #include "amdxdna_pci_drv.h"
30 #include "amdxdna_pm.h"
31
32 static int aie2_max_col = XRS_MAX_COL;
33 module_param(aie2_max_col, uint, 0600);
34 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
35
36 static char *npu_fw[] = {
37 "npu_7.sbin",
38 "npu.sbin"
39 };
40
41 /*
42 * The management mailbox channel is allocated by firmware.
43 * The related register and ring buffer information is on SRAM BAR.
44 * This struct is the register layout.
45 */
46 #define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */
47 struct mgmt_mbox_chann_info {
48 __u32 x2i_tail;
49 __u32 x2i_head;
50 __u32 x2i_buf;
51 __u32 x2i_buf_sz;
52 __u32 i2x_tail;
53 __u32 i2x_head;
54 __u32 i2x_buf;
55 __u32 i2x_buf_sz;
56 __u32 magic;
57 __u32 msi_id;
58 __u32 prot_major;
59 __u32 prot_minor;
60 __u32 rsvd[4];
61 };
62
aie2_check_protocol(struct amdxdna_dev_hdl * ndev,u32 fw_major,u32 fw_minor)63 static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
64 {
65 const struct aie2_fw_feature_tbl *feature;
66 bool found = false;
67
68 for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) {
69 if (feature->major != fw_major)
70 continue;
71 if (fw_minor < feature->min_minor)
72 continue;
73 if (feature->max_minor > 0 && fw_minor > feature->max_minor)
74 continue;
75
76 ndev->feature_mask |= feature->features;
77
78 /* firmware version matches one of the driver support entry */
79 found = true;
80 }
81
82 return found ? 0 : -EOPNOTSUPP;
83 }
84
aie2_dump_chann_info_debug(struct amdxdna_dev_hdl * ndev)85 static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
86 {
87 struct amdxdna_dev *xdna = ndev->xdna;
88
89 XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
90 XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
91 XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
92 XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size);
93 XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
94 XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
95 XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
96 XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size);
97 XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
98 XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
99 XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
100 }
101
aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl * ndev)102 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
103 {
104 struct mgmt_mbox_chann_info info_regs;
105 struct xdna_mailbox_chann_res *i2x;
106 struct xdna_mailbox_chann_res *x2i;
107 u32 addr, off;
108 u32 *reg;
109 int ret;
110 int i;
111
112 /*
113 * Once firmware is alive, it will write management channel
114 * information in SRAM BAR and write the address of that information
115 * at FW_ALIVE_OFF offset in SRMA BAR.
116 *
117 * Read a non-zero value from FW_ALIVE_OFF implies that firmware
118 * is alive.
119 */
120 ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
121 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
122 if (ret || !addr)
123 return -ETIME;
124
125 off = AIE2_SRAM_OFF(ndev, addr);
126 reg = (u32 *)&info_regs;
127 for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
128 reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
129
130 if (info_regs.magic != MGMT_MBOX_MAGIC) {
131 XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
132 ret = -EINVAL;
133 goto done;
134 }
135
136 i2x = &ndev->mgmt_i2x;
137 x2i = &ndev->mgmt_x2i;
138
139 i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
140 i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
141 i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
142 i2x->rb_size = info_regs.i2x_buf_sz;
143
144 x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
145 x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
146 x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
147 x2i->rb_size = info_regs.x2i_buf_sz;
148
149 ndev->mgmt_chan_idx = info_regs.msi_id;
150 ndev->mgmt_prot_major = info_regs.prot_major;
151 ndev->mgmt_prot_minor = info_regs.prot_minor;
152
153 ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor);
154
155 done:
156 aie2_dump_chann_info_debug(ndev);
157
158 /* Must clear address at FW_ALIVE_OFF */
159 writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
160
161 return ret;
162 }
163
aie2_runtime_cfg(struct amdxdna_dev_hdl * ndev,enum rt_config_category category,u32 * val)164 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
165 enum rt_config_category category, u32 *val)
166 {
167 const struct rt_config *cfg;
168 u32 value;
169 int ret;
170
171 for (cfg = ndev->priv->rt_config; cfg->type; cfg++) {
172 if (cfg->category != category)
173 continue;
174
175 if (cfg->feature_mask &&
176 bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX))
177 continue;
178
179 value = val ? *val : cfg->value;
180 ret = aie2_set_runtime_cfg(ndev, cfg->type, value);
181 if (ret) {
182 XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
183 cfg->type, value);
184 return ret;
185 }
186 }
187
188 return 0;
189 }
190
aie2_xdna_reset(struct amdxdna_dev_hdl * ndev)191 static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
192 {
193 int ret;
194
195 ret = aie2_suspend_fw(ndev);
196 if (ret) {
197 XDNA_ERR(ndev->xdna, "Suspend firmware failed");
198 return ret;
199 }
200
201 ret = aie2_resume_fw(ndev);
202 if (ret) {
203 XDNA_ERR(ndev->xdna, "Resume firmware failed");
204 return ret;
205 }
206
207 return 0;
208 }
209
aie2_mgmt_fw_init(struct amdxdna_dev_hdl * ndev)210 static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
211 {
212 int ret;
213
214 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL);
215 if (ret) {
216 XDNA_ERR(ndev->xdna, "Runtime config failed");
217 return ret;
218 }
219
220 ret = aie2_assign_mgmt_pasid(ndev, 0);
221 if (ret) {
222 XDNA_ERR(ndev->xdna, "Can not assign PASID");
223 return ret;
224 }
225
226 ret = aie2_xdna_reset(ndev);
227 if (ret) {
228 XDNA_ERR(ndev->xdna, "Reset firmware failed");
229 return ret;
230 }
231
232 return 0;
233 }
234
aie2_mgmt_fw_query(struct amdxdna_dev_hdl * ndev)235 static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
236 {
237 int ret;
238
239 ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
240 if (ret) {
241 XDNA_ERR(ndev->xdna, "query firmware version failed");
242 return ret;
243 }
244
245 ret = aie2_query_aie_version(ndev, &ndev->version);
246 if (ret) {
247 XDNA_ERR(ndev->xdna, "Query AIE version failed");
248 return ret;
249 }
250
251 ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
252 if (ret) {
253 XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
254 return ret;
255 }
256
257 ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
258
259 return 0;
260 }
261
aie2_mgmt_fw_fini(struct amdxdna_dev_hdl * ndev)262 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
263 {
264 if (aie2_suspend_fw(ndev))
265 XDNA_ERR(ndev->xdna, "Suspend_fw failed");
266 XDNA_DBG(ndev->xdna, "Firmware suspended");
267 }
268
aie2_xrs_load(void * cb_arg,struct xrs_action_load * action)269 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
270 {
271 struct amdxdna_hwctx *hwctx = cb_arg;
272 struct amdxdna_dev *xdna;
273 int ret;
274
275 xdna = hwctx->client->xdna;
276
277 hwctx->start_col = action->part.start_col;
278 hwctx->num_col = action->part.ncols;
279 ret = aie2_create_context(xdna->dev_handle, hwctx);
280 if (ret)
281 XDNA_ERR(xdna, "create context failed, ret %d", ret);
282
283 return ret;
284 }
285
aie2_xrs_unload(void * cb_arg)286 static int aie2_xrs_unload(void *cb_arg)
287 {
288 struct amdxdna_hwctx *hwctx = cb_arg;
289 struct amdxdna_dev *xdna;
290 int ret;
291
292 xdna = hwctx->client->xdna;
293
294 ret = aie2_destroy_context(xdna->dev_handle, hwctx);
295 if (ret)
296 XDNA_ERR(xdna, "destroy context failed, ret %d", ret);
297
298 return ret;
299 }
300
aie2_xrs_set_dft_dpm_level(struct drm_device * ddev,u32 dpm_level)301 static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
302 {
303 struct amdxdna_dev *xdna = to_xdna_dev(ddev);
304 struct amdxdna_dev_hdl *ndev;
305
306 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
307
308 ndev = xdna->dev_handle;
309 ndev->dft_dpm_level = dpm_level;
310 if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
311 return 0;
312
313 return aie2_pm_set_dpm(ndev, dpm_level);
314 }
315
316 static struct xrs_action_ops aie2_xrs_actions = {
317 .load = aie2_xrs_load,
318 .unload = aie2_xrs_unload,
319 .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
320 };
321
aie2_hw_stop(struct amdxdna_dev * xdna)322 static void aie2_hw_stop(struct amdxdna_dev *xdna)
323 {
324 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
325 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
326
327 if (ndev->dev_status <= AIE2_DEV_INIT) {
328 XDNA_ERR(xdna, "device is already stopped");
329 return;
330 }
331
332 aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL);
333 aie2_mgmt_fw_fini(ndev);
334 aie2_destroy_mgmt_chann(ndev);
335 drmm_kfree(&xdna->ddev, ndev->mbox);
336 ndev->mbox = NULL;
337 aie2_psp_stop(ndev->psp_hdl);
338 aie2_smu_fini(ndev);
339 aie2_error_async_events_free(ndev);
340 pci_disable_device(pdev);
341
342 ndev->dev_status = AIE2_DEV_INIT;
343 }
344
aie2_hw_start(struct amdxdna_dev * xdna)345 static int aie2_hw_start(struct amdxdna_dev *xdna)
346 {
347 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
348 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
349 struct xdna_mailbox_res mbox_res;
350 u32 xdna_mailbox_intr_reg;
351 int mgmt_mb_irq, ret;
352
353 if (ndev->dev_status >= AIE2_DEV_START) {
354 XDNA_INFO(xdna, "device is already started");
355 return 0;
356 }
357
358 ret = pci_enable_device(pdev);
359 if (ret) {
360 XDNA_ERR(xdna, "failed to enable device, ret %d", ret);
361 return ret;
362 }
363 pci_set_master(pdev);
364
365 mbox_res.ringbuf_base = ndev->sram_base;
366 mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
367 mbox_res.mbox_base = ndev->mbox_base;
368 mbox_res.mbox_size = MBOX_SIZE(ndev);
369 mbox_res.name = "xdna_mailbox";
370 ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
371 if (!ndev->mbox) {
372 XDNA_ERR(xdna, "failed to create mailbox device");
373 ret = -ENODEV;
374 goto disable_dev;
375 }
376
377 ndev->mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox);
378 if (!ndev->mgmt_chann) {
379 XDNA_ERR(xdna, "failed to alloc channel");
380 ret = -ENODEV;
381 goto disable_dev;
382 }
383
384 ret = aie2_smu_init(ndev);
385 if (ret) {
386 XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
387 goto free_channel;
388 }
389
390 ret = aie2_psp_start(ndev->psp_hdl);
391 if (ret) {
392 XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
393 goto fini_smu;
394 }
395
396 ret = aie2_get_mgmt_chann_info(ndev);
397 if (ret) {
398 XDNA_ERR(xdna, "firmware is not alive");
399 goto stop_psp;
400 }
401
402 mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
403 if (mgmt_mb_irq < 0) {
404 ret = mgmt_mb_irq;
405 XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
406 goto stop_psp;
407 }
408
409 xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
410 ret = xdna_mailbox_start_channel(ndev->mgmt_chann,
411 &ndev->mgmt_x2i,
412 &ndev->mgmt_i2x,
413 xdna_mailbox_intr_reg,
414 mgmt_mb_irq);
415 if (ret) {
416 XDNA_ERR(xdna, "failed to start management mailbox channel");
417 ret = -EINVAL;
418 goto stop_psp;
419 }
420
421 ret = aie2_mgmt_fw_init(ndev);
422 if (ret) {
423 XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
424 goto stop_fw;
425 }
426
427 ret = aie2_pm_init(ndev);
428 if (ret) {
429 XDNA_ERR(xdna, "failed to init pm, ret %d", ret);
430 goto stop_fw;
431 }
432
433 ret = aie2_mgmt_fw_query(ndev);
434 if (ret) {
435 XDNA_ERR(xdna, "failed to query fw, ret %d", ret);
436 goto stop_fw;
437 }
438
439 ret = aie2_error_async_events_alloc(ndev);
440 if (ret) {
441 XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
442 goto stop_fw;
443 }
444
445 ndev->dev_status = AIE2_DEV_START;
446
447 return 0;
448
449 stop_fw:
450 aie2_suspend_fw(ndev);
451 xdna_mailbox_stop_channel(ndev->mgmt_chann);
452 stop_psp:
453 aie2_psp_stop(ndev->psp_hdl);
454 fini_smu:
455 aie2_smu_fini(ndev);
456 free_channel:
457 xdna_mailbox_free_channel(ndev->mgmt_chann);
458 ndev->mgmt_chann = NULL;
459 disable_dev:
460 pci_disable_device(pdev);
461
462 return ret;
463 }
464
aie2_hw_suspend(struct amdxdna_dev * xdna)465 static int aie2_hw_suspend(struct amdxdna_dev *xdna)
466 {
467 struct amdxdna_client *client;
468
469 list_for_each_entry(client, &xdna->client_list, node)
470 aie2_hwctx_suspend(client);
471
472 aie2_hw_stop(xdna);
473
474 return 0;
475 }
476
aie2_hw_resume(struct amdxdna_dev * xdna)477 static int aie2_hw_resume(struct amdxdna_dev *xdna)
478 {
479 struct amdxdna_client *client;
480 int ret;
481
482 ret = aie2_hw_start(xdna);
483 if (ret) {
484 XDNA_ERR(xdna, "Start hardware failed, %d", ret);
485 return ret;
486 }
487
488 list_for_each_entry(client, &xdna->client_list, node) {
489 ret = aie2_hwctx_resume(client);
490 if (ret)
491 break;
492 }
493
494 return ret;
495 }
496
aie2_init(struct amdxdna_dev * xdna)497 static int aie2_init(struct amdxdna_dev *xdna)
498 {
499 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
500 void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
501 struct init_config xrs_cfg = { 0 };
502 struct amdxdna_dev_hdl *ndev;
503 struct psp_config psp_conf;
504 const struct firmware *fw;
505 unsigned long bars = 0;
506 char *fw_full_path;
507 int i, nvec, ret;
508
509 if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
510 XDNA_ERR(xdna, "Running under hypervisor not supported");
511 return -EINVAL;
512 }
513
514 if (!xdna->group) {
515 XDNA_ERR(xdna, "Running without IOMMU not supported");
516 return -EINVAL;
517 }
518
519 ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
520 if (!ndev)
521 return -ENOMEM;
522
523 ndev->priv = xdna->dev_info->dev_priv;
524 ndev->xdna = xdna;
525
526 for (i = 0; i < ARRAY_SIZE(npu_fw); i++) {
527 fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]);
528 if (!fw_full_path)
529 return -ENOMEM;
530
531 ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev);
532 kfree(fw_full_path);
533 if (!ret) {
534 XDNA_INFO(xdna, "Load firmware %s%s", ndev->priv->fw_path, npu_fw[i]);
535 break;
536 }
537 }
538
539 if (ret) {
540 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
541 ndev->priv->fw_path, ret);
542 return ret;
543 }
544
545 ret = pcim_enable_device(pdev);
546 if (ret) {
547 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
548 goto release_fw;
549 }
550
551 for (i = 0; i < PSP_MAX_REGS; i++)
552 set_bit(PSP_REG_BAR(ndev, i), &bars);
553
554 set_bit(xdna->dev_info->sram_bar, &bars);
555 set_bit(xdna->dev_info->smu_bar, &bars);
556 set_bit(xdna->dev_info->mbox_bar, &bars);
557
558 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
559 if (!test_bit(i, &bars))
560 continue;
561 tbl[i] = pcim_iomap(pdev, i, 0);
562 if (!tbl[i]) {
563 XDNA_ERR(xdna, "map bar %d failed", i);
564 ret = -ENOMEM;
565 goto release_fw;
566 }
567 }
568
569 ndev->sram_base = tbl[xdna->dev_info->sram_bar];
570 ndev->smu_base = tbl[xdna->dev_info->smu_bar];
571 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
572
573 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
574 if (ret) {
575 XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret);
576 goto release_fw;
577 }
578
579 nvec = pci_msix_vec_count(pdev);
580 if (nvec <= 0) {
581 XDNA_ERR(xdna, "does not get number of interrupt vector");
582 ret = -EINVAL;
583 goto release_fw;
584 }
585
586 ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
587 if (ret < 0) {
588 XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret);
589 goto release_fw;
590 }
591
592 psp_conf.fw_size = fw->size;
593 psp_conf.fw_buf = fw->data;
594 for (i = 0; i < PSP_MAX_REGS; i++)
595 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
596 ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf);
597 if (!ndev->psp_hdl) {
598 XDNA_ERR(xdna, "failed to create psp");
599 ret = -ENOMEM;
600 goto release_fw;
601 }
602 xdna->dev_handle = ndev;
603
604 ret = aie2_hw_start(xdna);
605 if (ret) {
606 XDNA_ERR(xdna, "start npu failed, ret %d", ret);
607 goto release_fw;
608 }
609
610 xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
611 for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
612 xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
613 xrs_cfg.sys_eff_factor = 1;
614 xrs_cfg.ddev = &xdna->ddev;
615 xrs_cfg.actions = &aie2_xrs_actions;
616 xrs_cfg.total_col = ndev->total_col;
617
618 xdna->xrs_hdl = xrsm_init(&xrs_cfg);
619 if (!xdna->xrs_hdl) {
620 XDNA_ERR(xdna, "Initialize resolver failed");
621 ret = -EINVAL;
622 goto stop_hw;
623 }
624
625 release_firmware(fw);
626 aie2_msg_init(ndev);
627 amdxdna_pm_init(xdna);
628 return 0;
629
630 stop_hw:
631 aie2_hw_stop(xdna);
632 release_fw:
633 release_firmware(fw);
634
635 return ret;
636 }
637
aie2_fini(struct amdxdna_dev * xdna)638 static void aie2_fini(struct amdxdna_dev *xdna)
639 {
640 amdxdna_pm_fini(xdna);
641 aie2_hw_stop(xdna);
642 }
643
aie2_get_aie_status(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)644 static int aie2_get_aie_status(struct amdxdna_client *client,
645 struct amdxdna_drm_get_info *args)
646 {
647 struct amdxdna_drm_query_aie_status status;
648 struct amdxdna_dev *xdna = client->xdna;
649 struct amdxdna_dev_hdl *ndev;
650 int ret;
651
652 ndev = xdna->dev_handle;
653 if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) {
654 XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
655 return -EFAULT;
656 }
657
658 if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
659 XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
660 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
661 return -EINVAL;
662 }
663
664 ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
665 status.buffer_size, &status.cols_filled);
666 if (ret) {
667 XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret);
668 return ret;
669 }
670
671 if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) {
672 XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
673 return -EFAULT;
674 }
675
676 return 0;
677 }
678
aie2_get_aie_metadata(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)679 static int aie2_get_aie_metadata(struct amdxdna_client *client,
680 struct amdxdna_drm_get_info *args)
681 {
682 struct amdxdna_drm_query_aie_metadata *meta;
683 struct amdxdna_dev *xdna = client->xdna;
684 struct amdxdna_dev_hdl *ndev;
685 int ret = 0;
686
687 ndev = xdna->dev_handle;
688 meta = kzalloc_obj(*meta);
689 if (!meta)
690 return -ENOMEM;
691
692 meta->col_size = ndev->metadata.size;
693 meta->cols = ndev->metadata.cols;
694 meta->rows = ndev->metadata.rows;
695
696 meta->version.major = ndev->metadata.version.major;
697 meta->version.minor = ndev->metadata.version.minor;
698
699 meta->core.row_count = ndev->metadata.core.row_count;
700 meta->core.row_start = ndev->metadata.core.row_start;
701 meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
702 meta->core.lock_count = ndev->metadata.core.lock_count;
703 meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
704
705 meta->mem.row_count = ndev->metadata.mem.row_count;
706 meta->mem.row_start = ndev->metadata.mem.row_start;
707 meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
708 meta->mem.lock_count = ndev->metadata.mem.lock_count;
709 meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
710
711 meta->shim.row_count = ndev->metadata.shim.row_count;
712 meta->shim.row_start = ndev->metadata.shim.row_start;
713 meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
714 meta->shim.lock_count = ndev->metadata.shim.lock_count;
715 meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
716
717 if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta)))
718 ret = -EFAULT;
719
720 kfree(meta);
721 return ret;
722 }
723
aie2_get_aie_version(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)724 static int aie2_get_aie_version(struct amdxdna_client *client,
725 struct amdxdna_drm_get_info *args)
726 {
727 struct amdxdna_drm_query_aie_version version;
728 struct amdxdna_dev *xdna = client->xdna;
729 struct amdxdna_dev_hdl *ndev;
730
731 ndev = xdna->dev_handle;
732 version.major = ndev->version.major;
733 version.minor = ndev->version.minor;
734
735 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
736 return -EFAULT;
737
738 return 0;
739 }
740
aie2_get_firmware_version(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)741 static int aie2_get_firmware_version(struct amdxdna_client *client,
742 struct amdxdna_drm_get_info *args)
743 {
744 struct amdxdna_drm_query_firmware_version version;
745 struct amdxdna_dev *xdna = client->xdna;
746
747 version.major = xdna->fw_ver.major;
748 version.minor = xdna->fw_ver.minor;
749 version.patch = xdna->fw_ver.sub;
750 version.build = xdna->fw_ver.build;
751
752 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
753 return -EFAULT;
754
755 return 0;
756 }
757
aie2_get_power_mode(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)758 static int aie2_get_power_mode(struct amdxdna_client *client,
759 struct amdxdna_drm_get_info *args)
760 {
761 struct amdxdna_drm_get_power_mode mode = {};
762 struct amdxdna_dev *xdna = client->xdna;
763 struct amdxdna_dev_hdl *ndev;
764
765 ndev = xdna->dev_handle;
766 mode.power_mode = ndev->pw_mode;
767
768 if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode)))
769 return -EFAULT;
770
771 return 0;
772 }
773
aie2_get_clock_metadata(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)774 static int aie2_get_clock_metadata(struct amdxdna_client *client,
775 struct amdxdna_drm_get_info *args)
776 {
777 struct amdxdna_drm_query_clock_metadata *clock;
778 struct amdxdna_dev *xdna = client->xdna;
779 struct amdxdna_dev_hdl *ndev;
780 int ret = 0;
781
782 ndev = xdna->dev_handle;
783 clock = kzalloc_obj(*clock);
784 if (!clock)
785 return -ENOMEM;
786
787 snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name),
788 "MP-NPU Clock");
789 clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
790 snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock");
791 clock->h_clock.freq_mhz = ndev->hclk_freq;
792
793 if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock)))
794 ret = -EFAULT;
795
796 kfree(clock);
797 return ret;
798 }
799
aie2_get_sensors(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)800 static int aie2_get_sensors(struct amdxdna_client *client,
801 struct amdxdna_drm_get_info *args)
802 {
803 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
804 struct amdxdna_drm_query_sensor sensor = {};
805 struct amd_pmf_npu_metrics npu_metrics;
806 u32 sensors_count = 0, i;
807 int ret;
808
809 ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics);
810 if (ret)
811 return ret;
812
813 sensor.type = AMDXDNA_SENSOR_TYPE_POWER;
814 sensor.input = npu_metrics.npu_power;
815 sensor.unitm = -3;
816 scnprintf(sensor.label, sizeof(sensor.label), "Total Power");
817 scnprintf(sensor.units, sizeof(sensor.units), "mW");
818
819 if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor)))
820 return -EFAULT;
821
822 sensors_count++;
823 if (args->buffer_size <= sensors_count * sizeof(sensor))
824 goto out;
825
826 for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) {
827 memset(&sensor, 0, sizeof(sensor));
828 sensor.input = npu_metrics.npu_busy[i];
829 sensor.type = AMDXDNA_SENSOR_TYPE_COLUMN_UTILIZATION;
830 sensor.unitm = 0;
831 scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i);
832 scnprintf(sensor.units, sizeof(sensor.units), "%%");
833
834 if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor),
835 &sensor, sizeof(sensor)))
836 return -EFAULT;
837
838 sensors_count++;
839 if (args->buffer_size <= sensors_count * sizeof(sensor))
840 goto out;
841 }
842
843 out:
844 args->buffer_size = sensors_count * sizeof(sensor);
845
846 return 0;
847 }
848
aie2_hwctx_status_cb(struct amdxdna_hwctx * hwctx,void * arg)849 static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
850 {
851 struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL;
852 struct amdxdna_drm_get_array *array_args = arg;
853 struct amdxdna_drm_hwctx_entry __user *buf;
854 struct app_health_report report;
855 struct amdxdna_dev_hdl *ndev;
856 u32 size;
857 int ret;
858
859 if (!array_args->num_element)
860 return -EINVAL;
861
862 tmp = kzalloc_obj(*tmp);
863 if (!tmp)
864 return -ENOMEM;
865
866 tmp->pid = hwctx->client->pid;
867 tmp->context_id = hwctx->id;
868 tmp->start_col = hwctx->start_col;
869 tmp->num_col = hwctx->num_col;
870 tmp->command_submissions = hwctx->priv->seq;
871 tmp->command_completions = hwctx->priv->completed;
872 tmp->pasid = hwctx->client->pasid;
873 tmp->heap_usage = hwctx->client->heap_usage;
874 tmp->priority = hwctx->qos.priority;
875 tmp->gops = hwctx->qos.gops;
876 tmp->fps = hwctx->qos.fps;
877 tmp->dma_bandwidth = hwctx->qos.dma_bandwidth;
878 tmp->latency = hwctx->qos.latency;
879 tmp->frame_exec_time = hwctx->qos.frame_exec_time;
880 tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE;
881 ndev = hwctx->client->xdna->dev_handle;
882 ret = aie2_query_app_health(ndev, hwctx->fw_ctx_id, &report);
883 if (!ret) {
884 /* Fill in app health report fields */
885 tmp->txn_op_idx = report.txn_op_id;
886 tmp->ctx_pc = report.ctx_pc;
887 tmp->fatal_error_type = report.fatal_info.fatal_type;
888 tmp->fatal_error_exception_type = report.fatal_info.exception_type;
889 tmp->fatal_error_exception_pc = report.fatal_info.exception_pc;
890 tmp->fatal_error_app_module = report.fatal_info.app_module;
891 }
892
893 buf = u64_to_user_ptr(array_args->buffer);
894 size = min(sizeof(*tmp), array_args->element_size);
895
896 if (copy_to_user(buf, tmp, size))
897 return -EFAULT;
898
899 array_args->buffer += size;
900 array_args->num_element--;
901
902 return 0;
903 }
904
aie2_get_hwctx_status(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)905 static int aie2_get_hwctx_status(struct amdxdna_client *client,
906 struct amdxdna_drm_get_info *args)
907 {
908 struct amdxdna_drm_get_array array_args;
909 struct amdxdna_dev *xdna = client->xdna;
910 struct amdxdna_client *tmp_client;
911 int ret;
912
913 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
914
915 array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx);
916 array_args.buffer = args->buffer;
917 array_args.num_element = args->buffer_size / array_args.element_size;
918 list_for_each_entry(tmp_client, &xdna->client_list, node) {
919 ret = amdxdna_hwctx_walk(tmp_client, &array_args,
920 aie2_hwctx_status_cb);
921 if (ret)
922 break;
923 }
924
925 args->buffer_size -= (u32)(array_args.buffer - args->buffer);
926 return 0;
927 }
928
aie2_query_resource_info(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)929 static int aie2_query_resource_info(struct amdxdna_client *client,
930 struct amdxdna_drm_get_info *args)
931 {
932 struct amdxdna_drm_get_resource_info res_info;
933 const struct amdxdna_dev_priv *priv;
934 struct amdxdna_dev_hdl *ndev;
935 struct amdxdna_dev *xdna;
936
937 xdna = client->xdna;
938 ndev = xdna->dev_handle;
939 priv = ndev->priv;
940
941 res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk;
942 res_info.npu_tops_max = ndev->max_tops;
943 res_info.npu_task_max = priv->hwctx_limit;
944 res_info.npu_tops_curr = ndev->curr_tops;
945 res_info.npu_task_curr = ndev->hwctx_num;
946
947 if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info)))
948 return -EFAULT;
949
950 return 0;
951 }
952
aie2_fill_hwctx_map(struct amdxdna_hwctx * hwctx,void * arg)953 static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg)
954 {
955 struct amdxdna_dev *xdna = hwctx->client->xdna;
956 u32 *map = arg;
957
958 if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) {
959 XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id,
960 xdna->dev_handle->priv->hwctx_limit);
961 return -EINVAL;
962 }
963
964 map[hwctx->fw_ctx_id] = hwctx->id;
965 return 0;
966 }
967
aie2_get_telemetry(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)968 static int aie2_get_telemetry(struct amdxdna_client *client,
969 struct amdxdna_drm_get_info *args)
970 {
971 struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL;
972 u32 telemetry_data_sz, header_sz, elem_num;
973 struct amdxdna_dev *xdna = client->xdna;
974 struct amdxdna_client *tmp_client;
975 int ret;
976
977 elem_num = xdna->dev_handle->priv->hwctx_limit;
978 header_sz = struct_size(header, map, elem_num);
979 if (args->buffer_size <= header_sz) {
980 XDNA_ERR(xdna, "Invalid buffer size");
981 return -EINVAL;
982 }
983
984 telemetry_data_sz = args->buffer_size - header_sz;
985 if (telemetry_data_sz > SZ_4M) {
986 XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz);
987 return -EINVAL;
988 }
989
990 header = kzalloc(header_sz, GFP_KERNEL);
991 if (!header)
992 return -ENOMEM;
993
994 if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) {
995 XDNA_ERR(xdna, "Failed to copy telemetry header from user");
996 return -EFAULT;
997 }
998
999 header->map_num_elements = elem_num;
1000 list_for_each_entry(tmp_client, &xdna->client_list, node) {
1001 ret = amdxdna_hwctx_walk(tmp_client, &header->map,
1002 aie2_fill_hwctx_map);
1003 if (ret)
1004 return ret;
1005 }
1006
1007 ret = aie2_query_telemetry(xdna->dev_handle,
1008 u64_to_user_ptr(args->buffer + header_sz),
1009 telemetry_data_sz, header);
1010 if (ret) {
1011 XDNA_ERR(xdna, "Query telemetry failed ret %d", ret);
1012 return ret;
1013 }
1014
1015 if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) {
1016 XDNA_ERR(xdna, "Copy header failed");
1017 return -EFAULT;
1018 }
1019
1020 return 0;
1021 }
1022
aie2_get_preempt_state(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)1023 static int aie2_get_preempt_state(struct amdxdna_client *client,
1024 struct amdxdna_drm_get_info *args)
1025 {
1026 struct amdxdna_drm_attribute_state state = {};
1027 struct amdxdna_dev *xdna = client->xdna;
1028 struct amdxdna_dev_hdl *ndev;
1029
1030 ndev = xdna->dev_handle;
1031 if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE)
1032 state.state = ndev->force_preempt_enabled;
1033 else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE)
1034 state.state = ndev->frame_boundary_preempt;
1035
1036 if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state)))
1037 return -EFAULT;
1038
1039 return 0;
1040 }
1041
aie2_get_info(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)1042 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
1043 {
1044 struct amdxdna_dev *xdna = client->xdna;
1045 int ret, idx;
1046
1047 if (!drm_dev_enter(&xdna->ddev, &idx))
1048 return -ENODEV;
1049
1050 ret = amdxdna_pm_resume_get_locked(xdna);
1051 if (ret)
1052 goto dev_exit;
1053
1054 switch (args->param) {
1055 case DRM_AMDXDNA_QUERY_AIE_STATUS:
1056 ret = aie2_get_aie_status(client, args);
1057 break;
1058 case DRM_AMDXDNA_QUERY_AIE_METADATA:
1059 ret = aie2_get_aie_metadata(client, args);
1060 break;
1061 case DRM_AMDXDNA_QUERY_AIE_VERSION:
1062 ret = aie2_get_aie_version(client, args);
1063 break;
1064 case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
1065 ret = aie2_get_clock_metadata(client, args);
1066 break;
1067 case DRM_AMDXDNA_QUERY_SENSORS:
1068 ret = aie2_get_sensors(client, args);
1069 break;
1070 case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
1071 ret = aie2_get_hwctx_status(client, args);
1072 break;
1073 case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION:
1074 ret = aie2_get_firmware_version(client, args);
1075 break;
1076 case DRM_AMDXDNA_GET_POWER_MODE:
1077 ret = aie2_get_power_mode(client, args);
1078 break;
1079 case DRM_AMDXDNA_QUERY_TELEMETRY:
1080 ret = aie2_get_telemetry(client, args);
1081 break;
1082 case DRM_AMDXDNA_QUERY_RESOURCE_INFO:
1083 ret = aie2_query_resource_info(client, args);
1084 break;
1085 case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE:
1086 case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE:
1087 ret = aie2_get_preempt_state(client, args);
1088 break;
1089 default:
1090 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1091 ret = -EOPNOTSUPP;
1092 }
1093
1094 amdxdna_pm_suspend_put(xdna);
1095 XDNA_DBG(xdna, "Got param %d", args->param);
1096
1097 dev_exit:
1098 drm_dev_exit(idx);
1099 return ret;
1100 }
1101
aie2_query_ctx_status_array(struct amdxdna_client * client,struct amdxdna_drm_get_array * args)1102 static int aie2_query_ctx_status_array(struct amdxdna_client *client,
1103 struct amdxdna_drm_get_array *args)
1104 {
1105 struct amdxdna_drm_get_array array_args;
1106 struct amdxdna_dev *xdna = client->xdna;
1107 struct amdxdna_client *tmp_client;
1108 int ret;
1109
1110 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
1111
1112 if (args->element_size > SZ_4K || args->num_element > SZ_1K) {
1113 XDNA_DBG(xdna, "Invalid element size %d or number of element %d",
1114 args->element_size, args->num_element);
1115 return -EINVAL;
1116 }
1117
1118 array_args.element_size = min(args->element_size,
1119 sizeof(struct amdxdna_drm_hwctx_entry));
1120 array_args.buffer = args->buffer;
1121 array_args.num_element = args->num_element * args->element_size /
1122 array_args.element_size;
1123 list_for_each_entry(tmp_client, &xdna->client_list, node) {
1124 ret = amdxdna_hwctx_walk(tmp_client, &array_args,
1125 aie2_hwctx_status_cb);
1126 if (ret)
1127 break;
1128 }
1129
1130 args->element_size = array_args.element_size;
1131 args->num_element = (u32)((array_args.buffer - args->buffer) /
1132 args->element_size);
1133
1134 return 0;
1135 }
1136
aie2_get_array(struct amdxdna_client * client,struct amdxdna_drm_get_array * args)1137 static int aie2_get_array(struct amdxdna_client *client,
1138 struct amdxdna_drm_get_array *args)
1139 {
1140 struct amdxdna_dev *xdna = client->xdna;
1141 int ret, idx;
1142
1143 if (!drm_dev_enter(&xdna->ddev, &idx))
1144 return -ENODEV;
1145
1146 ret = amdxdna_pm_resume_get_locked(xdna);
1147 if (ret)
1148 goto dev_exit;
1149
1150 switch (args->param) {
1151 case DRM_AMDXDNA_HW_CONTEXT_ALL:
1152 ret = aie2_query_ctx_status_array(client, args);
1153 break;
1154 case DRM_AMDXDNA_HW_LAST_ASYNC_ERR:
1155 ret = aie2_get_array_async_error(xdna->dev_handle, args);
1156 break;
1157 case DRM_AMDXDNA_BO_USAGE:
1158 ret = amdxdna_drm_get_bo_usage(&xdna->ddev, args);
1159 break;
1160 default:
1161 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1162 ret = -EOPNOTSUPP;
1163 }
1164
1165 amdxdna_pm_suspend_put(xdna);
1166 XDNA_DBG(xdna, "Got param %d", args->param);
1167
1168 dev_exit:
1169 drm_dev_exit(idx);
1170 return ret;
1171 }
1172
aie2_set_power_mode(struct amdxdna_client * client,struct amdxdna_drm_set_state * args)1173 static int aie2_set_power_mode(struct amdxdna_client *client,
1174 struct amdxdna_drm_set_state *args)
1175 {
1176 struct amdxdna_drm_set_power_mode power_state;
1177 enum amdxdna_power_mode_type power_mode;
1178 struct amdxdna_dev *xdna = client->xdna;
1179
1180 if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer),
1181 sizeof(power_state))) {
1182 XDNA_ERR(xdna, "Failed to copy power mode request into kernel");
1183 return -EFAULT;
1184 }
1185
1186 if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad)))
1187 return -EINVAL;
1188
1189 power_mode = power_state.power_mode;
1190 if (power_mode > POWER_MODE_TURBO) {
1191 XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
1192 return -EINVAL;
1193 }
1194
1195 return aie2_pm_set_mode(xdna->dev_handle, power_mode);
1196 }
1197
aie2_set_preempt_state(struct amdxdna_client * client,struct amdxdna_drm_set_state * args)1198 static int aie2_set_preempt_state(struct amdxdna_client *client,
1199 struct amdxdna_drm_set_state *args)
1200 {
1201 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
1202 struct amdxdna_drm_attribute_state state;
1203 u32 val;
1204 int ret;
1205
1206 if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state)))
1207 return -EFAULT;
1208
1209 if (state.state > 1)
1210 return -EINVAL;
1211
1212 if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad)))
1213 return -EINVAL;
1214
1215 if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) {
1216 ndev->force_preempt_enabled = state.state;
1217 } else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) {
1218 val = state.state;
1219 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT,
1220 &val);
1221 if (ret)
1222 return ret;
1223
1224 ndev->frame_boundary_preempt = state.state;
1225 }
1226
1227 return 0;
1228 }
1229
aie2_set_state(struct amdxdna_client * client,struct amdxdna_drm_set_state * args)1230 static int aie2_set_state(struct amdxdna_client *client,
1231 struct amdxdna_drm_set_state *args)
1232 {
1233 struct amdxdna_dev *xdna = client->xdna;
1234 int ret, idx;
1235
1236 if (!drm_dev_enter(&xdna->ddev, &idx))
1237 return -ENODEV;
1238
1239 ret = amdxdna_pm_resume_get_locked(xdna);
1240 if (ret)
1241 goto dev_exit;
1242
1243 switch (args->param) {
1244 case DRM_AMDXDNA_SET_POWER_MODE:
1245 ret = aie2_set_power_mode(client, args);
1246 break;
1247 case DRM_AMDXDNA_SET_FORCE_PREEMPT:
1248 case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT:
1249 ret = aie2_set_preempt_state(client, args);
1250 break;
1251 default:
1252 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1253 ret = -EOPNOTSUPP;
1254 break;
1255 }
1256
1257 amdxdna_pm_suspend_put(xdna);
1258 dev_exit:
1259 drm_dev_exit(idx);
1260 return ret;
1261 }
1262
1263 const struct amdxdna_dev_ops aie2_ops = {
1264 .init = aie2_init,
1265 .fini = aie2_fini,
1266 .resume = aie2_hw_resume,
1267 .suspend = aie2_hw_suspend,
1268 .get_aie_info = aie2_get_info,
1269 .set_aie_state = aie2_set_state,
1270 .hwctx_init = aie2_hwctx_init,
1271 .hwctx_fini = aie2_hwctx_fini,
1272 .hwctx_config = aie2_hwctx_config,
1273 .hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo,
1274 .cmd_submit = aie2_cmd_submit,
1275 .hmm_invalidate = aie2_hmm_invalidate,
1276 .get_array = aie2_get_array,
1277 };
1278