xref: /linux/drivers/accel/amdxdna/aie2_pci.c (revision b0dcdcb9ae757c8a8ba2fb24d34f8d147bae707b) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4  */
5 
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_gem_shmem_helper.h>
10 #include <drm/drm_managed.h>
11 #include <drm/drm_print.h>
12 #include <drm/gpu_scheduler.h>
13 #include <linux/cleanup.h>
14 #include <linux/errno.h>
15 #include <linux/firmware.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/pci.h>
19 #include <linux/xarray.h>
20 #include <asm/hypervisor.h>
21 
22 #include "aie2_msg_priv.h"
23 #include "aie2_pci.h"
24 #include "aie2_solver.h"
25 #include "amdxdna_ctx.h"
26 #include "amdxdna_gem.h"
27 #include "amdxdna_mailbox.h"
28 #include "amdxdna_pci_drv.h"
29 #include "amdxdna_pm.h"
30 
31 static int aie2_max_col = XRS_MAX_COL;
32 module_param(aie2_max_col, uint, 0600);
33 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
34 
35 static char *npu_fw[] = {
36 	"npu_7.sbin",
37 	"npu.sbin"
38 };
39 
40 /*
41  * The management mailbox channel is allocated by firmware.
42  * The related register and ring buffer information is on SRAM BAR.
43  * This struct is the register layout.
44  */
45 #define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */
46 struct mgmt_mbox_chann_info {
47 	__u32	x2i_tail;
48 	__u32	x2i_head;
49 	__u32	x2i_buf;
50 	__u32	x2i_buf_sz;
51 	__u32	i2x_tail;
52 	__u32	i2x_head;
53 	__u32	i2x_buf;
54 	__u32	i2x_buf_sz;
55 	__u32	magic;
56 	__u32	msi_id;
57 	__u32	prot_major;
58 	__u32	prot_minor;
59 	__u32	rsvd[4];
60 };
61 
62 static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
63 {
64 	const struct aie2_fw_feature_tbl *feature;
65 	bool found = false;
66 
67 	for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) {
68 		if (feature->major != fw_major)
69 			continue;
70 		if (fw_minor < feature->min_minor)
71 			continue;
72 		if (feature->max_minor > 0 && fw_minor > feature->max_minor)
73 			continue;
74 
75 		ndev->feature_mask |= feature->features;
76 
77 		/* firmware version matches one of the driver support entry */
78 		found = true;
79 	}
80 
81 	return found ? 0 : -EOPNOTSUPP;
82 }
83 
84 static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
85 {
86 	struct amdxdna_dev *xdna = ndev->xdna;
87 
88 	XDNA_DBG(xdna, "i2x tail    0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
89 	XDNA_DBG(xdna, "i2x head    0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
90 	XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
91 	XDNA_DBG(xdna, "i2x rsize   0x%x", ndev->mgmt_i2x.rb_size);
92 	XDNA_DBG(xdna, "x2i tail    0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
93 	XDNA_DBG(xdna, "x2i head    0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
94 	XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
95 	XDNA_DBG(xdna, "x2i rsize   0x%x", ndev->mgmt_x2i.rb_size);
96 	XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
97 	XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
98 	XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
99 }
100 
101 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
102 {
103 	struct mgmt_mbox_chann_info info_regs;
104 	struct xdna_mailbox_chann_res *i2x;
105 	struct xdna_mailbox_chann_res *x2i;
106 	u32 addr, off;
107 	u32 *reg;
108 	int ret;
109 	int i;
110 
111 	/*
112 	 * Once firmware is alive, it will write management channel
113 	 * information in SRAM BAR and write the address of that information
114 	 * at FW_ALIVE_OFF offset in SRMA BAR.
115 	 *
116 	 * Read a non-zero value from FW_ALIVE_OFF implies that firmware
117 	 * is alive.
118 	 */
119 	ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
120 				 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
121 	if (ret || !addr)
122 		return -ETIME;
123 
124 	off = AIE2_SRAM_OFF(ndev, addr);
125 	reg = (u32 *)&info_regs;
126 	for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
127 		reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
128 
129 	if (info_regs.magic != MGMT_MBOX_MAGIC) {
130 		XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
131 		ret = -EINVAL;
132 		goto done;
133 	}
134 
135 	i2x = &ndev->mgmt_i2x;
136 	x2i = &ndev->mgmt_x2i;
137 
138 	i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
139 	i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
140 	i2x->rb_start_addr   = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
141 	i2x->rb_size         = info_regs.i2x_buf_sz;
142 
143 	x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
144 	x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
145 	x2i->rb_start_addr   = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
146 	x2i->rb_size         = info_regs.x2i_buf_sz;
147 
148 	ndev->mgmt_chan_idx  = info_regs.msi_id;
149 	ndev->mgmt_prot_major = info_regs.prot_major;
150 	ndev->mgmt_prot_minor = info_regs.prot_minor;
151 
152 	ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor);
153 
154 done:
155 	aie2_dump_chann_info_debug(ndev);
156 
157 	/* Must clear address at FW_ALIVE_OFF */
158 	writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
159 
160 	return ret;
161 }
162 
163 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
164 		     enum rt_config_category category, u32 *val)
165 {
166 	const struct rt_config *cfg;
167 	u32 value;
168 	int ret;
169 
170 	for (cfg = ndev->priv->rt_config; cfg->type; cfg++) {
171 		if (cfg->category != category)
172 			continue;
173 
174 		if (cfg->feature_mask &&
175 		    bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX))
176 			continue;
177 
178 		value = val ? *val : cfg->value;
179 		ret = aie2_set_runtime_cfg(ndev, cfg->type, value);
180 		if (ret) {
181 			XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
182 				 cfg->type, value);
183 			return ret;
184 		}
185 	}
186 
187 	return 0;
188 }
189 
190 static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
191 {
192 	int ret;
193 
194 	ret = aie2_suspend_fw(ndev);
195 	if (ret) {
196 		XDNA_ERR(ndev->xdna, "Suspend firmware failed");
197 		return ret;
198 	}
199 
200 	ret = aie2_resume_fw(ndev);
201 	if (ret) {
202 		XDNA_ERR(ndev->xdna, "Resume firmware failed");
203 		return ret;
204 	}
205 
206 	return 0;
207 }
208 
209 static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
210 {
211 	int ret;
212 
213 	ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL);
214 	if (ret) {
215 		XDNA_ERR(ndev->xdna, "Runtime config failed");
216 		return ret;
217 	}
218 
219 	ret = aie2_assign_mgmt_pasid(ndev, 0);
220 	if (ret) {
221 		XDNA_ERR(ndev->xdna, "Can not assign PASID");
222 		return ret;
223 	}
224 
225 	ret = aie2_xdna_reset(ndev);
226 	if (ret) {
227 		XDNA_ERR(ndev->xdna, "Reset firmware failed");
228 		return ret;
229 	}
230 
231 	return 0;
232 }
233 
234 static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
235 {
236 	int ret;
237 
238 	ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
239 	if (ret) {
240 		XDNA_ERR(ndev->xdna, "query firmware version failed");
241 		return ret;
242 	}
243 
244 	ret = aie2_query_aie_version(ndev, &ndev->version);
245 	if (ret) {
246 		XDNA_ERR(ndev->xdna, "Query AIE version failed");
247 		return ret;
248 	}
249 
250 	ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
251 	if (ret) {
252 		XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
253 		return ret;
254 	}
255 
256 	ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
257 
258 	return 0;
259 }
260 
261 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
262 {
263 	if (aie2_suspend_fw(ndev))
264 		XDNA_ERR(ndev->xdna, "Suspend_fw failed");
265 	XDNA_DBG(ndev->xdna, "Firmware suspended");
266 }
267 
268 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
269 {
270 	struct amdxdna_hwctx *hwctx = cb_arg;
271 	struct amdxdna_dev *xdna;
272 	int ret;
273 
274 	xdna = hwctx->client->xdna;
275 
276 	hwctx->start_col = action->part.start_col;
277 	hwctx->num_col = action->part.ncols;
278 	ret = aie2_create_context(xdna->dev_handle, hwctx);
279 	if (ret)
280 		XDNA_ERR(xdna, "create context failed, ret %d", ret);
281 
282 	return ret;
283 }
284 
285 static int aie2_xrs_unload(void *cb_arg)
286 {
287 	struct amdxdna_hwctx *hwctx = cb_arg;
288 	struct amdxdna_dev *xdna;
289 	int ret;
290 
291 	xdna = hwctx->client->xdna;
292 
293 	ret = aie2_destroy_context(xdna->dev_handle, hwctx);
294 	if (ret)
295 		XDNA_ERR(xdna, "destroy context failed, ret %d", ret);
296 
297 	return ret;
298 }
299 
300 static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
301 {
302 	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
303 	struct amdxdna_dev_hdl *ndev;
304 
305 	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
306 
307 	ndev = xdna->dev_handle;
308 	ndev->dft_dpm_level = dpm_level;
309 	if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
310 		return 0;
311 
312 	return aie2_pm_set_dpm(ndev, dpm_level);
313 }
314 
315 static struct xrs_action_ops aie2_xrs_actions = {
316 	.load = aie2_xrs_load,
317 	.unload = aie2_xrs_unload,
318 	.set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
319 };
320 
321 static void aie2_hw_stop(struct amdxdna_dev *xdna)
322 {
323 	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
324 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
325 
326 	if (ndev->dev_status <= AIE2_DEV_INIT) {
327 		XDNA_ERR(xdna, "device is already stopped");
328 		return;
329 	}
330 
331 	aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL);
332 	aie2_mgmt_fw_fini(ndev);
333 	xdna_mailbox_stop_channel(ndev->mgmt_chann);
334 	xdna_mailbox_destroy_channel(ndev->mgmt_chann);
335 	ndev->mgmt_chann = NULL;
336 	drmm_kfree(&xdna->ddev, ndev->mbox);
337 	ndev->mbox = NULL;
338 	aie2_psp_stop(ndev->psp_hdl);
339 	aie2_smu_fini(ndev);
340 	aie2_error_async_events_free(ndev);
341 	pci_disable_device(pdev);
342 
343 	ndev->dev_status = AIE2_DEV_INIT;
344 }
345 
346 static int aie2_hw_start(struct amdxdna_dev *xdna)
347 {
348 	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
349 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
350 	struct xdna_mailbox_res mbox_res;
351 	u32 xdna_mailbox_intr_reg;
352 	int mgmt_mb_irq, ret;
353 
354 	if (ndev->dev_status >= AIE2_DEV_START) {
355 		XDNA_INFO(xdna, "device is already started");
356 		return 0;
357 	}
358 
359 	ret = pci_enable_device(pdev);
360 	if (ret) {
361 		XDNA_ERR(xdna, "failed to enable device, ret %d", ret);
362 		return ret;
363 	}
364 	pci_set_master(pdev);
365 
366 	ret = aie2_smu_init(ndev);
367 	if (ret) {
368 		XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
369 		goto disable_dev;
370 	}
371 
372 	ret = aie2_psp_start(ndev->psp_hdl);
373 	if (ret) {
374 		XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
375 		goto fini_smu;
376 	}
377 
378 	ret = aie2_get_mgmt_chann_info(ndev);
379 	if (ret) {
380 		XDNA_ERR(xdna, "firmware is not alive");
381 		goto stop_psp;
382 	}
383 
384 	mbox_res.ringbuf_base = ndev->sram_base;
385 	mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
386 	mbox_res.mbox_base = ndev->mbox_base;
387 	mbox_res.mbox_size = MBOX_SIZE(ndev);
388 	mbox_res.name = "xdna_mailbox";
389 	ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
390 	if (!ndev->mbox) {
391 		XDNA_ERR(xdna, "failed to create mailbox device");
392 		ret = -ENODEV;
393 		goto stop_psp;
394 	}
395 
396 	mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
397 	if (mgmt_mb_irq < 0) {
398 		ret = mgmt_mb_irq;
399 		XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
400 		goto stop_psp;
401 	}
402 
403 	xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
404 	ndev->mgmt_chann = xdna_mailbox_create_channel(ndev->mbox,
405 						       &ndev->mgmt_x2i,
406 						       &ndev->mgmt_i2x,
407 						       xdna_mailbox_intr_reg,
408 						       mgmt_mb_irq);
409 	if (!ndev->mgmt_chann) {
410 		XDNA_ERR(xdna, "failed to create management mailbox channel");
411 		ret = -EINVAL;
412 		goto stop_psp;
413 	}
414 
415 	ret = aie2_mgmt_fw_init(ndev);
416 	if (ret) {
417 		XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
418 		goto destroy_mgmt_chann;
419 	}
420 
421 	ret = aie2_pm_init(ndev);
422 	if (ret) {
423 		XDNA_ERR(xdna, "failed to init pm, ret %d", ret);
424 		goto destroy_mgmt_chann;
425 	}
426 
427 	ret = aie2_mgmt_fw_query(ndev);
428 	if (ret) {
429 		XDNA_ERR(xdna, "failed to query fw, ret %d", ret);
430 		goto destroy_mgmt_chann;
431 	}
432 
433 	ret = aie2_error_async_events_alloc(ndev);
434 	if (ret) {
435 		XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
436 		goto destroy_mgmt_chann;
437 	}
438 
439 	ndev->dev_status = AIE2_DEV_START;
440 
441 	return 0;
442 
443 destroy_mgmt_chann:
444 	xdna_mailbox_stop_channel(ndev->mgmt_chann);
445 	xdna_mailbox_destroy_channel(ndev->mgmt_chann);
446 stop_psp:
447 	aie2_psp_stop(ndev->psp_hdl);
448 fini_smu:
449 	aie2_smu_fini(ndev);
450 disable_dev:
451 	pci_disable_device(pdev);
452 
453 	return ret;
454 }
455 
456 static int aie2_hw_suspend(struct amdxdna_dev *xdna)
457 {
458 	struct amdxdna_client *client;
459 
460 	list_for_each_entry(client, &xdna->client_list, node)
461 		aie2_hwctx_suspend(client);
462 
463 	aie2_hw_stop(xdna);
464 
465 	return 0;
466 }
467 
468 static int aie2_hw_resume(struct amdxdna_dev *xdna)
469 {
470 	struct amdxdna_client *client;
471 	int ret;
472 
473 	ret = aie2_hw_start(xdna);
474 	if (ret) {
475 		XDNA_ERR(xdna, "Start hardware failed, %d", ret);
476 		return ret;
477 	}
478 
479 	list_for_each_entry(client, &xdna->client_list, node) {
480 		ret = aie2_hwctx_resume(client);
481 		if (ret)
482 			break;
483 	}
484 
485 	return ret;
486 }
487 
488 static int aie2_init(struct amdxdna_dev *xdna)
489 {
490 	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
491 	void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
492 	struct init_config xrs_cfg = { 0 };
493 	struct amdxdna_dev_hdl *ndev;
494 	struct psp_config psp_conf;
495 	const struct firmware *fw;
496 	unsigned long bars = 0;
497 	char *fw_full_path;
498 	int i, nvec, ret;
499 
500 	if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
501 		XDNA_ERR(xdna, "Running under hypervisor not supported");
502 		return -EINVAL;
503 	}
504 
505 	ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
506 	if (!ndev)
507 		return -ENOMEM;
508 
509 	ndev->priv = xdna->dev_info->dev_priv;
510 	ndev->xdna = xdna;
511 
512 	for (i = 0; i < ARRAY_SIZE(npu_fw); i++) {
513 		fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]);
514 		if (!fw_full_path)
515 			return -ENOMEM;
516 
517 		ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev);
518 		kfree(fw_full_path);
519 		if (!ret) {
520 			XDNA_INFO(xdna, "Load firmware %s%s", ndev->priv->fw_path, npu_fw[i]);
521 			break;
522 		}
523 	}
524 
525 	if (ret) {
526 		XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
527 			 ndev->priv->fw_path, ret);
528 		return ret;
529 	}
530 
531 	ret = pcim_enable_device(pdev);
532 	if (ret) {
533 		XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
534 		goto release_fw;
535 	}
536 
537 	for (i = 0; i < PSP_MAX_REGS; i++)
538 		set_bit(PSP_REG_BAR(ndev, i), &bars);
539 
540 	set_bit(xdna->dev_info->sram_bar, &bars);
541 	set_bit(xdna->dev_info->smu_bar, &bars);
542 	set_bit(xdna->dev_info->mbox_bar, &bars);
543 
544 	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
545 		if (!test_bit(i, &bars))
546 			continue;
547 		tbl[i] = pcim_iomap(pdev, i, 0);
548 		if (!tbl[i]) {
549 			XDNA_ERR(xdna, "map bar %d failed", i);
550 			ret = -ENOMEM;
551 			goto release_fw;
552 		}
553 	}
554 
555 	ndev->sram_base = tbl[xdna->dev_info->sram_bar];
556 	ndev->smu_base = tbl[xdna->dev_info->smu_bar];
557 	ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
558 
559 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
560 	if (ret) {
561 		XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret);
562 		goto release_fw;
563 	}
564 
565 	nvec = pci_msix_vec_count(pdev);
566 	if (nvec <= 0) {
567 		XDNA_ERR(xdna, "does not get number of interrupt vector");
568 		ret = -EINVAL;
569 		goto release_fw;
570 	}
571 
572 	ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
573 	if (ret < 0) {
574 		XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret);
575 		goto release_fw;
576 	}
577 
578 	psp_conf.fw_size = fw->size;
579 	psp_conf.fw_buf = fw->data;
580 	for (i = 0; i < PSP_MAX_REGS; i++)
581 		psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
582 	ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf);
583 	if (!ndev->psp_hdl) {
584 		XDNA_ERR(xdna, "failed to create psp");
585 		ret = -ENOMEM;
586 		goto release_fw;
587 	}
588 	xdna->dev_handle = ndev;
589 
590 	ret = aie2_hw_start(xdna);
591 	if (ret) {
592 		XDNA_ERR(xdna, "start npu failed, ret %d", ret);
593 		goto release_fw;
594 	}
595 
596 	xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
597 	for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
598 		xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
599 	xrs_cfg.sys_eff_factor = 1;
600 	xrs_cfg.ddev = &xdna->ddev;
601 	xrs_cfg.actions = &aie2_xrs_actions;
602 	xrs_cfg.total_col = ndev->total_col;
603 
604 	xdna->xrs_hdl = xrsm_init(&xrs_cfg);
605 	if (!xdna->xrs_hdl) {
606 		XDNA_ERR(xdna, "Initialize resolver failed");
607 		ret = -EINVAL;
608 		goto stop_hw;
609 	}
610 
611 	release_firmware(fw);
612 	aie2_msg_init(ndev);
613 	amdxdna_pm_init(xdna);
614 	return 0;
615 
616 stop_hw:
617 	aie2_hw_stop(xdna);
618 release_fw:
619 	release_firmware(fw);
620 
621 	return ret;
622 }
623 
624 static void aie2_fini(struct amdxdna_dev *xdna)
625 {
626 	amdxdna_pm_fini(xdna);
627 	aie2_hw_stop(xdna);
628 }
629 
630 static int aie2_get_aie_status(struct amdxdna_client *client,
631 			       struct amdxdna_drm_get_info *args)
632 {
633 	struct amdxdna_drm_query_aie_status status;
634 	struct amdxdna_dev *xdna = client->xdna;
635 	struct amdxdna_dev_hdl *ndev;
636 	int ret;
637 
638 	ndev = xdna->dev_handle;
639 	if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) {
640 		XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
641 		return -EFAULT;
642 	}
643 
644 	if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
645 		XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
646 			 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
647 		return -EINVAL;
648 	}
649 
650 	ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
651 				status.buffer_size, &status.cols_filled);
652 	if (ret) {
653 		XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret);
654 		return ret;
655 	}
656 
657 	if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) {
658 		XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
659 		return -EFAULT;
660 	}
661 
662 	return 0;
663 }
664 
665 static int aie2_get_aie_metadata(struct amdxdna_client *client,
666 				 struct amdxdna_drm_get_info *args)
667 {
668 	struct amdxdna_drm_query_aie_metadata *meta;
669 	struct amdxdna_dev *xdna = client->xdna;
670 	struct amdxdna_dev_hdl *ndev;
671 	int ret = 0;
672 
673 	ndev = xdna->dev_handle;
674 	meta = kzalloc_obj(*meta);
675 	if (!meta)
676 		return -ENOMEM;
677 
678 	meta->col_size = ndev->metadata.size;
679 	meta->cols = ndev->metadata.cols;
680 	meta->rows = ndev->metadata.rows;
681 
682 	meta->version.major = ndev->metadata.version.major;
683 	meta->version.minor = ndev->metadata.version.minor;
684 
685 	meta->core.row_count = ndev->metadata.core.row_count;
686 	meta->core.row_start = ndev->metadata.core.row_start;
687 	meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
688 	meta->core.lock_count = ndev->metadata.core.lock_count;
689 	meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
690 
691 	meta->mem.row_count = ndev->metadata.mem.row_count;
692 	meta->mem.row_start = ndev->metadata.mem.row_start;
693 	meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
694 	meta->mem.lock_count = ndev->metadata.mem.lock_count;
695 	meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
696 
697 	meta->shim.row_count = ndev->metadata.shim.row_count;
698 	meta->shim.row_start = ndev->metadata.shim.row_start;
699 	meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
700 	meta->shim.lock_count = ndev->metadata.shim.lock_count;
701 	meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
702 
703 	if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta)))
704 		ret = -EFAULT;
705 
706 	kfree(meta);
707 	return ret;
708 }
709 
710 static int aie2_get_aie_version(struct amdxdna_client *client,
711 				struct amdxdna_drm_get_info *args)
712 {
713 	struct amdxdna_drm_query_aie_version version;
714 	struct amdxdna_dev *xdna = client->xdna;
715 	struct amdxdna_dev_hdl *ndev;
716 
717 	ndev = xdna->dev_handle;
718 	version.major = ndev->version.major;
719 	version.minor = ndev->version.minor;
720 
721 	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
722 		return -EFAULT;
723 
724 	return 0;
725 }
726 
727 static int aie2_get_firmware_version(struct amdxdna_client *client,
728 				     struct amdxdna_drm_get_info *args)
729 {
730 	struct amdxdna_drm_query_firmware_version version;
731 	struct amdxdna_dev *xdna = client->xdna;
732 
733 	version.major = xdna->fw_ver.major;
734 	version.minor = xdna->fw_ver.minor;
735 	version.patch = xdna->fw_ver.sub;
736 	version.build = xdna->fw_ver.build;
737 
738 	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
739 		return -EFAULT;
740 
741 	return 0;
742 }
743 
744 static int aie2_get_power_mode(struct amdxdna_client *client,
745 			       struct amdxdna_drm_get_info *args)
746 {
747 	struct amdxdna_drm_get_power_mode mode = {};
748 	struct amdxdna_dev *xdna = client->xdna;
749 	struct amdxdna_dev_hdl *ndev;
750 
751 	ndev = xdna->dev_handle;
752 	mode.power_mode = ndev->pw_mode;
753 
754 	if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode)))
755 		return -EFAULT;
756 
757 	return 0;
758 }
759 
760 static int aie2_get_clock_metadata(struct amdxdna_client *client,
761 				   struct amdxdna_drm_get_info *args)
762 {
763 	struct amdxdna_drm_query_clock_metadata *clock;
764 	struct amdxdna_dev *xdna = client->xdna;
765 	struct amdxdna_dev_hdl *ndev;
766 	int ret = 0;
767 
768 	ndev = xdna->dev_handle;
769 	clock = kzalloc_obj(*clock);
770 	if (!clock)
771 		return -ENOMEM;
772 
773 	snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name),
774 		 "MP-NPU Clock");
775 	clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
776 	snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock");
777 	clock->h_clock.freq_mhz = ndev->hclk_freq;
778 
779 	if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock)))
780 		ret = -EFAULT;
781 
782 	kfree(clock);
783 	return ret;
784 }
785 
786 static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
787 {
788 	struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL;
789 	struct amdxdna_drm_get_array *array_args = arg;
790 	struct amdxdna_drm_hwctx_entry __user *buf;
791 	u32 size;
792 
793 	if (!array_args->num_element)
794 		return -EINVAL;
795 
796 	tmp = kzalloc_obj(*tmp);
797 	if (!tmp)
798 		return -ENOMEM;
799 
800 	tmp->pid = hwctx->client->pid;
801 	tmp->context_id = hwctx->id;
802 	tmp->start_col = hwctx->start_col;
803 	tmp->num_col = hwctx->num_col;
804 	tmp->command_submissions = hwctx->priv->seq;
805 	tmp->command_completions = hwctx->priv->completed;
806 	tmp->pasid = hwctx->client->pasid;
807 	tmp->priority = hwctx->qos.priority;
808 	tmp->gops = hwctx->qos.gops;
809 	tmp->fps = hwctx->qos.fps;
810 	tmp->dma_bandwidth = hwctx->qos.dma_bandwidth;
811 	tmp->latency = hwctx->qos.latency;
812 	tmp->frame_exec_time = hwctx->qos.frame_exec_time;
813 	tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE;
814 
815 	buf = u64_to_user_ptr(array_args->buffer);
816 	size = min(sizeof(*tmp), array_args->element_size);
817 
818 	if (copy_to_user(buf, tmp, size))
819 		return -EFAULT;
820 
821 	array_args->buffer += size;
822 	array_args->num_element--;
823 
824 	return 0;
825 }
826 
827 static int aie2_get_hwctx_status(struct amdxdna_client *client,
828 				 struct amdxdna_drm_get_info *args)
829 {
830 	struct amdxdna_drm_get_array array_args;
831 	struct amdxdna_dev *xdna = client->xdna;
832 	struct amdxdna_client *tmp_client;
833 	int ret;
834 
835 	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
836 
837 	array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx);
838 	array_args.buffer = args->buffer;
839 	array_args.num_element = args->buffer_size / array_args.element_size;
840 	list_for_each_entry(tmp_client, &xdna->client_list, node) {
841 		ret = amdxdna_hwctx_walk(tmp_client, &array_args,
842 					 aie2_hwctx_status_cb);
843 		if (ret)
844 			break;
845 	}
846 
847 	args->buffer_size -= (u32)(array_args.buffer - args->buffer);
848 	return 0;
849 }
850 
851 static int aie2_query_resource_info(struct amdxdna_client *client,
852 				    struct amdxdna_drm_get_info *args)
853 {
854 	struct amdxdna_drm_get_resource_info res_info;
855 	const struct amdxdna_dev_priv *priv;
856 	struct amdxdna_dev_hdl *ndev;
857 	struct amdxdna_dev *xdna;
858 
859 	xdna = client->xdna;
860 	ndev = xdna->dev_handle;
861 	priv = ndev->priv;
862 
863 	res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk;
864 	res_info.npu_tops_max = ndev->max_tops;
865 	res_info.npu_task_max = priv->hwctx_limit;
866 	res_info.npu_tops_curr = ndev->curr_tops;
867 	res_info.npu_task_curr = ndev->hwctx_num;
868 
869 	if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info)))
870 		return -EFAULT;
871 
872 	return 0;
873 }
874 
875 static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg)
876 {
877 	struct amdxdna_dev *xdna = hwctx->client->xdna;
878 	u32 *map = arg;
879 
880 	if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) {
881 		XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id,
882 			 xdna->dev_handle->priv->hwctx_limit);
883 		return -EINVAL;
884 	}
885 
886 	map[hwctx->fw_ctx_id] = hwctx->id;
887 	return 0;
888 }
889 
890 static int aie2_get_telemetry(struct amdxdna_client *client,
891 			      struct amdxdna_drm_get_info *args)
892 {
893 	struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL;
894 	u32 telemetry_data_sz, header_sz, elem_num;
895 	struct amdxdna_dev *xdna = client->xdna;
896 	struct amdxdna_client *tmp_client;
897 	int ret;
898 
899 	elem_num = xdna->dev_handle->priv->hwctx_limit;
900 	header_sz = struct_size(header, map, elem_num);
901 	if (args->buffer_size <= header_sz) {
902 		XDNA_ERR(xdna, "Invalid buffer size");
903 		return -EINVAL;
904 	}
905 
906 	telemetry_data_sz = args->buffer_size - header_sz;
907 	if (telemetry_data_sz > SZ_4M) {
908 		XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz);
909 		return -EINVAL;
910 	}
911 
912 	header = kzalloc(header_sz, GFP_KERNEL);
913 	if (!header)
914 		return -ENOMEM;
915 
916 	if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) {
917 		XDNA_ERR(xdna, "Failed to copy telemetry header from user");
918 		return -EFAULT;
919 	}
920 
921 	header->map_num_elements = elem_num;
922 	list_for_each_entry(tmp_client, &xdna->client_list, node) {
923 		ret = amdxdna_hwctx_walk(tmp_client, &header->map,
924 					 aie2_fill_hwctx_map);
925 		if (ret)
926 			return ret;
927 	}
928 
929 	ret = aie2_query_telemetry(xdna->dev_handle,
930 				   u64_to_user_ptr(args->buffer + header_sz),
931 				   telemetry_data_sz, header);
932 	if (ret) {
933 		XDNA_ERR(xdna, "Query telemetry failed ret %d", ret);
934 		return ret;
935 	}
936 
937 	if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) {
938 		XDNA_ERR(xdna, "Copy header failed");
939 		return -EFAULT;
940 	}
941 
942 	return 0;
943 }
944 
945 static int aie2_get_preempt_state(struct amdxdna_client *client,
946 				  struct amdxdna_drm_get_info *args)
947 {
948 	struct amdxdna_drm_attribute_state state = {};
949 	struct amdxdna_dev *xdna = client->xdna;
950 	struct amdxdna_dev_hdl *ndev;
951 
952 	ndev = xdna->dev_handle;
953 	if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE)
954 		state.state = ndev->force_preempt_enabled;
955 	else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE)
956 		state.state = ndev->frame_boundary_preempt;
957 
958 	if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state)))
959 		return -EFAULT;
960 
961 	return 0;
962 }
963 
964 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
965 {
966 	struct amdxdna_dev *xdna = client->xdna;
967 	int ret, idx;
968 
969 	if (!drm_dev_enter(&xdna->ddev, &idx))
970 		return -ENODEV;
971 
972 	ret = amdxdna_pm_resume_get_locked(xdna);
973 	if (ret)
974 		goto dev_exit;
975 
976 	switch (args->param) {
977 	case DRM_AMDXDNA_QUERY_AIE_STATUS:
978 		ret = aie2_get_aie_status(client, args);
979 		break;
980 	case DRM_AMDXDNA_QUERY_AIE_METADATA:
981 		ret = aie2_get_aie_metadata(client, args);
982 		break;
983 	case DRM_AMDXDNA_QUERY_AIE_VERSION:
984 		ret = aie2_get_aie_version(client, args);
985 		break;
986 	case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
987 		ret = aie2_get_clock_metadata(client, args);
988 		break;
989 	case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
990 		ret = aie2_get_hwctx_status(client, args);
991 		break;
992 	case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION:
993 		ret = aie2_get_firmware_version(client, args);
994 		break;
995 	case DRM_AMDXDNA_GET_POWER_MODE:
996 		ret = aie2_get_power_mode(client, args);
997 		break;
998 	case DRM_AMDXDNA_QUERY_TELEMETRY:
999 		ret = aie2_get_telemetry(client, args);
1000 		break;
1001 	case DRM_AMDXDNA_QUERY_RESOURCE_INFO:
1002 		ret = aie2_query_resource_info(client, args);
1003 		break;
1004 	case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE:
1005 	case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE:
1006 		ret = aie2_get_preempt_state(client, args);
1007 		break;
1008 	default:
1009 		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1010 		ret = -EOPNOTSUPP;
1011 	}
1012 
1013 	amdxdna_pm_suspend_put(xdna);
1014 	XDNA_DBG(xdna, "Got param %d", args->param);
1015 
1016 dev_exit:
1017 	drm_dev_exit(idx);
1018 	return ret;
1019 }
1020 
1021 static int aie2_query_ctx_status_array(struct amdxdna_client *client,
1022 				       struct amdxdna_drm_get_array *args)
1023 {
1024 	struct amdxdna_drm_get_array array_args;
1025 	struct amdxdna_dev *xdna = client->xdna;
1026 	struct amdxdna_client *tmp_client;
1027 	int ret;
1028 
1029 	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
1030 
1031 	if (args->element_size > SZ_4K || args->num_element > SZ_1K) {
1032 		XDNA_DBG(xdna, "Invalid element size %d or number of element %d",
1033 			 args->element_size, args->num_element);
1034 		return -EINVAL;
1035 	}
1036 
1037 	array_args.element_size = min(args->element_size,
1038 				      sizeof(struct amdxdna_drm_hwctx_entry));
1039 	array_args.buffer = args->buffer;
1040 	array_args.num_element = args->num_element * args->element_size /
1041 				array_args.element_size;
1042 	list_for_each_entry(tmp_client, &xdna->client_list, node) {
1043 		ret = amdxdna_hwctx_walk(tmp_client, &array_args,
1044 					 aie2_hwctx_status_cb);
1045 		if (ret)
1046 			break;
1047 	}
1048 
1049 	args->element_size = array_args.element_size;
1050 	args->num_element = (u32)((array_args.buffer - args->buffer) /
1051 				  args->element_size);
1052 
1053 	return 0;
1054 }
1055 
1056 static int aie2_get_array(struct amdxdna_client *client,
1057 			  struct amdxdna_drm_get_array *args)
1058 {
1059 	struct amdxdna_dev *xdna = client->xdna;
1060 	int ret, idx;
1061 
1062 	if (!drm_dev_enter(&xdna->ddev, &idx))
1063 		return -ENODEV;
1064 
1065 	ret = amdxdna_pm_resume_get_locked(xdna);
1066 	if (ret)
1067 		goto dev_exit;
1068 
1069 	switch (args->param) {
1070 	case DRM_AMDXDNA_HW_CONTEXT_ALL:
1071 		ret = aie2_query_ctx_status_array(client, args);
1072 		break;
1073 	case DRM_AMDXDNA_HW_LAST_ASYNC_ERR:
1074 		ret = aie2_get_array_async_error(xdna->dev_handle, args);
1075 		break;
1076 	default:
1077 		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1078 		ret = -EOPNOTSUPP;
1079 	}
1080 
1081 	amdxdna_pm_suspend_put(xdna);
1082 	XDNA_DBG(xdna, "Got param %d", args->param);
1083 
1084 dev_exit:
1085 	drm_dev_exit(idx);
1086 	return ret;
1087 }
1088 
1089 static int aie2_set_power_mode(struct amdxdna_client *client,
1090 			       struct amdxdna_drm_set_state *args)
1091 {
1092 	struct amdxdna_drm_set_power_mode power_state;
1093 	enum amdxdna_power_mode_type power_mode;
1094 	struct amdxdna_dev *xdna = client->xdna;
1095 
1096 	if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer),
1097 			   sizeof(power_state))) {
1098 		XDNA_ERR(xdna, "Failed to copy power mode request into kernel");
1099 		return -EFAULT;
1100 	}
1101 
1102 	if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad)))
1103 		return -EINVAL;
1104 
1105 	power_mode = power_state.power_mode;
1106 	if (power_mode > POWER_MODE_TURBO) {
1107 		XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
1108 		return -EINVAL;
1109 	}
1110 
1111 	return aie2_pm_set_mode(xdna->dev_handle, power_mode);
1112 }
1113 
1114 static int aie2_set_preempt_state(struct amdxdna_client *client,
1115 				  struct amdxdna_drm_set_state *args)
1116 {
1117 	struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
1118 	struct amdxdna_drm_attribute_state state;
1119 	u32 val;
1120 	int ret;
1121 
1122 	if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state)))
1123 		return -EFAULT;
1124 
1125 	if (state.state > 1)
1126 		return -EINVAL;
1127 
1128 	if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad)))
1129 		return -EINVAL;
1130 
1131 	if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) {
1132 		ndev->force_preempt_enabled = state.state;
1133 	} else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) {
1134 		val = state.state;
1135 		ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT,
1136 				       &val);
1137 		if (ret)
1138 			return ret;
1139 
1140 		ndev->frame_boundary_preempt = state.state;
1141 	}
1142 
1143 	return 0;
1144 }
1145 
1146 static int aie2_set_state(struct amdxdna_client *client,
1147 			  struct amdxdna_drm_set_state *args)
1148 {
1149 	struct amdxdna_dev *xdna = client->xdna;
1150 	int ret, idx;
1151 
1152 	if (!drm_dev_enter(&xdna->ddev, &idx))
1153 		return -ENODEV;
1154 
1155 	ret = amdxdna_pm_resume_get_locked(xdna);
1156 	if (ret)
1157 		goto dev_exit;
1158 
1159 	switch (args->param) {
1160 	case DRM_AMDXDNA_SET_POWER_MODE:
1161 		ret = aie2_set_power_mode(client, args);
1162 		break;
1163 	case DRM_AMDXDNA_SET_FORCE_PREEMPT:
1164 	case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT:
1165 		ret = aie2_set_preempt_state(client, args);
1166 		break;
1167 	default:
1168 		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1169 		ret = -EOPNOTSUPP;
1170 		break;
1171 	}
1172 
1173 	amdxdna_pm_suspend_put(xdna);
1174 dev_exit:
1175 	drm_dev_exit(idx);
1176 	return ret;
1177 }
1178 
1179 const struct amdxdna_dev_ops aie2_ops = {
1180 	.init = aie2_init,
1181 	.fini = aie2_fini,
1182 	.resume = aie2_hw_resume,
1183 	.suspend = aie2_hw_suspend,
1184 	.get_aie_info = aie2_get_info,
1185 	.set_aie_state = aie2_set_state,
1186 	.hwctx_init = aie2_hwctx_init,
1187 	.hwctx_fini = aie2_hwctx_fini,
1188 	.hwctx_config = aie2_hwctx_config,
1189 	.hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo,
1190 	.cmd_submit = aie2_cmd_submit,
1191 	.hmm_invalidate = aie2_hmm_invalidate,
1192 	.get_array = aie2_get_array,
1193 };
1194