xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2026 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/xarray.h>
26 #include <drm/drm_mm.h>
27 #include <linux/random.h>
28 #include "amdgpu.h"
29 #include "amdgpu_sdma.h"
30 #include "amdgpu_ualink.h"
31 #include "amdgpu_xgmi.h"
32 #include "amdgpu_dma_buf.h"
33 #include "psp_gfx_if.h"
34 #include <linux/math.h>
35 #include <linux/sysfs.h>
36 #include <linux/string.h>
37 
38 static void deactivate_accelerator(struct amdgpu_device *adev);
39 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
40 				u32 remote_accel_id, u32 dw0, u32 dw1,
41 				u32 dw2, u32 dw3);
42 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev,
43 				    u32 flush_type);
44 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
45 						struct amdgpu_bo *bos[], u32 n_bos,
46 						struct drm_exec *exec,
47 						bool interruptible);
48 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
49 						   struct drm_exec *exec);
50 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
51 						  u32 remote_accel_id, u32 state,
52 						  u32 generation_count);
53 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo);
54 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
55 					  u32 remote_accel_id, u64 addr,
56 					  u32 size_in_pages, u32 flush_type);
57 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev);
58 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev);
59 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
60 					   u32 remote_accel_id);
61 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
62 					  u32 remote_accel_id);
63 
64 #define STRIP_NPA(addr)						\
65 	(((u64)(addr) & ~AMDGPU_UALINK_NPA_ADDR_GPUID_MASK))
66 
67 #define GENERATE_NPA(addr, remote_acc_id)			\
68 		((u64)(((u64)(addr)) |				\
69 		 ((u64)(remote_acc_id) << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT)))
70 
71 static const struct drm_client_funcs ualink_client_funcs = {
72 	.unregister	= drm_client_release,
73 };
74 
75 static int amdgpu_ualink_drm_client_create(struct amdgpu_device *adev)
76 {
77 	int ret;
78 
79 	ret = drm_client_init(&adev->ddev, &adev->ualink.client, "ualink",
80 			      &ualink_client_funcs);
81 	if (ret) {
82 		dev_err(adev->dev, "Failed to init UALink DRM client: %d\n",
83 			ret);
84 		return ret;
85 	}
86 
87 	drm_client_register(&adev->ualink.client);
88 
89 	return 0;
90 }
91 
92 static void amdgpu_ualink_object_fini(struct amdgpu_device *adev)
93 {
94 	if (!adev->ualink.info)
95 		return;
96 
97 	kobject_put(&adev->ualink.stations->kobj);
98 	kobject_put(&adev->ualink.config->kobj);
99 	kobject_put(&adev->ualink.setup->kobj);
100 	kobject_put(&adev->ualink.info->kobj);
101 	adev->ualink.stations = NULL;
102 	adev->ualink.config = NULL;
103 	adev->ualink.setup = NULL;
104 	adev->ualink.info = NULL;
105 }
106 
107 static bool __check_ppod_info(struct amdgpu_device *adev,
108 			      const struct amdgpu_ualink_info *info)
109 {
110 	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
111 
112 	if (ppod->size <= 0 || ppod->size > AMDGPU_UALINK_ACCEL_MAX) {
113 		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
114 			ppod->size, AMDGPU_UALINK_ACCEL_MAX);
115 		return false;
116 	}
117 	if (ppod->accel_id >= ppod->size) {
118 		dev_dbg(adev->dev,
119 			"Accelerator ID %u greater or equal pPod size %u\n",
120 			ppod->accel_id, ppod->size);
121 		return false;
122 	}
123 
124 	return true;
125 }
126 
127 static bool __check_vpod_info(struct amdgpu_device *adev,
128 			      const struct amdgpu_ualink_info *info)
129 {
130 	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
131 	const struct amdgpu_ualink_vpod_info *vpod = &info->vpod;
132 	unsigned int weight;
133 
134 	if (vpod->size == 0 || vpod->size > ppod->size) {
135 		dev_warn(adev->dev,
136 			 "UALINK: vPod size %u out of range [1..%u]\n",
137 			 vpod->size, ppod->size);
138 		return false;
139 	}
140 	if (vpod->addr_mode >= AMDGPU_UALINK_ADDR_MODE_MAX) {
141 		dev_warn(adev->dev,
142 			 "UALINK: invalid addr mode %u\n",
143 			 vpod->addr_mode);
144 		return false;
145 	}
146 	weight =
147 		bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
148 	if (weight != vpod->size) {
149 		dev_warn(adev->dev,
150 			 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n",
151 			 vpod->size, weight);
152 		return false;
153 	}
154 	if (!test_bit(ppod->accel_id, vpod->active_accel_bits)) {
155 		dev_warn(adev->dev,
156 			 "UALINK: accelerator ID %u not listed in vpod_active_accels\n",
157 			 ppod->accel_id);
158 		return false;
159 	}
160 
161 	return true;
162 }
163 
164 static void
165 amdgpu_ualink_info_set_accel_state(struct amdgpu_device *adev,
166 				   struct amdgpu_ualink_info *info,
167 				   enum psp_gfx_ual_config_state cfg_state)
168 {
169 	enum amdgpu_ualink_accel_state cur;
170 	enum amdgpu_ualink_accel_state target;
171 	bool ppod_validated;
172 	bool vpod_validated;
173 
174 	if (!info)
175 		return;
176 
177 	cur = info->accel_state;
178 
179 	ppod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED &&
180 			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
181 	vpod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED &&
182 			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
183 
184 	switch (cfg_state) {
185 	case UAL_CFG_IDLE:
186 		return;
187 	case UAL_CFG_PPOD:
188 		target = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
189 		break;
190 	case UAL_CFG_VPOD:
191 	case UAL_CFG_STATION:
192 		target = AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
193 		break;
194 	case UAL_CFG_COMPLETE:
195 		target = AMDGPU_UALINK_ACCEL_STATE_READY;
196 		break;
197 	default:
198 		dev_dbg(adev->dev, "invalid configuration state %u", cfg_state);
199 		return;
200 	}
201 
202 	/* ppod stage: should be part of a ppod first */
203 	if (!ppod_validated && !__check_ppod_info(adev, info)) {
204 		if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
205 			info->accel_state =
206 				AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
207 		} else {
208 			info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
209 			dev_err(adev->dev,
210 				"ppod configuration is invalid, setting to error state");
211 		}
212 		return;
213 	}
214 	if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
215 		if (!ppod_validated)
216 			info->accel_state =
217 				AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
218 		return;
219 	}
220 
221 	/* A vpod_id of 0 means the GPU is not part of any vPod */
222 	if (info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
223 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
224 		return;
225 	}
226 
227 	/* vpod stage: required to reach vpod_configured or ready */
228 	if (!vpod_validated && !__check_vpod_info(adev, info)) {
229 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
230 		dev_err(adev->dev,
231 			"vpod configuration is invalid, setting to error state");
232 		return;
233 	}
234 	if (target == AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
235 		if (!vpod_validated)
236 			info->accel_state =
237 				AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
238 		return;
239 	}
240 
241 	/* complete stage: advance to ready unless already ready/active */
242 	if (cur < AMDGPU_UALINK_ACCEL_STATE_READY ||
243 	    cur > AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
244 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_READY;
245 }
246 
247 static int amdgpu_ualink_update_vpod_config(struct amdgpu_device *adev)
248 {
249 	/* TBD: Do updates/cleanup based on updated vpod configuration */
250 	return 0;
251 }
252 
253 static int amdgpu_ualink_update_accel_state(
254 	struct amdgpu_device *adev, enum amdgpu_ualink_accel_state prev_state,
255 	u32 prev_vpod_id, enum psp_gfx_ual_config_state cfg_state)
256 {
257 	dev_info(adev->dev,
258 		 "UALINK: update_accel_state: prev_state=%d prev_vpod_id=%u new_vpod_id=%u cfg_state=%d\n",
259 		 prev_state, prev_vpod_id, adev->ualink.info->vpod.id,
260 		 cfg_state);
261 
262 	/* If the device is already active and its vpod_id is unchanged, the
263 	 * update does not affect vpod membership. Skip the local vpod
264 	 * integrity check and re-activation.
265 	 */
266 	if (prev_state == AMDGPU_UALINK_ACCEL_STATE_ACTIVE &&
267 	    adev->ualink.info->vpod.id == prev_vpod_id) {
268 		amdgpu_ualink_update_vpod_config(adev);
269 
270 		/* Same vpod_id, but the member set may have changed (vPod grown
271 		 * or shrunk while this GPU stayed ACTIVE). If so, run the local
272 		 * vpod activation path: once every local peer has committed the
273 		 * new config (integrity passes), it bounces the affected ACTIVE
274 		 * peers to rebuild links/GART for the new member set.
275 		 */
276 		if (amdgpu_ualink_vpod_membership_changed(adev)) {
277 			dev_info(adev->dev,
278 				 "UALINK: update_accel_state: ACTIVE vpod_id=%u membership changed, reconfiguring\n",
279 				 adev->ualink.info->vpod.id);
280 			scoped_guard(mutex, &mgpu_info.mutex)
281 				__amdgpu_ualink_activate_vpod_locked(adev);
282 		} else {
283 			dev_info(adev->dev,
284 				 "UALINK: update_accel_state: already ACTIVE, vpod_id/membership unchanged\n");
285 		}
286 		return 0;
287 	}
288 
289 	/* A new vpod_id of 0 means this GPU was removed from the vPod. */
290 	if (adev->ualink.info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
291 		dev_info(adev->dev,
292 			 "UALINK: update_accel_state: vpod_id=0, removing accelerator from vPod\n");
293 		amdgpu_ualink_update_vpod_config(adev);
294 		scoped_guard(mutex, &mgpu_info.mutex)
295 			deactivate_accelerator(adev);
296 		return 0;
297 	}
298 
299 	/* GPU joining a new vpod should be with invalid vpod id*/
300 	scoped_guard(mutex, &mgpu_info.mutex) {
301 		if (prev_vpod_id == AMDGPU_UALINK_VPOD_ID_INVALID) {
302 			dev_info(adev->dev,
303 				 "UALINK: update_accel_state: joining vpod_id=%u\n",
304 				 adev->ualink.info->vpod.id);
305 			amdgpu_ualink_info_set_accel_state(
306 				adev, adev->ualink.info, cfg_state);
307 			__amdgpu_ualink_activate_vpod_locked(adev);
308 		} else {
309 			/* GPU should first get removal which will set invalid vpod_id
310 			* and then join a new vpod
311 			*/
312 			dev_err(adev->dev,
313 				"Invalid vpod transition from %u to %u\n",
314 				prev_vpod_id, adev->ualink.info->vpod.id);
315 			return -EINVAL;
316 		}
317 	}
318 
319 	return 0;
320 }
321 
322 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev)
323 {
324 	enum amdgpu_ualink_accel_state prev_state;
325 	enum psp_gfx_ual_config_state cfg_state;
326 	u32 prev_vpod_id;
327 	int r, qerr;
328 
329 	/* TBD: Stop ASP interrupts if driver faced an issue */
330 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
331 		u32 status;
332 
333 		dev_dbg(adev->dev,
334 			"UALink not initialized, skipping config update\n");
335 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
336 		return psp_ual_send_completion(
337 			&adev->psp, adev->ualink.psp_if_ver,
338 			PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, status);
339 	}
340 
341 	prev_state = adev->ualink.info->accel_state;
342 	prev_vpod_id = adev->ualink.info->vpod.id;
343 
344 	qerr = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
345 				  adev->ualink.info, &cfg_state);
346 
347 	/*TBD: find the right value of status to be sent to ASP*/
348 	r = psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
349 				    PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, 0);
350 	if (r || qerr)
351 		goto err;
352 
353 	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
354 					     cfg_state);
355 	if (r)
356 		goto err;
357 
358 	return 0;
359 
360 err:
361 	scoped_guard(mutex, &mgpu_info.mutex) {
362 		deactivate_accelerator(adev);
363 		adev->ualink.info->accel_state =
364 			AMDGPU_UALINK_ACCEL_STATE_ERROR;
365 	}
366 	dev_err(adev->dev,
367 		"UALink config update failed, setting to error state");
368 
369 	return r;
370 }
371 
372 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev)
373 {
374 	u32 status = 0;
375 
376 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
377 		dev_dbg(adev->dev,
378 			"UALink not initialized, skipping pause update\n");
379 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
380 		goto out;
381 	}
382 
383 	dev_dbg(adev->dev, "UALink pause command is not handled\n");
384 
385 out:
386 	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
387 				       PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID,
388 				       status);
389 }
390 
391 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev)
392 {
393 	u32 status = 0;
394 
395 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
396 		dev_dbg(adev->dev,
397 			"UALink not initialized, skipping pause update\n");
398 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
399 		goto out;
400 	}
401 
402 	dev_dbg(adev->dev, "UALink resume command is not handled\n");
403 
404 out:
405 	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
406 				       PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID,
407 				       status);
408 }
409 
410 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block)
411 {
412 	struct amdgpu_device *adev = ip_block->adev;
413 	int r;
414 
415 	if (!adev->ualink.info)
416 		return 0;
417 
418 	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
419 	r = psp_ual_get_interface_version(&adev->psp, &adev->ualink.psp_if_ver);
420 	if (r) {
421 		adev->ualink.psp_if_ver = 0xffffffff;
422 		goto disable;
423 	}
424 	dev_info(adev->dev, "Found UALink interface version 0x%x\n",
425 		 adev->ualink.psp_if_ver);
426 
427 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_HW;
428 
429 	return 0;
430 disable:
431 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
432 	return 0;
433 }
434 
435 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block)
436 {
437 	struct amdgpu_device *adev = ip_block->adev;
438 
439 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
440 		return 0;
441 
442 	scoped_guard(mutex, &mgpu_info.mutex)
443 		deactivate_accelerator(adev);
444 
445 	return 0;
446 }
447 
448 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block)
449 {
450 	struct amdgpu_device *adev = ip_block->adev;
451 	enum psp_gfx_ual_config_state cfg_state;
452 	int r;
453 
454 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_HW)
455 		return 0;
456 
457 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
458 			       adev->ualink.info, &cfg_state);
459 	if (r)
460 		return r;
461 
462 	scoped_guard(mutex, &mgpu_info.mutex) {
463 		amdgpu_ualink_info_set_accel_state(adev, adev->ualink.info,
464 						   cfg_state);
465 		__amdgpu_ualink_activate_vpod_locked(adev);
466 	}
467 
468 	r = amdgpu_ualink_drm_client_create(adev);
469 	if (r) {
470 		dev_err(adev->dev, "Failed to create UALink DRM client: %d\n",
471 			r);
472 		goto error;
473 	}
474 
475 	/* Consider UALink initialized only at this stage */
476 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_COMPLETE;
477 
478 	return 0;
479 
480 error:
481 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
482 	return r;
483 }
484 
485 /****************************************************************************
486  * UALink info and configuration in sysfs
487  *
488  * Using kobj_attribute and not device_attribute here because the UALink
489  * attributes are part of their own nested kobjects and not the amdgpu
490  * device itself or a subdevice.
491  ****************************************************************************/
492 
493 /* UALink configuration APIs are being deprecated before upstreaming and are
494  * not enabled by default. They are replaced by a unified in-band/side-band
495  * approach through the IFoE driver. Querying of the current configuration
496  * will continue to be supported in sysfs.
497  *
498  * Uncomment the following line to enable the deprecated configuration APIs
499  * for testing purposes only. The code to support these APIs will be removed
500  * in a future patch.
501  */
502 /* #define UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS */
503 
504 #define UALINK_VALUE_SHOW(prefix, name, field, format)			\
505 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
506 					       struct kobj_attribute *attr,\
507 					       char *buf)		\
508 {									\
509 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
510 									\
511 	return sysfs_emit(buf, format"\n", info->field);		\
512 }
513 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
514 #define UALINK_VALUE_STORE(prefix, name, field, type, base)		\
515 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
516 						struct kobj_attribute *attr,\
517 						const char *buf, size_t count)\
518 {									\
519 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
520 	int r;								\
521 									\
522 	r = kstrto##type(buf, base, &info->field);			\
523 	if (r < 0)							\
524 		return r;						\
525 	return count;							\
526 }
527 #endif
528 
529 static ssize_t show_idbits(unsigned long *bits, unsigned int nbits, char *buf)
530 {
531 	ssize_t len = 0;
532 	unsigned int id;
533 
534 	for_each_set_bit(id, bits, nbits)
535 		len += sysfs_emit_at(buf, len, "%u ", id);
536 	if (len)
537 		len--;
538 	return len + sysfs_emit_at(buf, len, "\n");
539 }
540 #define UALINK_IDBITS_SHOW(prefix, name, field)				\
541 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
542 					       struct kobj_attribute *attr,\
543 					       char *buf)		\
544 {									\
545 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
546 									\
547 	return show_idbits(info->field, sizeof(info->field)*8, buf);	\
548 }
549 
550 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
551 static ssize_t store_idbits(unsigned long *bits, unsigned int nbits,
552 			    const char *buf, size_t count)
553 {
554 	char *dup_buf, *str, *tok;
555 	ssize_t r;
556 	u32 id;
557 
558 	bitmap_zero(bits, nbits);
559 	if (!count)
560 		return 0;
561 
562 	dup_buf = kstrndup(buf, count, GFP_KERNEL);
563 	if (unlikely(!dup_buf))
564 		return -ENOMEM;
565 	str = dup_buf;
566 	do {
567 		str += strspn(str, " ");
568 		tok = strsep(&str, ", ");
569 		r = kstrtou32(tok, 10, &id);
570 		if (r < 0)
571 			goto err;
572 		if (id >= nbits) {
573 			r = -ERANGE;
574 			goto err;
575 		}
576 		set_bit(id, bits);
577 	} while (str);
578 	kfree(dup_buf);
579 
580 	return count;
581 
582 err:
583 	kfree(dup_buf);
584 	return r;
585 }
586 #define UALINK_IDBITS_STORE(prefix, name, field, nbits)			\
587 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
588 						struct kobj_attribute *attr,\
589 						const char *buf, size_t count)\
590 {									\
591 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
592 	DECLARE_BITMAP(bits, nbits);					\
593 	ssize_t r = store_idbits(bits, nbits, buf, count);		\
594 									\
595 	if (r >= 0)							\
596 		bitmap_copy(info->field, bits, nbits);			\
597 	return r;							\
598 }
599 #endif
600 
601 static ssize_t show_idarray(const u32 *array, u32 size, char *buf)
602 {
603 	ssize_t len = 0;
604 	unsigned int i;
605 
606 	for (i = 0; i < size; i++)
607 		len += sysfs_emit_at(buf, len, "%u ", array[i]);
608 	if (len)
609 		len--;
610 	return len + sysfs_emit_at(buf, len, "\n");
611 }
612 #define UALINK_IDARRAY_SHOW(prefix, name, field, size)			\
613 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
614 					       struct kobj_attribute *attr,\
615 					       char *buf)		\
616 {									\
617 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
618 									\
619 	return show_idarray(info->field, info->size, buf);		\
620 }
621 
622 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
623 static ssize_t store_idarray(u32 *array, u32 *size, u32 max_id, u32 max_size,
624 			     const char *buf, size_t count)
625 {
626 	char *dup_buf, *str, *tok;
627 	unsigned int index = 0;
628 	ssize_t r;
629 	u32 id;
630 
631 	*size = 0;
632 	if (!count)
633 		return 0;
634 
635 	dup_buf = kstrndup(buf, count, GFP_KERNEL);
636 	if (unlikely(!dup_buf))
637 		return -ENOMEM;
638 	str = dup_buf;
639 	do {
640 		str += strspn(str, " ");
641 		tok = strsep(&str, ", ");
642 		r = kstrtou32(tok, 10, &id);
643 		if (r < 0)
644 			goto err;
645 		if (index >= max_size) {
646 			r = -ENOSPC;
647 			goto err;
648 		}
649 		if (id >= max_id) {
650 			r = -ERANGE;
651 			goto err;
652 		}
653 		array[index++] = id;
654 	} while (str);
655 	kfree(dup_buf);
656 
657 	*size = index;
658 	return count;
659 
660 err:
661 	kfree(dup_buf);
662 	return r;
663 }
664 #define UALINK_IDARRAY_STORE(prefix, name, field, size, max_id)		\
665 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
666 						struct kobj_attribute *attr,\
667 						const char *buf, size_t count)\
668 {									\
669 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
670 	u32 new_array[ARRAY_SIZE(info->field)];				\
671 	u32 new_size;							\
672 	ssize_t r = store_idarray(new_array, &new_size, max_id,		\
673 				  ARRAY_SIZE(new_array), buf, count);	\
674 									\
675 	if (r >= 0) {							\
676 		memcpy(info->field, new_array, sizeof(new_array));	\
677 		info->size = new_size;					\
678 	}								\
679 	return r;							\
680 }
681 #endif
682 
683 #define UALINK_UUID_SHOW(prefix, name, field)				\
684 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
685 					       struct kobj_attribute *attr,\
686 					       char *buf)		\
687 {									\
688 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
689 									\
690 	return sysfs_emit(buf, "%pU\n", &info->field);			\
691 }
692 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
693 #define UALINK_UUID_STORE(prefix, name, field)				\
694 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
695 						struct kobj_attribute *attr,\
696 						const char *buf, size_t count)\
697 {									\
698 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
699 	int r;								\
700 									\
701 	r = uuid_parse(buf, &info->field);				\
702 	return r ? r : count;						\
703 }
704 #endif
705 
706 static ssize_t show_enum(u32 x, const char * const values[], unsigned int n,
707 			 char *buf)
708 {
709 	return sysfs_emit(buf, "%s\n", x < n && values[x] ? values[x] : "invalid");
710 }
711 #define UALINK_ENUM_SHOW(prefix, name, field)				\
712 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
713 					       struct kobj_attribute *attr,\
714 					       char *buf)		\
715 {									\
716 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
717 									\
718 	return show_enum(info->field, ualink_##name##_values,		\
719 			 ARRAY_SIZE(ualink_##name##_values), buf);	\
720 }
721 
722 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
723 static ssize_t store_enum(u32 *x, const char * const values[], unsigned int n,
724 			  const char *buf, size_t count)
725 {
726 	unsigned int i;
727 
728 	for (i = 0; i < n; i++) {
729 		if (values[i] && sysfs_streq(buf, values[i])) {
730 			*x = i;
731 			return count;
732 		}
733 	}
734 	return -EINVAL;
735 }
736 #define UALINK_ENUM_STORE(prefix, name, field)				\
737 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
738 						struct kobj_attribute *attr,\
739 						const char *buf, size_t count)\
740 {									\
741 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
742 									\
743 	return store_enum(&info->field, ualink_##name##_values,		\
744 			 ARRAY_SIZE(ualink_##name##_values),		\
745 			 buf, count);					\
746 }
747 #endif
748 static const char * const ualink_link_type_values[] = {
749 	"UALoE", "UALink"
750 };
751 static const char * const ualink_addr_mode_values[] = {
752 	"source-aliasing", "source-identification"
753 };
754 static const char * const ualink_accel_state_values[] = {
755 	[AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED]	= "unconfigured",
756 	[AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED]	= "ppod_configured",
757 	[AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED]	= "vpod_configured",
758 	[AMDGPU_UALINK_ACCEL_STATE_READY]		= "ready",
759 	[AMDGPU_UALINK_ACCEL_STATE_ACTIVE]		= "active",
760 	[AMDGPU_UALINK_ACCEL_STATE_ERROR]		= "error",
761 };
762 static_assert(ARRAY_SIZE(ualink_accel_state_values) ==
763 	      AMDGPU_UALINK_ACCEL_STATE_MAX);
764 
765 UALINK_ENUM_SHOW(info, link_type, link_type);
766 UALINK_VALUE_SHOW(info, accel_id,  ppod.accel_id,  "%u");
767 UALINK_VALUE_SHOW(info, bandwidth, ppod.bandwidth, "%u");
768 UALINK_VALUE_SHOW(info, latency,   ppod.latency,   "%u");
769 UALINK_UUID_SHOW(info, ppod_id, ppod.id);
770 UALINK_VALUE_SHOW(info, ppod_size, ppod.size,      "%u");
771 UALINK_VALUE_SHOW(info, vpod_id,   vpod.id,        "%u");
772 UALINK_VALUE_SHOW(info, vpod_size, vpod.size,      "%u");
773 UALINK_IDBITS_SHOW(info, vpod_active_accels, vpod.active_accel_bits);
774 UALINK_ENUM_SHOW(info, addr_mode, vpod.addr_mode);
775 UALINK_ENUM_SHOW(info, accel_state, accel_state);
776 UALINK_IDARRAY_SHOW(info, local_accels, local_accels, n_local_accels);
777 
778 static struct amdgpu_device *ualink_xcp_kobj_to_adev(struct kobject *kobj)
779 {
780 	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
781 					      ualink.kobj);
782 
783 	return xcp->xcp_mgr->adev;
784 }
785 
786 #define UALINK_XCP_INFO_SHOW(name)					\
787 static ssize_t ualink_xcp_info_##name##_show(struct kobject *kobj,	\
788 		struct kobj_attribute *attr, char *buf)			\
789 {									\
790 	struct amdgpu_device *adev = ualink_xcp_kobj_to_adev(kobj);	\
791 									\
792 	return ualink_info_##name##_show(&adev->ualink.info->kobj,	\
793 					 attr, buf);			\
794 }
795 
796 UALINK_XCP_INFO_SHOW(link_type)
797 UALINK_XCP_INFO_SHOW(accel_id)
798 UALINK_XCP_INFO_SHOW(bandwidth)
799 UALINK_XCP_INFO_SHOW(latency)
800 UALINK_XCP_INFO_SHOW(ppod_id)
801 UALINK_XCP_INFO_SHOW(ppod_size)
802 UALINK_XCP_INFO_SHOW(vpod_id)
803 UALINK_XCP_INFO_SHOW(vpod_size)
804 UALINK_XCP_INFO_SHOW(vpod_active_accels)
805 UALINK_XCP_INFO_SHOW(addr_mode)
806 UALINK_XCP_INFO_SHOW(accel_state)
807 UALINK_XCP_INFO_SHOW(local_accels)
808 
809 #define UALINK_INFO_ATTR(name) __ATTR(name, 0444, ualink_info_##name##_show, NULL)
810 static struct kobj_attribute ualink_info_link_type = UALINK_INFO_ATTR(link_type);
811 static struct kobj_attribute ualink_info_accel_id  = UALINK_INFO_ATTR(accel_id);
812 static struct kobj_attribute ualink_info_bandwidth = UALINK_INFO_ATTR(bandwidth);
813 static struct kobj_attribute ualink_info_latency   = UALINK_INFO_ATTR(latency);
814 static struct kobj_attribute ualink_info_ppod_id   = UALINK_INFO_ATTR(ppod_id);
815 static struct kobj_attribute ualink_info_ppod_size = UALINK_INFO_ATTR(ppod_size);
816 static struct kobj_attribute ualink_info_vpod_id   = UALINK_INFO_ATTR(vpod_id);
817 static struct kobj_attribute ualink_info_vpod_size = UALINK_INFO_ATTR(vpod_size);
818 static struct kobj_attribute ualink_info_vpod_active_accels = UALINK_INFO_ATTR(vpod_active_accels);
819 static struct kobj_attribute ualink_info_addr_mode = UALINK_INFO_ATTR(addr_mode);
820 static struct kobj_attribute ualink_info_accel_state = UALINK_INFO_ATTR(accel_state);
821 static struct kobj_attribute ualink_info_local_accels = UALINK_INFO_ATTR(local_accels);
822 
823 static const struct attribute *ualink_info_attrs[] = {
824 	&ualink_info_link_type.attr,
825 	&ualink_info_accel_id.attr,
826 	&ualink_info_bandwidth.attr,
827 	&ualink_info_latency.attr,
828 	&ualink_info_ppod_id.attr,
829 	&ualink_info_ppod_size.attr,
830 	&ualink_info_vpod_id.attr,
831 	&ualink_info_vpod_size.attr,
832 	&ualink_info_vpod_active_accels.attr,
833 	&ualink_info_addr_mode.attr,
834 	&ualink_info_accel_state.attr,
835 	&ualink_info_local_accels.attr,
836 	NULL
837 };
838 
839 #define UALINK_XCP_INFO_ATTR(name) \
840 	__ATTR(name, 0444, ualink_xcp_info_##name##_show, NULL)
841 static struct kobj_attribute ualink_xcp_info_link_type = UALINK_XCP_INFO_ATTR(link_type);
842 static struct kobj_attribute ualink_xcp_info_accel_id  = UALINK_XCP_INFO_ATTR(accel_id);
843 static struct kobj_attribute ualink_xcp_info_bandwidth = UALINK_XCP_INFO_ATTR(bandwidth);
844 static struct kobj_attribute ualink_xcp_info_latency   = UALINK_XCP_INFO_ATTR(latency);
845 static struct kobj_attribute ualink_xcp_info_ppod_id   = UALINK_XCP_INFO_ATTR(ppod_id);
846 static struct kobj_attribute ualink_xcp_info_ppod_size = UALINK_XCP_INFO_ATTR(ppod_size);
847 static struct kobj_attribute ualink_xcp_info_vpod_id   = UALINK_XCP_INFO_ATTR(vpod_id);
848 static struct kobj_attribute ualink_xcp_info_vpod_size = UALINK_XCP_INFO_ATTR(vpod_size);
849 static struct kobj_attribute ualink_xcp_info_vpod_active_accels = UALINK_XCP_INFO_ATTR(vpod_active_accels);
850 static struct kobj_attribute ualink_xcp_info_addr_mode = UALINK_XCP_INFO_ATTR(addr_mode);
851 static struct kobj_attribute ualink_xcp_info_accel_state = UALINK_XCP_INFO_ATTR(accel_state);
852 static struct kobj_attribute ualink_xcp_info_local_accels = UALINK_XCP_INFO_ATTR(local_accels);
853 
854 static struct attribute *ualink_xcp_info_attrs[] = {
855 	&ualink_xcp_info_link_type.attr,
856 	&ualink_xcp_info_accel_id.attr,
857 	&ualink_xcp_info_bandwidth.attr,
858 	&ualink_xcp_info_latency.attr,
859 	&ualink_xcp_info_ppod_id.attr,
860 	&ualink_xcp_info_ppod_size.attr,
861 	&ualink_xcp_info_vpod_id.attr,
862 	&ualink_xcp_info_vpod_size.attr,
863 	&ualink_xcp_info_vpod_active_accels.attr,
864 	&ualink_xcp_info_addr_mode.attr,
865 	&ualink_xcp_info_accel_state.attr,
866 	&ualink_xcp_info_local_accels.attr,
867 	NULL
868 };
869 
870 static umode_t ualink_xcp_info_is_visible(struct kobject *kobj,
871 					  struct attribute *attr, int n)
872 {
873 	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
874 					      ualink.kobj);
875 
876 	if (!xcp->valid)
877 		return 0;
878 
879 	return attr->mode;
880 }
881 
882 static const struct attribute_group ualink_xcp_info_group = {
883 	.attrs = ualink_xcp_info_attrs,
884 	.is_visible = ualink_xcp_info_is_visible,
885 };
886 
887 static const struct kobj_type ualink_xcp_info_ktype = {
888 	.sysfs_ops = &kobj_sysfs_ops
889 };
890 
891 static void ualink_info_release(struct kobject *kobj)
892 {
893 	kfree(to_ualink_info(kobj));
894 }
895 
896 static const struct kobj_type ualink_info_ktype = {
897 	.release = ualink_info_release,
898 	.sysfs_ops = &kobj_sysfs_ops
899 };
900 
901 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
902 UALINK_VALUE_SHOW(ppod_setup, accel_id,  ppod.accel_id,  "%u");
903 UALINK_VALUE_SHOW(ppod_setup, bandwidth, ppod.bandwidth, "%u");
904 UALINK_VALUE_SHOW(ppod_setup, latency,   ppod.latency,   "%u");
905 UALINK_UUID_SHOW(ppod_setup, ppod_id, ppod.id);
906 UALINK_VALUE_SHOW(ppod_setup, ppod_size, ppod.size,      "%u");
907 UALINK_IDARRAY_SHOW(ppod_setup, local_accels, local_accels, n_local_accels);
908 
909 UALINK_VALUE_STORE(ppod_setup, accel_id,  ppod.accel_id,  u32, 10);
910 UALINK_VALUE_STORE(ppod_setup, bandwidth, ppod.bandwidth, u32, 10);
911 UALINK_VALUE_STORE(ppod_setup, latency,   ppod.latency,   u32, 10);
912 UALINK_UUID_STORE(ppod_setup, ppod_id,   ppod.id);
913 UALINK_VALUE_STORE(ppod_setup, ppod_size, ppod.size,      u32, 10);
914 UALINK_IDARRAY_STORE(ppod_setup, local_accels, local_accels, n_local_accels,
915 		     AMDGPU_UALINK_ACCEL_MAX);
916 
917 static enum amdgpu_ualink_accel_state
918 check_ppod_state(struct amdgpu_device *adev,
919 		 const struct amdgpu_ualink_ppod_setup *setup)
920 {
921 	if (setup->ppod.size <= 0 || setup->ppod.size > AMDGPU_UALINK_ACCEL_MAX) {
922 		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
923 			setup->ppod.size, AMDGPU_UALINK_ACCEL_MAX);
924 		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
925 	}
926 	if (setup->ppod.accel_id >= setup->ppod.size) {
927 		dev_dbg(adev->dev, "Accelerator ID %u greater or equal pPod size %u\n",
928 			setup->ppod.accel_id, setup->ppod.size);
929 		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
930 	}
931 	return AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
932 }
933 
934 static ssize_t ualink_ppod_setup_commit_store(struct kobject *kobj,
935 					      struct kobj_attribute *attr,
936 					      const char *buf, size_t count)
937 {
938 	struct amdgpu_ualink_ppod_setup *setup = to_ualink_ppod_setup(kobj);
939 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
940 	struct device *dev = kobj_to_dev(info->kobj.parent);
941 	struct drm_device *ddev = dev_get_drvdata(dev);
942 	struct amdgpu_device *adev = drm_to_adev(ddev);
943 	int r;
944 
945 	if (!sysfs_streq(buf, "true"))
946 		return -EINVAL;
947 
948 	r = psp_ual_set_ppod_config(&adev->psp, adev->ualink.psp_if_ver,
949 				    setup);
950 	if (r)
951 		return r;
952 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
953 	if (r)
954 		return r;
955 
956 	/*
957 	 * Hold mgpu_info.mutex to serialize with activate_local_vpod() in
958 	 * vpod_config_commit_store which also holds this lock. Without it,
959 	 * deactivate_accelerator() can race with activate_accelerator(),
960 	 * causing concurrent vm_fini / vm_init on the same NPA VM.
961 	 */
962 	mutex_lock(&mgpu_info.mutex);
963 	if (info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_READY)
964 		deactivate_accelerator(adev);
965 	info->accel_state = check_ppod_state(adev, setup);
966 	/* PPOD is expected to be configured first */
967 	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
968 	mutex_unlock(&mgpu_info.mutex);
969 
970 	/* TODO: If accel_state was ACTIVE, reset all connections */
971 
972 	return count;
973 }
974 
975 #define UALINK_PPOD_SETUP_ATTR(name) __ATTR(name, 0600,			\
976 		ualink_ppod_setup_##name##_show,			\
977 		ualink_ppod_setup_##name##_store)
978 static struct kobj_attribute ualink_ppod_setup_accel_id  = UALINK_PPOD_SETUP_ATTR(accel_id);
979 static struct kobj_attribute ualink_ppod_setup_bandwidth = UALINK_PPOD_SETUP_ATTR(bandwidth);
980 static struct kobj_attribute ualink_ppod_setup_latency   = UALINK_PPOD_SETUP_ATTR(latency);
981 static struct kobj_attribute ualink_ppod_setup_ppod_id   = UALINK_PPOD_SETUP_ATTR(ppod_id);
982 static struct kobj_attribute ualink_ppod_setup_ppod_size = UALINK_PPOD_SETUP_ATTR(ppod_size);
983 static struct kobj_attribute ualink_ppod_setup_local_accels = UALINK_PPOD_SETUP_ATTR(local_accels);
984 static struct kobj_attribute ualink_ppod_setup_commit = __ATTR(commit, 0200, NULL,
985 							       ualink_ppod_setup_commit_store);
986 
987 static const struct attribute *ualink_ppod_setup_attrs[] = {
988 	&ualink_ppod_setup_accel_id.attr,
989 	&ualink_ppod_setup_bandwidth.attr,
990 	&ualink_ppod_setup_latency.attr,
991 	&ualink_ppod_setup_ppod_id.attr,
992 	&ualink_ppod_setup_ppod_size.attr,
993 	&ualink_ppod_setup_local_accels.attr,
994 	&ualink_ppod_setup_commit.attr,
995 	NULL
996 };
997 #endif
998 
999 static void ualink_ppod_setup_release(struct kobject *kobj)
1000 {
1001 	kfree(to_ualink_ppod_setup(kobj));
1002 }
1003 
1004 static const struct kobj_type ualink_ppod_setup_ktype = {
1005 	.release = ualink_ppod_setup_release,
1006 	.sysfs_ops = &kobj_sysfs_ops
1007 };
1008 
1009 static struct amdgpu_device *find_peer_adev(unsigned int accel_id)
1010 {
1011 	unsigned int i;
1012 
1013 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1014 		struct amdgpu_device *peer_adev = mgpu_info.gpu_ins[i].adev;
1015 
1016 		if (peer_adev->ualink.info &&
1017 		    peer_adev->ualink.info->ppod.accel_id == accel_id)
1018 			return peer_adev;
1019 	}
1020 
1021 	return NULL;
1022 }
1023 
1024 static bool amdgpu_ualink_is_local_accel(struct amdgpu_device *adev,
1025 					 u32 accel_id)
1026 {
1027 	struct amdgpu_ualink_info *info = adev->ualink.info;
1028 	unsigned int i;
1029 
1030 	if (!info)
1031 		return false;
1032 
1033 	for (i = 0; i < info->n_local_accels; i++) {
1034 		if (info->local_accels[i] == accel_id)
1035 			return true;
1036 	}
1037 	return false;
1038 }
1039 
1040 static void activate_accelerator(struct amdgpu_device *adev)
1041 {
1042 	int r;
1043 
1044 	if (adev->ualink.info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
1045 		return;
1046 
1047 	dev_info(adev->dev,
1048 		 "UALINK: activating accelerator accel_id=%u (accel_state=%d)\n",
1049 		 adev->ualink.info->ppod.accel_id,
1050 		 adev->ualink.info->accel_state);
1051 
1052 	/* Enable incoming NPA address translation with NPA VMID */
1053 	r = psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
1054 				   adev->vm_manager.npa_vmid, true);
1055 	if (r) {
1056 		dev_err(adev->dev, "Failed to set NPA config\n");
1057 		return;
1058 	}
1059 
1060 	r = amdgpu_ualink_manager_start(adev);
1061 	if (r) {
1062 		dev_err(adev->dev, "Failed to start UALink manager\n");
1063 		return;
1064 	}
1065 
1066 	r = amdgpu_ualink_sw_init(adev);
1067 	if (r) {
1068 		dev_err(adev->dev, "Failed to init UALink sw r=%d\n", r);
1069 		amdgpu_ualink_manager_stop(adev);
1070 		return;
1071 	}
1072 
1073 	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
1074 	dev_info(adev->dev,
1075 		 "UALINK: accelerator accel_id=%u is now ACTIVE\n",
1076 		 adev->ualink.info->ppod.accel_id);
1077 }
1078 
1079 static void deactivate_accelerator(struct amdgpu_device *adev)
1080 {
1081 	if (adev->ualink.info->accel_state < AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
1082 		return;
1083 
1084 	dev_info(adev->dev,
1085 		 "UALINK: deactivating accelerator accel_id=%u (accel_state=%d -> PPOD_CONFIGURED)\n",
1086 		 adev->ualink.info->ppod.accel_id,
1087 		 adev->ualink.info->accel_state);
1088 
1089 	/* Disable incoming NPA address translation with NPA VMID */
1090 	psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
1091 			       adev->vm_manager.npa_vmid, false);
1092 	/* ignore return value */
1093 	adev->ualink.info->accel_state =
1094 		AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
1095 
1096 	amdgpu_ualink_sw_fini(adev);
1097 	amdgpu_ualink_manager_stop(adev);
1098 }
1099 
1100 static void activate_local_vpod(struct amdgpu_device *adev)
1101 {
1102 	struct amdgpu_ualink_info *info = adev->ualink.info;
1103 	struct amdgpu_device *peer_adev;
1104 	unsigned int accel_id;
1105 	unsigned int i;
1106 
1107 	for (i = 0; i < info->n_local_accels; i++) {
1108 		accel_id = info->local_accels[i];
1109 
1110 		peer_adev = find_peer_adev(accel_id);
1111 		if (WARN_ON(!peer_adev || !peer_adev->ualink.info))
1112 			/* info->local_accels is corrupted? */
1113 			continue;
1114 
1115 		/* Bring the peer up to match the current vPod membership.
1116 		 *
1117 		 * A peer that is not yet ACTIVE (or whose ACTIVE membership
1118 		 * changed on a grow/shrink) has its remote metadata, links and
1119 		 * GART mappings built for a stale member set (or none at all).
1120 		 * Firmware only accepts a full metadata reload while halted, so
1121 		 * surgical per-peer deltas are not possible: fully bounce the
1122 		 * accelerator. deactivate_accelerator() is a no-op when the peer
1123 		 * is not ACTIVE, so this handles first-time bring-up too.
1124 		 * Unchanged ACTIVE peers are left untouched.
1125 		 */
1126 		if (amdgpu_ualink_vpod_membership_changed(peer_adev)) {
1127 			dev_info(peer_adev->dev,
1128 				 "UALINK: (re)configuring vpod for accel_id=%u\n",
1129 				 peer_adev->ualink.info->ppod.accel_id);
1130 			deactivate_accelerator(peer_adev);
1131 			activate_accelerator(peer_adev);
1132 		}
1133 	}
1134 }
1135 
1136 static inline bool __is_vpod_peer(struct amdgpu_ualink_info *info,
1137 				  struct amdgpu_ualink_info *peer_info)
1138 {
1139 	return peer_info->vpod.id == info->vpod.id &&
1140 	       uuid_equal(&peer_info->ppod.id, &info->ppod.id);
1141 }
1142 
1143 static int __check_local_vpod_integrity(struct amdgpu_device *adev)
1144 {
1145 	DECLARE_BITMAP(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
1146 	struct amdgpu_ualink_info *info = adev->ualink.info;
1147 	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
1148 	struct amdgpu_ualink_info *peer_info;
1149 	struct amdgpu_device *peer_adev;
1150 	unsigned int i, n_local_accels;
1151 	unsigned int accel_id;
1152 	/* Check that all local accelerators listed in vpod_active_accels have
1153 	 * matching pod IDs
1154 	 */
1155 	bitmap_zero(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
1156 	n_local_accels = 0;
1157 	__set_bit(info->ppod.accel_id, local_accel_ids);
1158 	local_accels[n_local_accels++] = info->ppod.accel_id;
1159 
1160 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1161 		peer_adev = mgpu_info.gpu_ins[i].adev;
1162 		if (peer_adev == adev || !peer_adev->ualink.info)
1163 			continue;
1164 
1165 		peer_info = peer_adev->ualink.info;
1166 		/* peer device ppod not configured */
1167 		if (peer_info->accel_state <
1168 		    AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
1169 			dev_info(adev->dev,
1170 				 "UALINK: integrity EAGAIN: peer accel_id=%u not ppod-configured (accel_state=%d)\n",
1171 				 peer_info->ppod.accel_id,
1172 				 peer_info->accel_state);
1173 			return -EAGAIN;
1174 		}
1175 
1176 		accel_id = peer_info->ppod.accel_id;
1177 		if (!test_bit(accel_id, info->vpod.active_accel_bits))
1178 			continue;
1179 		/* peer device vpod not configured */
1180 		if (peer_info->accel_state <
1181 		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
1182 			dev_info(adev->dev,
1183 				 "UALINK: integrity EAGAIN: vpod peer accel_id=%u not vpod-configured (accel_state=%d)\n",
1184 				 accel_id, peer_info->accel_state);
1185 			return -EAGAIN;
1186 		}
1187 
1188 		if (!uuid_equal(&peer_info->ppod.id, &info->ppod.id)) {
1189 			dev_warn(adev->dev,
1190 				 "UALINK: integrity fail: peer %u ppod_id doesn't match: %pU != %pU\n",
1191 				 peer_info->ppod.accel_id, &peer_info->ppod.id,
1192 				 &info->ppod.id);
1193 			return -EINVAL;
1194 		}
1195 
1196 		if (peer_info->ppod.size != info->ppod.size) {
1197 			dev_warn(adev->dev,
1198 				 "UALINK: integrity fail: peer %u ppod_size doesn't match: %u != %u\n",
1199 				 accel_id, peer_info->ppod.size,
1200 				 info->ppod.size);
1201 			return -EINVAL;
1202 		}
1203 
1204 		if (peer_info->vpod.id != info->vpod.id) {
1205 			dev_warn(adev->dev,
1206 				 "UALINK: integrity fail: peer %u vpod_id doesn't match: %u != %u\n",
1207 				 accel_id, peer_info->vpod.id, info->vpod.id);
1208 			return -EINVAL;
1209 		}
1210 		if (peer_info->vpod.size != info->vpod.size) {
1211 			dev_warn(adev->dev,
1212 				 "UALINK: integrity fail: peer %u vpod_size doesn't match: %u != %u\n",
1213 				 accel_id, peer_info->vpod.size,
1214 				 info->vpod.size);
1215 			return -EINVAL;
1216 		}
1217 		if (peer_info->vpod.addr_mode != info->vpod.addr_mode) {
1218 			dev_warn(adev->dev,
1219 				 "UALINK: integrity fail: peer %u addr_mode doesn't match: %u != %u\n",
1220 				 accel_id, peer_info->vpod.addr_mode,
1221 				 info->vpod.addr_mode);
1222 			return -EINVAL;
1223 		}
1224 		if (!bitmap_equal(peer_info->vpod.active_accel_bits,
1225 				  info->vpod.active_accel_bits,
1226 				  AMDGPU_UALINK_ACCEL_MAX)) {
1227 			dev_warn(adev->dev,
1228 				 "UALINK: integrity fail: peer %u vpod_active_accels don't match\n",
1229 				 accel_id);
1230 			return -EINVAL;
1231 		}
1232 
1233 		if (__test_and_set_bit(accel_id, local_accel_ids)) {
1234 			dev_warn(adev->dev,
1235 				 "UALINK: integrity fail: duplicate accel_id %u among local vpod peers\n",
1236 				 accel_id);
1237 			return -EINVAL;
1238 		}
1239 		local_accels[n_local_accels++] = accel_id;
1240 	}
1241 
1242 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1243 		peer_adev = mgpu_info.gpu_ins[i].adev;
1244 		peer_info = peer_adev->ualink.info;
1245 
1246 		if (!peer_info)
1247 			continue;
1248 		if (peer_adev != adev && !__is_vpod_peer(info, peer_info))
1249 			continue;
1250 
1251 		peer_info->n_local_accels = n_local_accels;
1252 		memcpy(peer_info->local_accels, local_accels,
1253 		       sizeof(local_accels));
1254 	}
1255 
1256 	return 0;
1257 }
1258 
1259 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev)
1260 {
1261 	int ret;
1262 
1263 	if (adev->ualink.info->accel_state <
1264 		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED ||
1265 	    adev->ualink.info->accel_state ==
1266 		    AMDGPU_UALINK_ACCEL_STATE_ERROR)
1267 		return;
1268 
1269 	ret = __check_local_vpod_integrity(adev);
1270 	if (ret && ret != -EAGAIN) {
1271 		dev_err(adev->dev,
1272 			"Local vpod integrity check failed: %d\n", ret);
1273 		return;
1274 	}
1275 	if (ret == -EAGAIN) {
1276 		dev_info(adev->dev,
1277 			 "UALINK: activate deferred, waiting for local vpod peers to reach VPOD_CONFIGURED\n");
1278 		return;
1279 	}
1280 
1281 	dev_info(adev->dev,
1282 		 "UALINK: integrity OK, applying local vpod (%u local accels)\n",
1283 		 adev->ualink.info->n_local_accels);
1284 	activate_local_vpod(adev);
1285 }
1286 
1287 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1288 UALINK_VALUE_SHOW(vpod_config, vpod_id,   vpod.id,   "%u");
1289 UALINK_VALUE_SHOW(vpod_config, vpod_size, vpod.size, "%u");
1290 UALINK_IDBITS_SHOW(vpod_config, vpod_active_accels, vpod.active_accel_bits);
1291 UALINK_ENUM_SHOW(vpod_config, addr_mode, vpod.addr_mode);
1292 
1293 UALINK_VALUE_STORE(vpod_config, vpod_id,   vpod.id,   u32, 10);
1294 UALINK_VALUE_STORE(vpod_config, vpod_size, vpod.size, u32, 10);
1295 UALINK_IDBITS_STORE(vpod_config, vpod_active_accels, vpod.active_accel_bits,
1296 		    AMDGPU_UALINK_ACCEL_MAX);
1297 UALINK_ENUM_STORE(vpod_config, addr_mode, vpod.addr_mode);
1298 
1299 static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj,
1300 					       struct kobj_attribute *attr,
1301 					       const char *buf, size_t count)
1302 {
1303 	struct amdgpu_ualink_vpod_config *config = to_ualink_vpod_config(kobj);
1304 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
1305 	struct device *dev = kobj_to_dev(info->kobj.parent);
1306 	struct drm_device *ddev = dev_get_drvdata(dev);
1307 	struct amdgpu_device *adev = drm_to_adev(ddev);
1308 	enum amdgpu_ualink_accel_state prev_state;
1309 	u32 prev_vpod_id;
1310 	int r;
1311 
1312 	if (!sysfs_streq(buf, "true"))
1313 		return -EINVAL;
1314 
1315 	dev_dbg(adev->dev,
1316 		"UALINK: vpod-commit enter: accel_state=%d staged vpod_id=%u vpod_size=%u addr_mode=%u\n",
1317 		info->accel_state, config->vpod.id, config->vpod.size,
1318 		config->vpod.addr_mode);
1319 
1320 	if (info->accel_state < AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
1321 		dev_warn(adev->dev,
1322 			 "UALINK: vpod-commit rejected, ppod not configured (accel_state=%d)\n",
1323 			 info->accel_state);
1324 		return -EINVAL;
1325 	}
1326 
1327 	prev_state = info->accel_state;
1328 	prev_vpod_id = info->vpod.id;
1329 	r = psp_ual_set_vpod_config(&adev->psp, adev->ualink.psp_if_ver,
1330 				    config);
1331 	if (r) {
1332 		dev_warn(adev->dev,
1333 			 "UALINK: vpod-commit psp_ual_set_vpod_config failed: %d\n",
1334 			 r);
1335 		return r;
1336 	}
1337 
1338 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
1339 	if (r) {
1340 		dev_warn(adev->dev,
1341 			 "UALINK: vpod-commit psp_ual_query_info failed: %d\n",
1342 			 r);
1343 		return r;
1344 	}
1345 
1346 	dev_dbg(adev->dev,
1347 		"UALINK: vpod-commit fw read-back: vpod_id=%u vpod_size=%u accel_id=%u ppod.size=%u addr_mode=%u\n",
1348 		info->vpod.id, info->vpod.size, info->ppod.accel_id,
1349 		info->ppod.size, info->vpod.addr_mode);
1350 
1351 	if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID &&
1352 	    !__check_vpod_info(adev, info)) {
1353 		dev_err(adev->dev,
1354 			"UALINK: vpod-commit __check_vpod_info() failed: %d\n",
1355 			r);
1356 		return -EINVAL;
1357 	}
1358 
1359 	/* The integrity check makes sure each new GPU is consistent with the
1360 	 * other GPUs already in the vPod. All known local GPUs can become
1361 	 * "ready" at the same time.
1362 	 *
1363 	 * Misconfiguration of one GPU does not reduce the state of other GPUs
1364 	 * already in the vPod. GPU is intentionally not put to ERROR state if
1365 	 * misconfiguration occurs.
1366 	 */
1367 	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
1368 					     UAL_CFG_VPOD);
1369 	if (r)
1370 		return r;
1371 
1372 	return count;
1373 }
1374 
1375 #define UALINK_VPOD_CONFIG_ATTR(name) __ATTR(name, 0600,		\
1376 		ualink_vpod_config_##name##_show,			\
1377 		ualink_vpod_config_##name##_store)
1378 static struct kobj_attribute ualink_vpod_config_vpod_id   = UALINK_VPOD_CONFIG_ATTR(vpod_id);
1379 static struct kobj_attribute ualink_vpod_config_vpod_size = UALINK_VPOD_CONFIG_ATTR(vpod_size);
1380 static struct kobj_attribute ualink_vpod_config_vpod_active_accels =
1381 						UALINK_VPOD_CONFIG_ATTR(vpod_active_accels);
1382 static struct kobj_attribute ualink_vpod_config_addr_mode = UALINK_VPOD_CONFIG_ATTR(addr_mode);
1383 static struct kobj_attribute ualink_vpod_config_commit = __ATTR(commit, 0200, NULL,
1384 								ualink_vpod_config_commit_store);
1385 
1386 static const struct attribute *ualink_vpod_config_attrs[] = {
1387 	&ualink_vpod_config_vpod_id.attr,
1388 	&ualink_vpod_config_vpod_size.attr,
1389 	&ualink_vpod_config_vpod_active_accels.attr,
1390 	&ualink_vpod_config_addr_mode.attr,
1391 	&ualink_vpod_config_commit.attr,
1392 	NULL
1393 };
1394 #endif
1395 
1396 static void ualink_vpod_config_release(struct kobject *kobj)
1397 {
1398 	kfree(to_ualink_vpod_config(kobj));
1399 }
1400 
1401 static const struct kobj_type ualink_vpod_config_ktype = {
1402 	.release = ualink_vpod_config_release,
1403 	.sysfs_ops = &kobj_sysfs_ops
1404 };
1405 
1406 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1407 UALINK_VALUE_SHOW(station_config, flags, flags, "0x%x");
1408 
1409 UALINK_VALUE_STORE(station_config, flags, flags, u8, 16);
1410 
1411 static size_t lane_bitmap_to_string(char *str,
1412 		const struct amdgpu_ualink_station_config *stations)
1413 {
1414 	size_t i;
1415 
1416 	for (i = 0; i < stations->n_stations; i++)
1417 		str[i] = stations->lane_en_bitmap[i] +
1418 			(stations->lane_en_bitmap[i] < 10 ? '0' : 'a' - 10);
1419 	str[i] = '\0';
1420 
1421 	return i;
1422 }
1423 static ssize_t
1424 ualink_station_config_lane_en_bitmap_show(struct kobject *kobj,
1425 					  struct kobj_attribute *attr,
1426 					  char *buf)
1427 {
1428 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1429 	char str[AMDGPU_UALINK_STATIONS_MAX + 1];
1430 
1431 	lane_bitmap_to_string(str, stations);
1432 	return sysfs_emit(buf, "%s\n", str);
1433 }
1434 static ssize_t
1435 ualink_station_config_lane_en_bitmap_store(struct kobject *kobj,
1436 					   struct kobj_attribute *attr,
1437 					   const char *buf, size_t count)
1438 {
1439 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1440 	u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX];
1441 	u32 n_stations = 0;
1442 	bool end = false;
1443 	size_t i;
1444 
1445 	for (i = 0; i < count; i++) {
1446 		/* Accept any number of \n in the end */
1447 		if (buf[i] == '\n') {
1448 			end = true;
1449 			continue;
1450 		}
1451 		/* Don't accept any more data after \n */
1452 		if (end)
1453 			return -EINVAL;
1454 		/* Don't accept more data than the array size */
1455 		if (i >= sizeof(lane_en_bitmap))
1456 			return -ENOSPC;
1457 		/* Accept hex digits */
1458 		if (buf[i] >= '0' && buf[i] <= '9')
1459 			lane_en_bitmap[n_stations++] = buf[i] - '0';
1460 		else if (buf[i] >= 'a' && buf[i] <= 'f')
1461 			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'a';
1462 		else if (buf[i] >= 'A' && buf[i] <= 'F')
1463 			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'A';
1464 		else
1465 			return -EINVAL;
1466 	}
1467 
1468 	memcpy(stations->lane_en_bitmap, lane_en_bitmap, sizeof(lane_en_bitmap));
1469 	stations->n_stations = n_stations;
1470 	return i;
1471 }
1472 
1473 static ssize_t ualink_station_config_commit_store(struct kobject *kobj,
1474 						  struct kobj_attribute *attr,
1475 						  const char *buf, size_t count)
1476 {
1477 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1478 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
1479 	struct device *dev = kobj_to_dev(info->kobj.parent);
1480 	struct drm_device *ddev = dev_get_drvdata(dev);
1481 	struct amdgpu_device *adev = drm_to_adev(ddev);
1482 	int r;
1483 
1484 	/* DF reconfiguration does not interact with accelerator state */
1485 
1486 	if (!sysfs_streq(buf, "true"))
1487 		return -EINVAL;
1488 
1489 	r = psp_ual_set_station_config(&adev->psp, adev->ualink.psp_if_ver, stations);
1490 	if (r)
1491 		return r;
1492 
1493 	return count;
1494 }
1495 
1496 #define UALINK_STATION_CONFIG_ATTR(name) __ATTR(name, 0600,		\
1497 		ualink_station_config_##name##_show,			\
1498 		ualink_station_config_##name##_store)
1499 static struct kobj_attribute ualink_station_config_flags  = UALINK_STATION_CONFIG_ATTR(flags);
1500 static struct kobj_attribute ualink_station_config_lane_en_bitmap =
1501 							UALINK_STATION_CONFIG_ATTR(lane_en_bitmap);
1502 static struct kobj_attribute ualink_station_config_commit =
1503 				__ATTR(commit, 0200, NULL, ualink_station_config_commit_store);
1504 
1505 static const struct attribute *ualink_station_config_attrs[] = {
1506 	&ualink_station_config_flags.attr,
1507 	&ualink_station_config_lane_en_bitmap.attr,
1508 	&ualink_station_config_commit.attr,
1509 	NULL
1510 };
1511 #endif
1512 
1513 static void ualink_station_config_release(struct kobject *kobj)
1514 {
1515 	kfree(to_ualink_station_config(kobj));
1516 }
1517 
1518 static const struct kobj_type ualink_station_config_ktype = {
1519 	.release = ualink_station_config_release,
1520 	.sysfs_ops = &kobj_sysfs_ops
1521 };
1522 
1523 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block)
1524 {
1525 	struct amdgpu_ualink_station_config *stations;
1526 	struct amdgpu_ualink_vpod_config *vpod_config;
1527 	struct amdgpu_ualink_ppod_setup *ppod_setup;
1528 	struct amdgpu_device *adev = ip_block->adev;
1529 	struct amdgpu_ualink_info *info;
1530 
1531 	info = kzalloc(sizeof(*info), GFP_KERNEL);
1532 	ppod_setup = kzalloc(sizeof(*ppod_setup), GFP_KERNEL);
1533 	vpod_config = kzalloc(sizeof(*vpod_config), GFP_KERNEL);
1534 	stations = kzalloc(sizeof(*stations), GFP_KERNEL);
1535 	if (!info || !ppod_setup || !vpod_config || !stations) {
1536 		kfree(info);
1537 		kfree(ppod_setup);
1538 		kfree(vpod_config);
1539 		kfree(stations);
1540 		return -ENOMEM;
1541 	}
1542 
1543 	info->ppod.accel_id = 0xffffffff;
1544 	info->ppod.bandwidth = 0xffffffff;
1545 	info->ppod.latency = 0xffffffff;
1546 	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
1547 	info->vpod.addr_mode = AMDGPU_UALINK_ADDR_MODE_MAX;
1548 
1549 	/*
1550 	 * Initialize the kobjects here so their lifetime is tied to the UALink
1551 	 * manager software state. amdgpu_ualink_object_fini() drops the final
1552 	 * reference via kobject_put().
1553 	 */
1554 	kobject_init(&info->kobj, &ualink_info_ktype);
1555 	kobject_init(&ppod_setup->kobj, &ualink_ppod_setup_ktype);
1556 	kobject_init(&vpod_config->kobj, &ualink_vpod_config_ktype);
1557 	kobject_init(&stations->kobj, &ualink_station_config_ktype);
1558 
1559 	adev->ualink.info = info;
1560 	adev->ualink.setup = ppod_setup;
1561 	adev->ualink.config = vpod_config;
1562 	adev->ualink.stations = stations;
1563 
1564 	return 0;
1565 }
1566 
1567 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block)
1568 {
1569 	amdgpu_ualink_object_fini(ip_block->adev);
1570 	return 0;
1571 }
1572 
1573 static int ualink_kobj_add(struct kobject *kobj, struct kobject *parent,
1574 			   const char *name, const struct attribute **attrs)
1575 {
1576 	int r;
1577 
1578 	r = kobject_add(kobj, parent, "%s", name);
1579 	if (r)
1580 		return r;
1581 	r = sysfs_create_files(kobj, attrs);
1582 	if (r)
1583 		kobject_del(kobj);
1584 
1585 	return r;
1586 }
1587 
1588 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev);
1589 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev);
1590 
1591 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev)
1592 {
1593 	struct amdgpu_ualink_info *info = adev->ualink.info;
1594 	int r;
1595 
1596 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
1597 		return 0;
1598 
1599 	/* ualink parent node */
1600 	r = ualink_kobj_add(&info->kobj, &adev->dev->kobj, "ualink",
1601 			    ualink_info_attrs);
1602 	if (r)
1603 		goto err;
1604 
1605 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1606 	r = ualink_kobj_add(&adev->ualink.setup->kobj, &info->kobj, "setup",
1607 			    ualink_ppod_setup_attrs);
1608 	if (r)
1609 		goto err_info;
1610 
1611 	r = ualink_kobj_add(&adev->ualink.config->kobj, &info->kobj, "config",
1612 			    ualink_vpod_config_attrs);
1613 	if (r)
1614 		goto err_setup;
1615 
1616 	r = ualink_kobj_add(&adev->ualink.stations->kobj, &info->kobj,
1617 			    "stations", ualink_station_config_attrs);
1618 	if (r)
1619 		goto err_config;
1620 #else
1621 	r = kobject_add(&adev->ualink.setup->kobj, &info->kobj, "%s",
1622 			"setup");
1623 	if (r)
1624 		goto err_info;
1625 
1626 	r = kobject_add(&adev->ualink.config->kobj, &info->kobj, "%s",
1627 			"config");
1628 	if (r)
1629 		goto err_setup;
1630 
1631 	r = kobject_add(&adev->ualink.stations->kobj, &info->kobj, "%s",
1632 			"stations");
1633 	if (r)
1634 		goto err_config;
1635 #endif
1636 
1637 	amdgpu_ualink_xcp_sysfs_init(adev);
1638 	adev->ualink.sysfs_init = true;
1639 
1640 	return 0;
1641 
1642 err_config:
1643 	kobject_del(&adev->ualink.config->kobj);
1644 err_setup:
1645 	kobject_del(&adev->ualink.setup->kobj);
1646 err_info:
1647 	kobject_del(&info->kobj);
1648 err:
1649 	dev_warn(adev->dev, "Failed to create UALink sysfs: %d\n", r);
1650 	return 0;
1651 }
1652 
1653 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev)
1654 {
1655 	if (!adev->ualink.sysfs_init)
1656 		return;
1657 
1658 	amdgpu_ualink_xcp_sysfs_fini(adev);
1659 	kobject_del(&adev->ualink.stations->kobj);
1660 	kobject_del(&adev->ualink.config->kobj);
1661 	kobject_del(&adev->ualink.setup->kobj);
1662 	kobject_del(&adev->ualink.info->kobj);
1663 	adev->ualink.sysfs_init = false;
1664 }
1665 
1666 static int amdgpu_ualink_xcp_sysfs_add(struct amdgpu_xcp *xcp)
1667 {
1668 	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
1669 	int r;
1670 
1671 	r = kobject_init_and_add(&xcp->ualink.kobj, &ualink_xcp_info_ktype,
1672 				 &xcp->ddev->dev->kobj, "ualink");
1673 	if (r)
1674 		goto err;
1675 
1676 	r = sysfs_create_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1677 	if (r)
1678 		goto err;
1679 
1680 	/* pin info so it outlives this node regardless of teardown order */
1681 	kobject_get(&adev->ualink.info->kobj);
1682 	xcp->ualink.sysfs = true;
1683 	return 0;
1684 err:
1685 	kobject_put(&xcp->ualink.kobj);
1686 	return r;
1687 }
1688 
1689 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp)
1690 {
1691 	if (!xcp->ualink.sysfs)
1692 		return;
1693 
1694 	sysfs_update_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1695 }
1696 
1697 static void amdgpu_ualink_xcp_sysfs_remove(struct amdgpu_xcp *xcp)
1698 {
1699 	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
1700 
1701 	if (!xcp->ualink.sysfs)
1702 		return;
1703 
1704 	/* group is only populated for valid partitions */
1705 	if (xcp->valid)
1706 		sysfs_remove_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1707 	kobject_put(&adev->ualink.info->kobj);
1708 	kobject_put(&xcp->ualink.kobj);
1709 	xcp->ualink.sysfs = false;
1710 }
1711 
1712 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev)
1713 {
1714 	struct amdgpu_xcp *xcp;
1715 	int i;
1716 
1717 	if (!adev->xcp_mgr)
1718 		return;
1719 
1720 	for (i = 0; i < MAX_XCP; i++) {
1721 		xcp = &adev->xcp_mgr->xcp[i];
1722 		if (!xcp->ddev || amdgpu_xcp_is_primary(xcp))
1723 			continue;
1724 		amdgpu_ualink_xcp_sysfs_add(xcp);
1725 	}
1726 }
1727 
1728 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev)
1729 {
1730 	struct amdgpu_xcp *xcp;
1731 	int i;
1732 
1733 	if (!adev->xcp_mgr)
1734 		return;
1735 
1736 	for (i = 0; i < MAX_XCP; i++) {
1737 		xcp = &adev->xcp_mgr->xcp[i];
1738 		amdgpu_ualink_xcp_sysfs_remove(xcp);
1739 	}
1740 }
1741 
1742 static int amdgpu_ualink_npa_alloc_va(struct amdgpu_device *adev,
1743 			       struct drm_mm_node *mm_node,
1744 			       u64 va, u64 range_start,
1745 			       u64 range_end, int size)
1746 {
1747 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1748 	u64 alignment = 0;
1749 	int ret;
1750 
1751 	mutex_lock(&npa_mm->mm_lock);
1752 	if (va) {
1753 		mm_node->start = va;
1754 		mm_node->size = size;
1755 		ret = drm_mm_reserve_node(&npa_mm->mm, mm_node);
1756 	} else {
1757 		if (!range_start && !range_end) {
1758 			range_start = npa_mm->va_start;
1759 			range_end = npa_mm->va_start + npa_mm->va_size;
1760 		}
1761 
1762 		/* If size is greater than 2MB (or 512 pages),
1763 		 * align it to 2MB (or 512 pages) granularity.
1764 		 */
1765 		if (size >= 0x200)
1766 			alignment = 0x200;
1767 
1768 		ret = drm_mm_insert_node_in_range(&npa_mm->mm, mm_node, size,
1769 						  alignment, 0, range_start,
1770 						  range_end, 0);
1771 	}
1772 	mutex_unlock(&npa_mm->mm_lock);
1773 
1774 	if (ret)
1775 		dev_err(adev->dev, "Failed to allocate NPA address\n");
1776 
1777 	return ret;
1778 }
1779 
1780 static void amdgpu_ualink_npa_free_va(struct amdgpu_device *adev,
1781 			       struct drm_mm_node *mm_node)
1782 {
1783 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1784 
1785 	mutex_lock(&npa_mm->mm_lock);
1786 	drm_mm_remove_node(mm_node);
1787 	mutex_unlock(&npa_mm->mm_lock);
1788 }
1789 
1790 static void amdgpu_ualink_npa_mm_init(struct amdgpu_device *adev)
1791 {
1792 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1793 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
1794 	u64 va_size;
1795 
1796 	mutex_init(&npa_mm->mm_lock);
1797 
1798 	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
1799 		/* 51 bit address space in Source-Identification mode.
1800 		 * 39 bits in page granularity.
1801 		 */
1802 		npa_mm->va_start = 0;
1803 		va_size = GENMASK_ULL(38, 0);
1804 	} else {
1805 		/* 41 bit address space in Source-Alias mode.
1806 		 * 29 bits in page granularity.
1807 		 */
1808 		/* First 8MB is reserved for Metadata NPAs */
1809 		npa_mm->va_start = 0x800;
1810 		va_size = GENMASK_ULL(28, 0);
1811 	}
1812 
1813 	npa_mm->va_size = va_size - npa_mm->va_start;
1814 	drm_mm_init(&npa_mm->mm, npa_mm->va_start, npa_mm->va_size);
1815 }
1816 
1817 static void amdgpu_ualink_npa_mm_fini(struct amdgpu_device *adev)
1818 {
1819 	mutex_destroy(&adev->ualink.npa_mm.mm_lock);
1820 	drm_mm_takedown(&adev->ualink.npa_mm.mm);
1821 }
1822 
1823 /* The caller of this function is expected to hold the XA lock when calling
1824  * this function.
1825  */
1826 static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev,
1827 				   struct amdgpu_ualink_handle *handle)
1828 {
1829 	bool unique = false;
1830 
1831 	do {
1832 		handle->handle_lo = get_random_u64();
1833 		/* Replace bottom 10 bits in handle_lo with accId */
1834 		handle->handle_lo &= ~AMDGPU_UALINK_HANDLE_ACCID_MASK;
1835 		handle->handle_lo |= adev->ualink.info->ppod.accel_id;
1836 
1837 		/* Don't generate/store a Handle with value 0. */
1838 		if (!handle->handle_lo)
1839 			continue;
1840 
1841 		/* Find if the handle already exists in the exporter xarray.
1842 		 * If it already exists, then regenerate the handle since we
1843 		 * want the handle to be unique.
1844 		 */
1845 		unique = !xa_load(&adev->ualink.exp_xa, handle->handle_lo);
1846 	} while (!unique);
1847 
1848 	handle->handle_hi = get_random_u64();
1849 	dev_dbg(adev->dev, "GENERATE-HANDLE: generated handle: %llx:%llx\n",
1850 		handle->handle_hi, handle->handle_lo);
1851 }
1852 
1853 static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref)
1854 {
1855 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
1856 	struct amdgpu_device *adev;
1857 
1858 	exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node,
1859 				   refcount);
1860 	adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev);
1861 	queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work);
1862 }
1863 
1864 static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref)
1865 {
1866 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
1867 	struct amdgpu_device *adev;
1868 
1869 	imp_xa_node = container_of(ref, struct amdgpu_ualink_imp_xa_node,
1870 				   refcount);
1871 	adev = imp_xa_node->adev;
1872 
1873 	dev_dbg(adev->dev, "IMP-CLEANUP: Remove entry from XA for handle:%llx:%llx\n",
1874 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1875 
1876 	/* Remove node from the Xarray */
1877 	if (!xa_erase(&adev->ualink.imp_xa, imp_xa_node->handle.handle_lo))
1878 		dev_err(adev->dev, "IMP-CLEANUP: Failed to find entry in XA for handle:%llx:%llx\n",
1879 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1880 
1881 	if (imp_xa_node->npa_addr && imp_xa_node->size && imp_xa_node->dmabuf) {
1882 		dev_dbg(adev->dev,
1883 			"IMP-CLEANUP: dmabuf free for NPA:%llx handle:%llx:%llx, fc:%lu\n",
1884 			imp_xa_node->npa_addr, imp_xa_node->handle.handle_hi,
1885 			imp_xa_node->handle.handle_lo,
1886 			file_count(imp_xa_node->dmabuf->file));
1887 		dma_buf_put(imp_xa_node->dmabuf);
1888 		drm_gem_handle_delete(adev->ualink.client.file,
1889 				      imp_xa_node->gem_handle);
1890 
1891 	}
1892 
1893 	dev_dbg(adev->dev, "IMP-CLEANUP: Freeing XA entry for handle:%llx:%llx\n",
1894 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1895 
1896 	kfree(imp_xa_node);
1897 }
1898 
1899 static int amdgpu_ualink_exp_xa_entry_get(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
1900 {
1901 	return kref_get_unless_zero(&exp_xa_node->refcount);
1902 }
1903 
1904 static void amdgpu_ualink_exp_xa_entry_put(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
1905 {
1906 	kref_put(&exp_xa_node->refcount, amdgpu_ualink_cleanup_exp_xa_node);
1907 }
1908 
1909 static int amdgpu_ualink_imp_xa_entry_get(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
1910 {
1911 	return kref_get_unless_zero(&imp_xa_node->refcount);
1912 }
1913 
1914 static void amdgpu_ualink_imp_xa_entry_put(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
1915 {
1916 	kref_put(&imp_xa_node->refcount, amdgpu_ualink_cleanup_imp_xa_node);
1917 }
1918 
1919 static int amdgpu_ualink_send_npa_release_msg(struct amdgpu_device *adev,
1920 				    u32 remote_acc_id,
1921 				    struct amdgpu_ualink_handle handle)
1922 {
1923 	u32 dw0, dw1, dw2, dw3;
1924 
1925 	dw0 = lower_32_bits(handle.handle_lo);
1926 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1927 	dw0 |= AMDGPU_UALINK_NPA_RELEASE_MSG;
1928 
1929 	dw1 = upper_32_bits(handle.handle_lo);
1930 	dw2 = lower_32_bits(handle.handle_hi);
1931 	dw3 = upper_32_bits(handle.handle_hi);
1932 
1933 	dev_dbg(adev->dev, "SEND NPA-RELEASE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1934 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1935 
1936 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1937 					      dw2, dw3);
1938 }
1939 
1940 static int amdgpu_ualink_send_npa_revoke_msg(struct amdgpu_device *adev,
1941 					u32 remote_acc_id,
1942 					struct amdgpu_ualink_handle handle)
1943 {
1944 	u32 dw0, dw1, dw2, dw3;
1945 
1946 	dw0 = lower_32_bits(handle.handle_lo);
1947 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1948 	dw0 |= AMDGPU_UALINK_NPA_REVOKE_MSG;
1949 
1950 	dw1 = upper_32_bits(handle.handle_lo);
1951 	dw2 = lower_32_bits(handle.handle_hi);
1952 	dw3 = upper_32_bits(handle.handle_hi);
1953 
1954 	dev_dbg(adev->dev, "SEND NPA-REVOKE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1955 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1956 
1957 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1958 					      dw2, dw3);
1959 }
1960 
1961 static int amdgpu_ualink_send_npa_fail_msg(struct amdgpu_device *adev,
1962 				    u32 remote_acc_id,
1963 				    struct amdgpu_ualink_handle handle,
1964 				    u32 fail_reason)
1965 {
1966 	u32 dw0, dw1, dw2, dw3;
1967 
1968 	dw0 = lower_32_bits(handle.handle_lo);
1969 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1970 	dw0 |= AMDGPU_UALINK_NPA_FAIL_MSG;
1971 
1972 	dw1 = upper_32_bits(handle.handle_lo);
1973 	dw2 = fail_reason & 0xFF;
1974 	dw3 = 0;
1975 
1976 	dev_dbg(adev->dev, "SEND NPA-FAIL: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1977 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1978 
1979 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1980 					      dw2, dw3);
1981 }
1982 
1983 static int amdgpu_ualink_send_npa_rsp_msg(struct amdgpu_device *adev,
1984 					  u32 remote_acc_id,
1985 					  struct amdgpu_ualink_handle handle,
1986 					  u32 npa_addr, u32 size)
1987 {
1988 	u32 dw0, dw1, dw2, dw3;
1989 
1990 	dw0 = lower_32_bits(handle.handle_lo);
1991 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1992 	dw0 |= AMDGPU_UALINK_NPA_RSP_MSG;
1993 
1994 	dw1 = upper_32_bits(handle.handle_lo);
1995 	dw2 = size;
1996 	dw3 = npa_addr;
1997 
1998 	dev_dbg(adev->dev, "SEND NPA-RSP: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1999 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
2000 
2001 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
2002 					      dw2, dw3);
2003 }
2004 
2005 static int amdgpu_ualink_send_npa_req_msg(struct amdgpu_device *adev,
2006 					  u32 remote_acc_id,
2007 					  struct amdgpu_ualink_handle handle)
2008 {
2009 	u32 dw0, dw1, dw2, dw3;
2010 
2011 	dw0 =  lower_32_bits(handle.handle_lo);
2012 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
2013 	dw0 |= AMDGPU_UALINK_NPA_REQ_MSG;
2014 
2015 	dw1 = upper_32_bits(handle.handle_lo);
2016 	dw2 = lower_32_bits(handle.handle_hi);
2017 	dw3 = upper_32_bits(handle.handle_hi);
2018 
2019 	dev_dbg(adev->dev, "SEND NPA-REQ: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
2020 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
2021 
2022 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
2023 					      dw2, dw3);
2024 }
2025 
2026 static u64 amdgpu_ualink_get_export_pte_flags(struct amdgpu_device *adev,
2027 				       struct amdgpu_bo *bo,
2028 				       u64 mapping_flags)
2029 {
2030 	u64 pte_flags = adev->gmc.init_pte_flags;
2031 
2032 	pte_flags |= (AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE |
2033 		      AMDGPU_PTE_WRITEABLE);
2034 	mapping_flags |= AMDGPU_VM_MTYPE_DEFAULT;
2035 
2036 	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, bo, mapping_flags,
2037 			      &pte_flags);
2038 
2039 	return pte_flags;
2040 }
2041 
2042 static int amdgpu_ualink_unmap_npa_addr(struct amdgpu_device *adev,
2043 					struct amdgpu_bo *bo,
2044 					u64 npa_addr, u64 size)
2045 {
2046 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2047 	uint64_t pte_value = adev->gmc.noretry_flags;
2048 	struct amdgpu_bo *bos[] = { bo };
2049 	struct dma_fence *fence = NULL;
2050 	struct drm_exec exec;
2051 	int r;
2052 
2053 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
2054 	if (unlikely(r)) {
2055 		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_npa_addr\n");
2056 		return r;
2057 	}
2058 
2059 	r = amdgpu_vm_update_range(adev, vm, false, false, true,
2060 				false, NULL, npa_addr, npa_addr + size - 1,
2061 				pte_value, 0, 0, NULL, NULL, &vm->last_update);
2062 	if (r) {
2063 		dev_err(adev->dev,
2064 			"Failed to unmap NPA addr (%llx) from NPA VM\n", npa_addr);
2065 		goto out;
2066 	}
2067 
2068 	r = amdgpu_vm_update_pdes(adev, vm, false);
2069 	if (r) {
2070 		dev_err(adev->dev,
2071 			"Failed %d to update page directories during unmapping NPA: 0x%llx\n",
2072 			r, npa_addr);
2073 		goto out;
2074 	}
2075 
2076 	fence = dma_fence_get(vm->last_update);
2077 	if (fence) {
2078 		r = dma_fence_wait(fence, false);
2079 		dma_fence_put(fence);
2080 		fence = NULL;
2081 		if (r) {
2082 			dev_dbg(adev->dev,
2083 				"UNMAP-NPA: dma fence wait failed, error: %d\n", r);
2084 			goto out;
2085 		}
2086 	}
2087 
2088 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2089 out:
2090 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2091 
2092 	return r;
2093 }
2094 
2095 static int amdgpu_ualink_map_npa_addr(struct amdgpu_device *adev, u64 npa_addr,
2096 				u64 size, struct amdgpu_bo *bo, u64 offset,
2097 				u64 pte_flags)
2098 {
2099 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2100 	struct amdgpu_bo *bos[] = { bo };
2101 	struct dma_fence *fence = NULL;
2102 	struct drm_exec exec;
2103 	int r;
2104 
2105 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
2106 	if (unlikely(r)) {
2107 		dev_err(adev->dev, "Failed to reserve VM and BO in map_npa_addr\n");
2108 		return r;
2109 	}
2110 
2111 	r = amdgpu_vm_update_range(adev, vm, false, false, true,
2112 				false, NULL, npa_addr, npa_addr + size - 1,
2113 				pte_flags, offset, adev->vm_manager.vram_base_offset,
2114 				bo->tbo.resource, NULL, &vm->last_update);
2115 	if (r) {
2116 		dev_warn(adev->dev,
2117 			"Failed to map NPA addr (%llx) into NPA VM\n", npa_addr);
2118 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2119 		goto out;
2120 	}
2121 
2122 	r = amdgpu_vm_update_pdes(adev, vm, false);
2123 	if (r) {
2124 		dev_err(adev->dev,
2125 			"failed %d to update page directories for NPA: 0x%llx\n",
2126 			r, npa_addr);
2127 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2128 		goto unmap_npa;
2129 	}
2130 
2131 	fence = dma_fence_get(vm->last_update);
2132 	if (fence) {
2133 		r = dma_fence_wait(fence, false);
2134 		dma_fence_put(fence);
2135 		fence = NULL;
2136 		if (r) {
2137 			pr_debug("failed %d to dma fence wait\n", r);
2138 			amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2139 			goto unmap_npa;
2140 		}
2141 	}
2142 
2143 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2144 
2145 	/* TLB flush may be needed after updated page directories */
2146 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2147 
2148 	return 0;
2149 
2150 unmap_npa:
2151 	amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
2152 out:
2153 	return r;
2154 }
2155 
2156 static int amdgpu_ualink_send_hello_ack_msg(struct amdgpu_device *adev,
2157 					    u32 remote_acc_id)
2158 {
2159 	dev_dbg(adev->dev, "SEND HELLO-ACK: HELLO-ACK message to remote AccId:%u\n",
2160 		remote_acc_id);
2161 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id,
2162 		AMDGPU_UALINK_HELLO_ACK_MSG,
2163 		0, 0, 0);
2164 }
2165 
2166 static int amdgpu_ualink_send_hello_msg(struct amdgpu_device *adev,
2167 					u32 remote_acc_id)
2168 {
2169 	u32 dw0;
2170 
2171 	dw0 = AMDGPU_UALINK_HELLO_MSG;
2172 	dw0 |= (remote_acc_id << AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT);
2173 	dw0 |= (adev->ualink.info->ppod.accel_id <<
2174 		AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT);
2175 
2176 	dev_dbg(adev->dev, "SEND HELLO: HELLO message to remote AccId:%u\n",
2177 		remote_acc_id);
2178 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, 0,
2179 					      0, 0);
2180 }
2181 
2182 static u32 amdgpu_ualink_check_conn_ready(struct amdgpu_device *adev,
2183 					  u32 remote_acc_id, u32 gen_count)
2184 {
2185 	struct amdgpu_ualink_connection *conn_state;
2186 	u32 current_gen_count = 0;
2187 
2188 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2189 
2190 	/* Check if the connection is established. */
2191 	mutex_lock(&conn_state->lock);
2192 	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
2193 	    (!gen_count || conn_state->generation_count == gen_count))
2194 		current_gen_count = conn_state->generation_count;
2195 	mutex_unlock(&conn_state->lock);
2196 
2197 	return current_gen_count;
2198 }
2199 
2200 static void amdgpu_ualink_process_hello_ack_msg(struct amdgpu_device *adev,
2201 					       u32 sender_acc_id)
2202 {
2203 	struct amdgpu_ualink_connection *conn_state;
2204 
2205 	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
2206 		dev_err(adev->dev,
2207 			"HELLO-ACK: sender AccId out of range:%u\n",
2208 			sender_acc_id);
2209 		return;
2210 	}
2211 
2212 	conn_state = &adev->ualink.conn_state[sender_acc_id];
2213 
2214 	/* If we are in IN_PROGRESS state, then transition the connection
2215 	 * state to established and signal that the HELLO ACK is received.
2216 	 * Otherwise, ignore the HELLO ACK.
2217 	 */
2218 	mutex_lock(&conn_state->lock);
2219 
2220 	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) {
2221 		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
2222 		conn_state->generation_count++;
2223 		complete(&conn_state->hello_done);
2224 	} else {
2225 		dev_dbg(adev->dev,
2226 			"HELLO-ACK: already connected, ignoring from AccId:%u\n",
2227 			sender_acc_id);
2228 	}
2229 	mutex_unlock(&conn_state->lock);
2230 }
2231 
2232 static void amdgpu_ualink_process_hello_msg(struct amdgpu_device *adev,
2233 					   u32 receiver_acc_id,
2234 					   u32 sender_acc_id,
2235 					   u32 src_acc_id)
2236 {
2237 	struct amdgpu_ualink_connection *conn_state;
2238 	u32 generation_count;
2239 	int r;
2240 
2241 	if (receiver_acc_id != adev->ualink.info->ppod.accel_id) {
2242 		dev_err(adev->dev,
2243 			"HELLO: receiver AccId mismatch got:%u self:%u\n",
2244 			receiver_acc_id, adev->ualink.info->ppod.accel_id);
2245 		return;
2246 	}
2247 
2248 	/* src_acc_id is the AccId received in IH cookie. Confirm it
2249 	 * matches with the sender AccId.
2250 	 */
2251 	if (sender_acc_id != src_acc_id) {
2252 		dev_err(adev->dev,
2253 			"HELLO: sender AccId mismatch sender:%u IH cookie:%u\n",
2254 			sender_acc_id, src_acc_id);
2255 		return;
2256 	}
2257 
2258 	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
2259 		dev_err(adev->dev,
2260 			"HELLO: sender AccId out of range:%u\n",
2261 			sender_acc_id);
2262 		return;
2263 	}
2264 
2265 	conn_state = &adev->ualink.conn_state[sender_acc_id];
2266 
2267 	/* Check if connection is already established. If yes, then receiving HELLO msg
2268 	 * triggers a reset handling scenario.
2269 	 * If the connection is not ready, and we receive a HELLO msg, then
2270 	 * transition the state to PENDING.
2271 	 * Otherwise, leave it IN_PROGRESS.
2272 	 */
2273 	mutex_lock(&conn_state->lock);
2274 	if (conn_state->state != AMDGPU_UALINK_CONN_ESTABLISHED) {
2275 		if (conn_state->state == AMDGPU_UALINK_CONN_NOT_READY)
2276 			conn_state->state = AMDGPU_UALINK_CONN_PENDING;
2277 		/* otherwise, leave it IN_PROGRESS to signal the completion below */
2278 		mutex_unlock(&conn_state->lock);
2279 	} else {
2280 		/* Leave state ESTABLISHED so handle_connection_reset()
2281 		 * performs the transition and imp/exp XA cleanup.
2282 		 */
2283 		generation_count = conn_state->generation_count;
2284 		mutex_unlock(&conn_state->lock);
2285 
2286 		amdgpu_ualink_handle_connection_reset(adev, sender_acc_id,
2287 						AMDGPU_UALINK_CONN_PENDING,
2288 						generation_count);
2289 	}
2290 
2291 	/* Peer may have rebooted. */
2292 	r = amdgpu_ualink_ring_resync_ptrs(adev, sender_acc_id);
2293 	if (r < 0) {
2294 		dev_err(adev->dev,
2295 			"HELLO: resync sender AccId %u ptrs failed %d\n",
2296 			sender_acc_id, r);
2297 		goto out_set_state;
2298 	}
2299 	if (r)
2300 		amdgpu_ualink_reset_peer_rings(adev, sender_acc_id);
2301 
2302 	r = amdgpu_ualink_send_hello_ack_msg(adev, sender_acc_id);
2303 	if (r)
2304 		dev_err(adev->dev, "HELLO-ACK: send failed to remote AccId:%u\n",
2305 			sender_acc_id);
2306 
2307 out_set_state:
2308 	mutex_lock(&conn_state->lock);
2309 	if (r) {
2310 		conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
2311 	} else {
2312 		/* If we are in IN_PROGRESS state and we receivied the HELLO message,
2313 		 * upon receiving the HELLO message, transition the state to ESTABLISHED
2314 		 * and signal the completion.
2315 		 */
2316 		if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS)
2317 			complete(&conn_state->hello_done);
2318 		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
2319 		conn_state->generation_count++;
2320 	}
2321 	mutex_unlock(&conn_state->lock);
2322 }
2323 
2324 /* True if nothing is exported to, or imported from, this remote accelerator. */
2325 static bool amdgpu_ualink_peer_idle(struct amdgpu_device *adev, u32 remote_accel_id)
2326 {
2327 	/* Unlocked: a racing import at worst costs one redundant ring resync. */
2328 	return list_empty(&adev->ualink.imp_handles_list[remote_accel_id]) &&
2329 	       list_empty(&adev->ualink.exp_handles_list[remote_accel_id]);
2330 }
2331 
2332 static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev,
2333 					  u32 remote_acc_id)
2334 {
2335 	struct amdgpu_ualink_connection *conn_state;
2336 	int r;
2337 
2338 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2339 
2340 	/* Connection state management goes through different states.
2341 	 * The states are:
2342 	 * - NOT_READY: The connection is not ready.
2343 	 * - IN_PROGRESS: GPU sent HELLO message and is waiting for the HELLO_ACK.
2344 	 * - PENDING: GPU received HELLO message and is in the process of sending
2345 	 *            the HELLO_ACK.
2346 	 * - ESTABLISHED: The connection is established.
2347 	 */
2348 
2349 	/* Grab the lock and check if connection establishment was
2350 	 * already done by another thread.
2351 	 */
2352 	mutex_lock(&conn_state->lock);
2353 	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS ||
2354 	    conn_state->state == AMDGPU_UALINK_CONN_PENDING) {
2355 		r = -EAGAIN;
2356 		goto out;
2357 	}
2358 
2359 	if (conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED &&
2360 	    !amdgpu_ualink_peer_idle(adev, remote_acc_id)) {
2361 		r = 0;
2362 		goto out;
2363 	}
2364 
2365 	/* Peer is idle or not connected, it may have rebooted. Skip the
2366 	 * handshake if the rings are still in sync, it would reset the remote.
2367 	 */
2368 	r = amdgpu_ualink_ring_resync_ptrs(adev, remote_acc_id);
2369 	if (r < 0)
2370 		goto out;
2371 	if (!r && conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED)
2372 		goto out;
2373 
2374 	amdgpu_ualink_reset_peer_rings(adev, remote_acc_id);
2375 
2376 	conn_state->state = AMDGPU_UALINK_CONN_IN_PROGRESS;
2377 
2378 	mutex_unlock(&conn_state->lock);
2379 
2380 	/* Send HELLO message */
2381 	r = amdgpu_ualink_send_hello_msg(adev, remote_acc_id);
2382 	if (r) {
2383 		dev_warn(adev->dev,
2384 			"HELLO: Send failed to remote AccId:%u\n",
2385 			remote_acc_id);
2386 		goto reset_state;
2387 	}
2388 
2389 	/* Wait for the HELLO_ACK to come back */
2390 	/* complete(conn_state->hello_done) should be called from the IRQ
2391 	 * handler when the HELLO_ACK is received.
2392 	 */
2393 	r = wait_for_completion_interruptible_timeout(&conn_state->hello_done,
2394 				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
2395 	if (r == -ERESTARTSYS) {
2396 		dev_err_ratelimited(adev->dev,
2397 			"HELLO-ACK: interrupted by signal\n");
2398 		goto reset_state;
2399 	} else if (r == 0) {
2400 		dev_warn(adev->dev,
2401 			"HELLO-ACK: Timeout from remote AccId:%u\n",
2402 			remote_acc_id);
2403 		r = -ETIMEDOUT;
2404 		goto reset_state;
2405 	}
2406 
2407 	return 0;
2408 
2409 reset_state:
2410 	mutex_lock(&conn_state->lock);
2411 	conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
2412 out:
2413 	mutex_unlock(&conn_state->lock);
2414 
2415 	return r;
2416 }
2417 
2418 /* Invalidate an importer node's mappings and drop its last ref (frees the
2419  * node, dma-buf and GEM handle). Caller must have unlinked it from the
2420  * per-remote list and must not hold the imp_xa lock.
2421  */
2422 static void amdgpu_ualink_release_imp_xa_node(struct amdgpu_device *adev,
2423 				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
2424 {
2425 	struct amdgpu_bo *bo;
2426 
2427 	dev_dbg(adev->dev,
2428 		"IMP-CLEANUP: handle:%llx:%llx npa:%llx size:%llx\n",
2429 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
2430 		imp_xa_node->npa_addr, imp_xa_node->size);
2431 
2432 	if (imp_xa_node->dmabuf) {
2433 		bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
2434 		amdgpu_ualink_invalidate_import_mappings(bo);
2435 	}
2436 
2437 	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
2438 }
2439 
2440 static void amdgpu_ualink_cleanup_imp_xa_entries(struct amdgpu_device *adev,
2441 						 u32 remote_acc_id)
2442 {
2443 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
2444 	struct list_head *imp_handles_list;
2445 
2446 	dev_dbg(adev->dev,
2447 		"IMP-RESET: Cleaning up all XA entries for remote:%u\n",
2448 		remote_acc_id);
2449 
2450 	imp_handles_list = &adev->ualink.imp_handles_list[remote_acc_id];
2451 
2452 	xa_lock(&adev->ualink.imp_xa);
2453 	while (!list_empty(imp_handles_list)) {
2454 		imp_xa_node = list_first_entry(imp_handles_list,
2455 					struct amdgpu_ualink_imp_xa_node, list);
2456 		list_del_init(&imp_xa_node->list);
2457 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
2458 		xa_unlock(&adev->ualink.imp_xa);
2459 
2460 		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
2461 
2462 		xa_lock(&adev->ualink.imp_xa);
2463 	}
2464 	xa_unlock(&adev->ualink.imp_xa);
2465 }
2466 
2467 static void amdgpu_ualink_cleanup_exp_xa_entries(struct amdgpu_device *adev,
2468 						 u32 remote_acc_id)
2469 {
2470 	struct amdgpu_ualink_importer_entry *importer_entry;
2471 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2472 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
2473 	struct list_head *exp_handles_list;
2474 	struct drm_mm_node *mm_node;
2475 	u64 npa_addr, size;
2476 
2477 	/* Get the list head for the list containing all the handles
2478 	 * exported to this remote GPU.
2479 	 */
2480 	exp_handles_list = &adev->ualink.exp_handles_list[remote_acc_id];
2481 
2482 	dev_dbg(adev->dev,
2483 		"EXP-RESET: Cleaning up all XA entries for remote:%u\n",
2484 		remote_acc_id);
2485 
2486 	xa_lock(&adev->ualink.exp_xa);
2487 	while (!list_empty(exp_handles_list)) {
2488 		importer_entry = list_first_entry(exp_handles_list,
2489 					struct amdgpu_ualink_importer_entry, list);
2490 		list_del_init(&importer_entry->list);
2491 
2492 		exp_xa_node = importer_entry->parent;
2493 		if (!amdgpu_ualink_exp_xa_entry_get(exp_xa_node))
2494 			continue;
2495 
2496 		xa_unlock(&adev->ualink.exp_xa);
2497 		/* Clear the bit corresponding to this remote GPU in
2498 		 * the importer bitmap.
2499 		 */
2500 		if (!test_and_clear_bit(remote_acc_id,
2501 					exp_xa_node->importers_bitmap)) {
2502 			amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
2503 			xa_lock(&adev->ualink.exp_xa);
2504 			continue;
2505 		}
2506 
2507 		mutex_lock(&exp_xa_node->node_lock);
2508 		size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2509 		/* Unpin the BO */
2510 		if (likely(!amdgpu_bo_reserve(exp_xa_node->bo, true))) {
2511 			amdgpu_bo_unpin(exp_xa_node->bo);
2512 			amdgpu_bo_unreserve(exp_xa_node->bo);
2513 		} else {
2514 			dev_warn(adev->dev,
2515 				"EXP-RESET: BO reserve to unpin failed handle:%llx:%llx\n",
2516 				exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo);
2517 		}
2518 
2519 		dev_dbg(adev->dev,
2520 			"EXP-RESET: handle:%llx:%llx pin_count:%d, importers:%d\n",
2521 			exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
2522 			exp_xa_node->bo->tbo.pin_count,
2523 			bitmap_weight(exp_xa_node->importers_bitmap,
2524 			AMDGPU_UALINK_ACCEL_MAX));
2525 
2526 		WARN_ON(exp_xa_node->bo->tbo.pin_count <
2527 			bitmap_weight(exp_xa_node->importers_bitmap,
2528 			AMDGPU_UALINK_ACCEL_MAX));
2529 
2530 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
2531 			npa_addr = importer_entry->npa_addr;
2532 			mm_node = importer_entry->mm_node;
2533 			mutex_unlock(&exp_xa_node->node_lock);
2534 
2535 			dev_dbg(adev->dev,
2536 				"EXP-RESET: Unmap NPA:%llx size: %llx remote:%u handle:%llx:%llx\n",
2537 				npa_addr, size, remote_acc_id,
2538 				exp_xa_node->handle.handle_hi,
2539 				exp_xa_node->handle.handle_lo);
2540 			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
2541 						     npa_addr, size);
2542 			amdgpu_ualink_npa_free_va(adev, mm_node);
2543 			kfree(mm_node);
2544 
2545 			/* Reset the NPA addr and mm_node */
2546 			mutex_lock(&exp_xa_node->node_lock);
2547 			importer_entry->npa_addr = 0;
2548 			importer_entry->mm_node = NULL;
2549 		} else if (bitmap_empty(exp_xa_node->importers_bitmap,
2550 				AMDGPU_UALINK_ACCEL_MAX)) {
2551 			/* In Source-Aliasing mode, if there are no importers
2552 			 * for this handle, then we can unmap and free the
2553 			 * NPA address.
2554 			 */
2555 			mm_node = exp_xa_node->importer_entries[0].mm_node;
2556 			npa_addr = exp_xa_node->importer_entries[0].npa_addr;
2557 			mutex_unlock(&exp_xa_node->node_lock);
2558 
2559 			dev_dbg(adev->dev,
2560 				"EXP-RESET: Unmap NPA:%llx size: %llx handle:%llx:%llx\n",
2561 				npa_addr, size, exp_xa_node->handle.handle_hi,
2562 				exp_xa_node->handle.handle_lo);
2563 			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
2564 						     npa_addr, size);
2565 			amdgpu_ualink_npa_free_va(adev, mm_node);
2566 			kfree(mm_node);
2567 			mutex_lock(&exp_xa_node->node_lock);
2568 			exp_xa_node->importer_entries[0].npa_addr = 0;
2569 			exp_xa_node->importer_entries[0].mm_node = NULL;
2570 		}
2571 
2572 		mutex_unlock(&exp_xa_node->node_lock);
2573 		xa_lock(&adev->ualink.exp_xa);
2574 		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
2575 	}
2576 	xa_unlock(&adev->ualink.exp_xa);
2577 }
2578 
2579 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
2580 						u32 remote_acc_id, u32 state,
2581 						u32 generation_count)
2582 {
2583 	struct amdgpu_ualink_connection *conn_state;
2584 
2585 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2586 
2587 	mutex_lock(&conn_state->lock);
2588 	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
2589 	     (conn_state->generation_count == generation_count)) {
2590 		conn_state->state = state;
2591 		mutex_unlock(&conn_state->lock);
2592 
2593 		amdgpu_ualink_cleanup_imp_xa_entries(adev, remote_acc_id);
2594 		amdgpu_ualink_cleanup_exp_xa_entries(adev, remote_acc_id);
2595 	} else {
2596 		mutex_unlock(&conn_state->lock);
2597 	}
2598 }
2599 
2600 /* Set PTE.X = 1 for all importer entries to retry RPCs. */
2601 static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev,
2602 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2603 {
2604 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2605 	struct amdgpu_ualink_importer_entry *imp_entry;
2606 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2607 	u64 pte_flags, npa_addr, size;
2608 	struct dma_fence *fence = NULL;
2609 	struct amdgpu_bo *bo;
2610 	struct drm_exec exec;
2611 	u32 remote_acc_id;
2612 	int r;
2613 
2614 	bo = exp_xa_node->bo;
2615 	size = amdgpu_bo_ngpu_pages(bo);
2616 
2617 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false);
2618 	if (unlikely(r)) {
2619 		dev_err(adev->dev, "Failed to reserve VM and BO in force_retry_rpcs\n");
2620 		return;
2621 	}
2622 
2623 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2624 			 AMDGPU_UALINK_ACCEL_MAX) {
2625 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2626 			imp_entry = &exp_xa_node->importer_entries[0];
2627 		else
2628 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2629 		npa_addr = imp_entry->npa_addr;
2630 
2631 		/* If the connection state changed while we are freeing
2632 		 * the BO, then ignore this importer. We will unmap this
2633 		 * address eventually in amdgpu_ualink_unmap_all_npa_addr()
2634 		 * function. Also, we will clear the corresponding bit in
2635 		 * the importer_bitmap in the same function.
2636 		 */
2637 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2638 					imp_entry->generation_count))
2639 			continue;
2640 
2641 		/* Set PTE.X = 1 */
2642 		pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo,
2643 					AMDGPU_VM_PAGE_EXECUTABLE);
2644 		dev_dbg(adev->dev,
2645 			"RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n",
2646 			npa_addr, remote_acc_id, pte_flags);
2647 
2648 		r = amdgpu_vm_update_range(adev, vm, false, false, true,
2649 					false, NULL, npa_addr, npa_addr + size - 1,
2650 					pte_flags, 0, adev->vm_manager.vram_base_offset,
2651 					bo->tbo.resource, NULL, &vm->last_update);
2652 
2653 		if (r)
2654 			dev_warn(adev->dev,
2655 				"RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n",
2656 				npa_addr, remote_acc_id, r);
2657 
2658 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2659 			break;
2660 	}
2661 
2662 	r = amdgpu_vm_update_pdes(adev, vm, false);
2663 	if (r) {
2664 		dev_err(adev->dev,
2665 			"Failed %d to update page directories during force retry rpcs\n",
2666 			r);
2667 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2668 		return;
2669 	}
2670 
2671 	fence = dma_fence_get(vm->last_update);
2672 	if (fence) {
2673 		r = dma_fence_wait(fence, false);
2674 		dma_fence_put(fence);
2675 		fence = NULL;
2676 		if (r)
2677 			dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r);
2678 	}
2679 
2680 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2681 
2682 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2683 }
2684 
2685 static void amdgpu_ualink_send_tlb_shootdown(struct amdgpu_device *adev,
2686 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2687 {
2688 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2689 	u32 accel_id = adev->ualink.info->ppod.accel_id;
2690 	struct amdgpu_ualink_importer_entry *imp_entry;
2691 	u64 npa_addr, size;
2692 	u32 remote_acc_id;
2693 	int r;
2694 
2695 	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2696 
2697 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2698 			 AMDGPU_UALINK_ACCEL_MAX) {
2699 		/*
2700 		 * Replace the GPU-id bits with local accel_id
2701 		 */
2702 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2703 			imp_entry = &exp_xa_node->importer_entries[0];
2704 		else
2705 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2706 
2707 		npa_addr = STRIP_NPA(imp_entry->npa_addr);
2708 		npa_addr = GENERATE_NPA(npa_addr, accel_id);
2709 
2710 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2711 					imp_entry->generation_count)) {
2712 			clear_bit(remote_acc_id,
2713 				  exp_xa_node->importers_bitmap);
2714 			continue;
2715 		}
2716 
2717 		dev_dbg(adev->dev,
2718 			"EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n",
2719 			remote_acc_id);
2720 		r = amdgpu_ualink_remote_shootdown(adev, remote_acc_id,
2721 					npa_addr, size,
2722 					AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN);
2723 		if (r) {
2724 			dev_err(adev->dev,
2725 				"EXP-CLEANUP: TLB shootdown send failed to remote:%u\n",
2726 				remote_acc_id);
2727 
2728 			/* Send failed: mark the connection NOT_READY so the
2729 			 * rest of this cleanup (and other cleanups) skip the
2730 			 * unreachable peer via amdgpu_ualink_check_conn_ready().
2731 			 */
2732 			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
2733 						AMDGPU_UALINK_CONN_NOT_READY,
2734 						imp_entry->generation_count);
2735 		}
2736 	}
2737 }
2738 
2739 /* Unmap all NPA addresses associated with a BO (UALink handle). This function is used
2740  * only in Source Identification mode.
2741  */
2742 static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev,
2743 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2744 {
2745 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2746 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2747 	struct amdgpu_ualink_importer_entry *imp_entry;
2748 	u64 pte_value = adev->gmc.noretry_flags;
2749 	struct dma_fence *fence = NULL;
2750 	struct drm_exec exec;
2751 	u64 npa_addr, size;
2752 	u32 remote_acc_id;
2753 	int r;
2754 
2755 	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2756 
2757 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1,
2758 						 &exec, false);
2759 	if (unlikely(r)) {
2760 		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_all_npa_addr\n");
2761 		return;
2762 	}
2763 
2764 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2765 			 AMDGPU_UALINK_ACCEL_MAX) {
2766 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2767 			imp_entry = &exp_xa_node->importer_entries[0];
2768 		else
2769 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2770 		npa_addr = imp_entry->npa_addr;
2771 
2772 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2773 					imp_entry->generation_count)) {
2774 			clear_bit(remote_acc_id,
2775 				  exp_xa_node->importers_bitmap);
2776 			continue;
2777 		}
2778 
2779 		dev_dbg(adev->dev,
2780 			"UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n",
2781 			npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
2782 			remote_acc_id, pte_value);
2783 
2784 		r = amdgpu_vm_update_range(adev, vm, false,
2785 					   false, true, false, NULL, npa_addr,
2786 					   npa_addr + size - 1, pte_value, 0,
2787 					   0, NULL, NULL, &vm->last_update);
2788 
2789 		if (r)
2790 			dev_err(adev->dev,
2791 				"UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n",
2792 				npa_addr, exp_xa_node->handle.handle_hi,
2793 				exp_xa_node->handle.handle_lo, remote_acc_id);
2794 
2795 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2796 			break;
2797 	}
2798 
2799 	r = amdgpu_vm_update_pdes(adev, vm, false);
2800 	if (r) {
2801 		dev_err(adev->dev,
2802 			"Failed %d to update page directories during all NPA addresses unmapping\n",
2803 			r);
2804 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2805 		return;
2806 	}
2807 
2808 	fence = dma_fence_get(vm->last_update);
2809 	if (fence) {
2810 		r = dma_fence_wait(fence, false);
2811 		dma_fence_put(fence);
2812 		fence = NULL;
2813 		if (r)
2814 			dev_err(adev->dev,
2815 				"UNMAP-NPA: dma fence wait failed\n");
2816 	}
2817 
2818 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2819 
2820 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2821 }
2822 
2823 static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev,
2824 				struct amdgpu_ualink_exp_xa_node *exp_xa_node,
2825 				unsigned long *importers_bitmap)
2826 {
2827 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2828 	struct drm_mm_node *mm_node;
2829 	u32 remote_acc_id;
2830 
2831 	for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) {
2832 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2833 			mm_node = exp_xa_node->importer_entries[0].mm_node;
2834 		else
2835 			mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node;
2836 
2837 		if (!mm_node) {
2838 			if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2839 				break;
2840 			continue;
2841 		}
2842 
2843 		dev_dbg(adev->dev,
2844 			"FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n",
2845 			mm_node->start, exp_xa_node->handle.handle_hi,
2846 			exp_xa_node->handle.handle_lo, remote_acc_id);
2847 
2848 		amdgpu_ualink_npa_free_va(adev, mm_node);
2849 		kfree(mm_node);
2850 
2851 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2852 			break;
2853 	}
2854 }
2855 
2856 static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work)
2857 {
2858 	DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
2859 	struct amdgpu_ualink_importer_entry *imp_entry;
2860 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
2861 	struct amdgpu_ualink_handle handle;
2862 	struct amdgpu_device *adev;
2863 	struct amdgpu_bo *bo;
2864 	u32 remote_acc_id;
2865 	int r;
2866 
2867 	exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node,
2868 				   cleanup_work);
2869 	bo = exp_xa_node->bo;
2870 	adev = amdgpu_ttm_adev(bo->tbo.bdev);
2871 	handle = exp_xa_node->handle;
2872 
2873 	/* Revoking access to an exported memory follows the steps:
2874 	 * 1. Set PTE.X = 1 to retry for RPCs.
2875 	 * 2. Send Remote TLB Shootdowns to all importers.
2876 	 * 3. Unmap the NPA address from NPA VM.
2877 	 * 4. Drop the ref count for the BO.
2878 	 * 5. Send NPA_REVOKE to all importers.
2879 	 * 6. Wait for NPA_RELEASE from all importers.
2880 	 * 7. Free the NPA address once all responses are received.
2881 	 */
2882 	/* If there are no importers for this BO/handle */
2883 	if (bitmap_empty(exp_xa_node->importers_bitmap,
2884 			 AMDGPU_UALINK_ACCEL_MAX)) {
2885 		/* Release the dma_buf created at export time (used for the
2886 		 * local-import shortcut). The has-importers path below drops
2887 		 * it too; this branch must not skip it or the dma_buf and the
2888 		 * BO it pins are leaked.
2889 		 */
2890 		dma_buf_put(exp_xa_node->dmabuf);
2891 		/* Drop the BO reference so it can be freed. */
2892 		amdgpu_bo_unref(&bo);
2893 		exp_xa_node->bo = NULL;
2894 		goto free_node;
2895 	}
2896 
2897 	dev_dbg(adev->dev,
2898 		"EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n",
2899 		handle.handle_hi, handle.handle_lo,
2900 		AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap);
2901 
2902 	bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap,
2903 		    AMDGPU_UALINK_ACCEL_MAX);
2904 
2905 	/* Set PTE.X = 1 for NPA addresses from all importers*/
2906 	amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node);
2907 
2908 	/* Send TLB-shootdown to all importer GPUs */
2909 	amdgpu_ualink_send_tlb_shootdown(adev, exp_xa_node);
2910 
2911 	/* Unmap all NPA addresses for this BO from NPA VM */
2912 	amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
2913 
2914 	dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n",
2915 		handle.handle_hi, handle.handle_lo, bo->tbo.pin_count,
2916 		bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX));
2917 	WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap,
2918 						AMDGPU_UALINK_ACCEL_MAX));
2919 
2920 	/* Unpin the BO */
2921 	if (likely(!amdgpu_bo_reserve(bo, true))) {
2922 		bo->ualink_handle_lo = 0ULL;
2923 		for_each_set_bit(remote_acc_id, orig_importers_bitmap,
2924 				 AMDGPU_UALINK_ACCEL_MAX)
2925 			amdgpu_bo_unpin(bo);
2926 		amdgpu_bo_unreserve(bo);
2927 	} else {
2928 		dev_warn(adev->dev,
2929 			"EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n",
2930 			handle.handle_hi, handle.handle_lo);
2931 	}
2932 
2933 	/* Free the DMABuf */
2934 	dma_buf_put(exp_xa_node->dmabuf);
2935 	/* Drop the reference to the BO so it can be freed. */
2936 	amdgpu_bo_unref(&bo);
2937 	exp_xa_node->bo = NULL;
2938 
2939 	/* Seed npa_release_bitmap with all importers before any NPA-REVOKE, so
2940 	 * the release IRQ only sees it empty once every responder has replied.
2941 	 * Dead peers are dropped from it in the send loop below.
2942 	 */
2943 	bitmap_copy(exp_xa_node->npa_release_bitmap,
2944 		    exp_xa_node->importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
2945 
2946 	/* Send NPA-REVOKE to all importers which have imported this memory.
2947 	 * On send failure clear the bit (no response will arrive) and mark the
2948 	 * connection NOT_READY so other cleanups skip this unreachable peer.
2949 	 */
2950 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2951 				 AMDGPU_UALINK_ACCEL_MAX) {
2952 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2953 
2954 		/* Re-check the connection right before the send: concurrent
2955 		 * cleanup workers all seed the bitmap while the peer is still
2956 		 * ESTABLISHED, so once its egress dies and the first worker
2957 		 * marks it NOT_READY, this skips the rest instead of flooding
2958 		 * the log with failed LSDMA sends to the same dead peer.
2959 		 */
2960 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2961 					imp_entry->generation_count)) {
2962 			clear_bit(remote_acc_id,
2963 				  exp_xa_node->npa_release_bitmap);
2964 			continue;
2965 		}
2966 
2967 		dev_dbg(adev->dev,
2968 			"EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n",
2969 			remote_acc_id);
2970 		r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle);
2971 		if (r) {
2972 			dev_err(adev->dev,
2973 				"EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n",
2974 				remote_acc_id);
2975 			clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
2976 
2977 			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
2978 						AMDGPU_UALINK_CONN_NOT_READY,
2979 						imp_entry->generation_count);
2980 		}
2981 	}
2982 
2983 	/* Wait for the NPA_RELEASE to come back from all importers */
2984 	if (!bitmap_empty(exp_xa_node->npa_release_bitmap,
2985 			AMDGPU_UALINK_ACCEL_MAX)) {
2986 		dev_dbg(adev->dev,
2987 			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n",
2988 			handle.handle_hi, handle.handle_lo,
2989 			AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap);
2990 
2991 		/* Wait for the NPA_RELEASE to come back from all importers */
2992 		r = wait_for_completion_timeout(&exp_xa_node->npa_done,
2993 					msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
2994 
2995 		if (r == 0)
2996 			dev_warn(adev->dev,
2997 				"EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n",
2998 				handle.handle_hi, handle.handle_lo);
2999 	}
3000 
3001 	/* Free the NPA addresses given to all the importers */
3002 	amdgpu_ualink_free_all_npa_va(adev, exp_xa_node,
3003 				      orig_importers_bitmap);
3004 
3005 	/* Warn about all importers that didn't respond back with
3006 	 * NPA-RELEASE message and trigger connection timeout handling.
3007 	 */
3008 	for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap,
3009 			 AMDGPU_UALINK_ACCEL_MAX) {
3010 		dev_warn(adev->dev,
3011 			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n",
3012 			handle.handle_hi, handle.handle_lo, remote_acc_id);
3013 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
3014 		amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
3015 						AMDGPU_UALINK_CONN_NOT_READY,
3016 						imp_entry->generation_count);
3017 	}
3018 
3019 free_node:
3020 	xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo);
3021 	mutex_destroy(&exp_xa_node->node_lock);
3022 	kfree(exp_xa_node);
3023 }
3024 
3025 /* This function is a copy of amdgpu_dma_buf_move_notify() function.
3026  * amdgpu_dma_buf_move_notify is only called for import attachments.
3027  * But NPA DMABufs don't use attachments because they are imported
3028  * on the same device. So we need to invalidate the GPUVM mappings
3029  * manually.
3030  */
3031 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo)
3032 {
3033 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
3034 	struct ttm_operation_ctx ctx = { false, false };
3035 	struct ttm_placement placement = {};
3036 	struct amdgpu_vm_bo_base *bo_base;
3037 	struct drm_exec exec;
3038 	int r;
3039 
3040 	/*
3041 	 * Lock the BO together with every client VM page directory it is
3042 	 * mapped into in a single drm_exec transaction.
3043 	 */
3044 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
3045 	drm_exec_until_all_locked(&exec) {
3046 		r = drm_exec_lock_obj(&exec, &bo->tbo.base);
3047 		drm_exec_retry_on_contention(&exec);
3048 		if (unlikely(r))
3049 			goto fini;
3050 
3051 		for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
3052 			r = amdgpu_vm_lock_pd(bo_base->vm, &exec, 0);
3053 			drm_exec_retry_on_contention(&exec);
3054 			if (unlikely(r))
3055 				goto fini;
3056 		}
3057 	}
3058 
3059 	amdgpu_vm_bo_move(bo, NULL, false);
3060 
3061 	r = ttm_bo_validate(&bo->tbo, &placement, &ctx);
3062 	if (r) {
3063 		dev_err(adev->dev, "Failed to invalidate NPA DMA-buf import (%d)\n",
3064 			r);
3065 		goto fini;
3066 	}
3067 
3068 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
3069 		struct amdgpu_vm *vm = bo_base->vm;
3070 
3071 		/*
3072 		 * Fences for the two SDMA page table updates were already
3073 		 * reserved by amdgpu_vm_lock_pd() above (it reserves 2 +
3074 		 * num_fences on the same VM root PD dma_resv).
3075 		 */
3076 		r = amdgpu_vm_clear_freed(adev, vm, NULL);
3077 		if (!r)
3078 			r = amdgpu_vm_handle_moved(adev, vm, &exec.ticket);
3079 
3080 		if (r && r != -EBUSY)
3081 			dev_err(adev->dev, "Failed to invalidate VM page tables (%d))\n",
3082 				r);
3083 	}
3084 
3085 fini:
3086 	drm_exec_fini(&exec);
3087 }
3088 
3089 static int amdgpu_ualink_map_npa_to_dmabuf(struct amdgpu_device *adev,
3090 				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
3091 {
3092 	struct ttm_operation_ctx ctx = { false, false };
3093 	u32 initial_domain = AMDGPU_GEM_DOMAIN_NPA;
3094 	struct drm_gem_object *gobj = NULL;
3095 	u64 alloc_flags, npa_addr, size;
3096 	struct dma_buf *dmabuf;
3097 	struct amdgpu_bo *bo;
3098 	u32 handle;
3099 	int r;
3100 
3101 	npa_addr = imp_xa_node->npa_addr;
3102 	size = imp_xa_node->size;
3103 
3104 	dev_dbg(adev->dev, "Create NPA BO addr 0x%llx size in pages 0x%llx\n",
3105 		npa_addr, size);
3106 
3107 	/*
3108 	 * Create the BO directly in the NPA domain.
3109 	 */
3110 	alloc_flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
3111 	r = amdgpu_gem_object_create(adev, size * AMDGPU_GPU_PAGE_SIZE, 1,
3112 				     initial_domain, alloc_flags,
3113 				     ttm_bo_type_device, NULL, &gobj, 0);
3114 	if (r) {
3115 		dev_err(adev->dev, "Failed to create NPA BO. ret %d\n", r);
3116 		return r;
3117 	}
3118 
3119 	bo = gem_to_amdgpu_bo(gobj);
3120 
3121 	r = amdgpu_bo_reserve(bo, false);
3122 	if (unlikely(r != 0)) {
3123 		dev_err(adev->dev, "Failed to reserve NPA BO, r: %d\n", r);
3124 		goto err_reserve_failed;
3125 	}
3126 
3127 	/*
3128 	 * Drop the arbitrarily-placed NPA node and re-create it at the exact
3129 	 * remote window, mirroring amdgpu_bo_create_kernel_at().
3130 	 */
3131 	ttm_resource_free(&bo->tbo, &bo->tbo.resource);
3132 
3133 	bo->placements[0].fpfn = npa_addr;
3134 	bo->placements[0].lpfn = npa_addr + size;
3135 
3136 	r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.resource, &ctx);
3137 	amdgpu_bo_unreserve(bo);
3138 	if (r) {
3139 		dev_err(adev->dev,
3140 			"Failed to place NPA BO at 0x%llx, r: %d\n", npa_addr, r);
3141 		goto err_validate_failed;
3142 	}
3143 
3144 	r = drm_gem_handle_create(adev->ualink.client.file, gobj, &handle);
3145 	if (r) {
3146 		dev_err(adev->dev,
3147 			"Failed to get handle for NPA GEM object, r: %d\n", r);
3148 		goto err_validate_failed;
3149 	}
3150 	drm_gem_object_put(gobj);
3151 
3152 	dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, adev->ualink.client.file,
3153 						handle, DRM_CLOEXEC | DRM_RDWR);
3154 	if (IS_ERR(dmabuf)) {
3155 		r = PTR_ERR(dmabuf);
3156 		dev_err(adev->dev,
3157 			"Failed to generate DMABuf for NPA GEM object\n");
3158 		goto err_dmabuf_failed;
3159 	}
3160 
3161 	imp_xa_node->dmabuf = dmabuf;
3162 	imp_xa_node->gem_handle = handle;
3163 
3164 	return 0;
3165 
3166 err_dmabuf_failed:
3167 	drm_gem_handle_delete(adev->ualink.client.file, handle);
3168 	return r;
3169 err_validate_failed:
3170 err_reserve_failed:
3171 	drm_gem_object_put(gobj);
3172 
3173 	return r;
3174 }
3175 
3176 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo)
3177 {
3178 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
3179 	struct amdgpu_ualink_importer_entry *imp_entry;
3180 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3181 	u32 remote_acc_id;
3182 	int r;
3183 
3184 	if (!bo->ualink_handle_lo)
3185 		return;
3186 
3187 	/* Remove the entry from the Xarray. */
3188 	xa_lock(&adev->ualink.exp_xa);
3189 	exp_xa_node = __xa_erase(&adev->ualink.exp_xa,
3190 				 bo->ualink_handle_lo);
3191 	if (!exp_xa_node) {
3192 		xa_unlock(&adev->ualink.exp_xa);
3193 		dev_warn(adev->dev,
3194 			 "Exp XA: handle_lo:%llx not found\n",
3195 			 bo->ualink_handle_lo);
3196 		return;
3197 	}
3198 
3199 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
3200 			 AMDGPU_UALINK_ACCEL_MAX) {
3201 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
3202 		list_del_init(&imp_entry->list);
3203 	}
3204 	xa_unlock(&adev->ualink.exp_xa);
3205 
3206 	/* Add it to the Handle_Invalid xarray */
3207 	r = xa_err(xa_store(&adev->ualink.handle_invalid_xa,
3208 			    bo->ualink_handle_lo,
3209 			    exp_xa_node, GFP_KERNEL));
3210 	if (r)
3211 		dev_err(adev->dev,
3212 			"Handle_Invalid XA store failed handle:%llx:%llx error:%d\n",
3213 			exp_xa_node->handle.handle_hi,
3214 			exp_xa_node->handle.handle_lo, r);
3215 
3216 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3217 }
3218 
3219 static void amdgpu_ualink_process_npa_release_msg(struct amdgpu_device *adev,
3220 						 u32 remote_acc_id,
3221 						 struct amdgpu_ualink_handle handle)
3222 {
3223 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3224 
3225 	xa_lock(&adev->ualink.handle_invalid_xa);
3226 	exp_xa_node = xa_load(&adev->ualink.handle_invalid_xa, handle.handle_lo);
3227 	if (!exp_xa_node) {
3228 		dev_warn(adev->dev,
3229 			 "NPA-RELEASE: Handle (%llx:%llx) not found\n",
3230 			 handle.handle_hi, handle.handle_lo);
3231 		goto out;
3232 	}
3233 
3234 	/* Confirm that the complete handle matches. */
3235 	if (handle.handle_hi != exp_xa_node->handle.handle_hi) {
3236 		dev_warn(adev->dev,
3237 			 "NPA-RELEASE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
3238 			 exp_xa_node->handle.handle_hi,
3239 			 handle.handle_hi, handle.handle_lo, remote_acc_id);
3240 		goto out;
3241 	}
3242 
3243 	/* Test and clear the bit corresponding to the remote GPU id to signal
3244 	 * the arrival of NPA_RELEASE message from it.
3245 	 * If the corresponding bit wasn't set, then raise a warning and ignore
3246 	 * the NPA-Release message from the remote GPU.
3247 	 */
3248 	if (!test_and_clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap)) {
3249 		dev_warn(adev->dev,
3250 			 "NPA-RELEASE: unexpected from remote:%u handle:%llx:%llx\n",
3251 			 remote_acc_id, handle.handle_hi, handle.handle_lo);
3252 		goto out;
3253 	}
3254 
3255 	/* Signal completion if NPA_Release received from all importers */
3256 	if (bitmap_empty(exp_xa_node->npa_release_bitmap,
3257 			 AMDGPU_UALINK_ACCEL_MAX))
3258 		complete(&exp_xa_node->npa_done);
3259 
3260 out:
3261 	xa_unlock(&adev->ualink.handle_invalid_xa);
3262 }
3263 
3264 static void amdgpu_ualink_process_npa_revoke_msg(struct amdgpu_device *adev,
3265 						u32 remote_acc_id,
3266 						struct amdgpu_ualink_handle handle)
3267 {
3268 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3269 	struct amdgpu_bo *bo;
3270 	int r = 0;
3271 
3272 	/* Remove the entry from the Xarray. */
3273 	xa_lock(&adev->ualink.imp_xa);
3274 	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
3275 	if (!imp_xa_node) {
3276 		xa_unlock(&adev->ualink.imp_xa);
3277 		dev_warn(adev->dev,
3278 			 "NPA-REVOKE: Handle (%llx:%llx) not found\n",
3279 			 handle.handle_hi, handle.handle_lo);
3280 		return;
3281 	}
3282 
3283 	/* Confirm that the complete handle matches. */
3284 	if (handle.handle_hi != imp_xa_node->handle.handle_hi) {
3285 		xa_unlock(&adev->ualink.imp_xa);
3286 		dev_warn(adev->dev,
3287 			 "NPA-REVOKE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
3288 			 imp_xa_node->handle.handle_hi,
3289 			 handle.handle_hi, handle.handle_lo, remote_acc_id);
3290 		return;
3291 	}
3292 
3293 	switch (READ_ONCE(imp_xa_node->node_state)) {
3294 	case AMDGPU_UALINK_NODE_READY:
3295 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
3296 		list_del_init(&imp_xa_node->list);
3297 		xa_unlock(&adev->ualink.imp_xa);
3298 
3299 		/* Invalidate the GPUVM mappings */
3300 		bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
3301 		amdgpu_ualink_invalidate_import_mappings(bo);
3302 
3303 		/* Drop the refcount for the node */
3304 		amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
3305 		break;
3306 	case AMDGPU_UALINK_NODE_PENDING:
3307 		/* The import is still building the dma-buf and nothing has
3308 		 * been handed to user-space yet. The importing thread sees
3309 		 * the teardown state and unwinds.
3310 		 */
3311 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
3312 		xa_unlock(&adev->ualink.imp_xa);
3313 		break;
3314 	default:
3315 		/* NPA-REVOKE always follows NPA-RSP, so a NOT_READY node means
3316 		 * a stale revoke, and a node in teardown is already being
3317 		 * released by whoever moved it there.
3318 		 */
3319 		xa_unlock(&adev->ualink.imp_xa);
3320 		return;
3321 	}
3322 
3323 	r = amdgpu_ualink_send_npa_release_msg(adev, remote_acc_id, handle);
3324 	if (r)
3325 		dev_err(adev->dev,
3326 			"NPA-Release send failed remote:%u handle:%llx:%llx error:%d\n",
3327 			remote_acc_id, handle.handle_hi, handle.handle_lo, r);
3328 }
3329 
3330 static void amdgpu_ualink_process_npa_fail_msg(struct amdgpu_device *adev,
3331 				       u32 remote_acc_id, u64 partial_handle,
3332 				       u32 fail_reason)
3333 {
3334 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3335 	int r = 0;
3336 
3337 	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
3338 		dev_warn(adev->dev,
3339 			"NPA-FAIL: no connection with remote AccId:%u\n",
3340 			remote_acc_id);
3341 		goto conn_setup;
3342 	}
3343 
3344 	xa_lock(&adev->ualink.imp_xa);
3345 	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
3346 	if (!imp_xa_node) {
3347 		xa_unlock(&adev->ualink.imp_xa);
3348 		dev_warn(adev->dev,
3349 			"NPA-FAIL: imp XA handle not found:%llx\n",
3350 			partial_handle);
3351 		return;
3352 	}
3353 
3354 	imp_xa_node->fail_reason = fail_reason;
3355 	/* Signal completion done to signal response received for NPA-REQ
3356 	 * message.
3357 	 * If the node is in NOT_READY state, then set the node state to
3358 	 * PENDING and signal the completion. If the node is not in NOT_READY
3359 	 * state, then it is an unsolicited NPA-FAIL message and we
3360 	 * log a debug message.
3361 	 */
3362 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
3363 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
3364 		complete(&imp_xa_node->npa_done);
3365 	} else {
3366 		dev_dbg(adev->dev,
3367 			"NPA-FAIL: unsolicited for handle:%llx:%llx from AccId:%u\n",
3368 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
3369 			remote_acc_id);
3370 	}
3371 	xa_unlock(&adev->ualink.imp_xa);
3372 
3373 	return;
3374 
3375 conn_setup:
3376 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3377 	if (r)
3378 		dev_warn(adev->dev,
3379 			"NPA-FAIL: connection setup failed with remote AccId:%u\n",
3380 			remote_acc_id);
3381 }
3382 
3383 static void amdgpu_ualink_process_npa_rsp_msg(struct amdgpu_device *adev,
3384 				      u32 remote_acc_id, u64 partial_handle,
3385 				      u64 npa_addr, u64 size)
3386 {
3387 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3388 	int r = 0;
3389 
3390 	/* Check if the connection is established. If it is not, then start
3391 	 * connection setup.
3392 	 */
3393 	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
3394 		dev_warn(adev->dev,
3395 			"NPA-RSP: no connection with remote AccId:%u\n",
3396 			remote_acc_id);
3397 		goto conn_setup;
3398 	}
3399 
3400 	xa_lock(&adev->ualink.imp_xa);
3401 	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
3402 	if (!imp_xa_node) {
3403 		xa_unlock(&adev->ualink.imp_xa);
3404 		dev_warn(adev->dev,
3405 			"NPA-RSP: imp XA handle not found:%llx\n", partial_handle);
3406 		return;
3407 	}
3408 
3409 	/* NPA addr received in NPA-RSP is page aligned and without the remote
3410 	 * GPU-id in Bits 41-50. Assemble back the NPA address before storing
3411 	 * it.
3412 	 */
3413 	imp_xa_node->npa_addr = GENERATE_NPA(npa_addr, remote_acc_id);
3414 	/* Size is in number of GPU pages granularity. */
3415 	imp_xa_node->size = size;
3416 
3417 	/* Signal completion done to signal NPA_RSP received.
3418 	 * If the node is in NOT_READY state, then set the node state to
3419 	 * PENDING and signal the completion. If the node is not in NOT_READY
3420 	 * state, then it is an unsolicited NPA-RSP message and we
3421 	 * log a debug message.
3422 	 */
3423 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
3424 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
3425 		complete(&imp_xa_node->npa_done);
3426 	} else {
3427 		dev_dbg(adev->dev,
3428 			"NPA-RSP: unsolicited for handle:%llx:%llx from AccId:%u\n",
3429 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
3430 			remote_acc_id);
3431 	}
3432 	xa_unlock(&adev->ualink.imp_xa);
3433 
3434 	return;
3435 
3436 conn_setup:
3437 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3438 	if (r)
3439 		dev_warn(adev->dev,
3440 			"NPA-RSP: connection setup failed with remote AccId:%u\n",
3441 			remote_acc_id);
3442 }
3443 
3444 static void amdgpu_ualink_process_npa_req_msg(struct amdgpu_device *adev,
3445 				      u32 remote_acc_id,
3446 				      struct amdgpu_ualink_handle handle)
3447 {
3448 	struct amdgpu_ualink_importer_entry *importer_entry, *npa_addr_entry;
3449 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
3450 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3451 	u64 range_start, range_end, pte_flags;
3452 	struct drm_mm_node *mm_node = NULL;
3453 	int r = 0, fail_reason = 0;
3454 	bool send_npa_fail = true;
3455 	u64 npa_addr = 0, size;
3456 	struct amdgpu_bo *bo;
3457 	u32 gen_count;
3458 
3459 	/* Check if the connection is established. If it is not, then start
3460 	 * connection setup.
3461 	 */
3462 	gen_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
3463 	if (!gen_count) {
3464 		dev_warn(adev->dev,
3465 			"NPA-REQ: no connection with remote AccId:%u\n",
3466 			remote_acc_id);
3467 		goto conn_setup;
3468 	}
3469 
3470 	/* Check entry exists in Exporter XA. If yes, increase the refcount
3471 	 * for the node.
3472 	 */
3473 	xa_lock(&adev->ualink.exp_xa);
3474 	exp_xa_node = xa_load(&adev->ualink.exp_xa, handle.handle_lo);
3475 	if (!exp_xa_node  || (handle.handle_hi != exp_xa_node->handle.handle_hi) ||
3476 	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
3477 		xa_unlock(&adev->ualink.exp_xa);
3478 		dev_warn(adev->dev,
3479 			"NPA-REQ: exp XA handle not found handle:%llx:%llx\n",
3480 			handle.handle_hi, handle.handle_lo);
3481 		fail_reason = AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE;
3482 		goto handle_invalid_fail;
3483 	}
3484 	xa_unlock(&adev->ualink.exp_xa);
3485 
3486 	bo = exp_xa_node->bo;
3487 	size = amdgpu_bo_ngpu_pages(bo);
3488 
3489 	/* Pin the BO */
3490 	r = amdgpu_bo_reserve(bo, true);
3491 	if (unlikely(r)) {
3492 		dev_warn(adev->dev,
3493 			"NPA-REQ: BO reserve failed handle:%llx:%llx\n",
3494 			handle.handle_hi, handle.handle_lo);
3495 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3496 		goto bo_reserve_fail;
3497 	}
3498 	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
3499 	amdgpu_bo_unreserve(bo);
3500 	if (r) {
3501 		dev_warn(adev->dev,
3502 			"NPA-REQ: BO pin failed handle:%llx:%llx\n",
3503 			handle.handle_hi, handle.handle_lo);
3504 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3505 		goto bo_pin_fail;
3506 	}
3507 
3508 	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
3509 		mutex_lock(&exp_xa_node->node_lock);
3510 		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
3511 		npa_addr_entry = importer_entry;
3512 		mutex_unlock(&exp_xa_node->node_lock);
3513 		/* Check if NPA address is already allocated for this importer.
3514 		 * If yes, then send the NPA-FAIL message back to the remote GPU.
3515 		 */
3516 		if (importer_entry->npa_addr) {
3517 			fail_reason = AMDGPU_UALINK_NPA_FAIL_DUPLICATE;
3518 			goto npa_duplicate_fail;
3519 		}
3520 
3521 		range_start = ((u64)remote_acc_id << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT) |
3522 				AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED;
3523 		range_end = range_start | AMDGPU_UALINK_NPA_ADDR_RANGE_MASK;
3524 	} else {
3525 		/* We store NPA-address in importer_entries[0] in
3526 		 * Source-Aliasing mode.
3527 		 */
3528 		mutex_lock(&exp_xa_node->node_lock);
3529 		npa_addr_entry = &exp_xa_node->importer_entries[0];
3530 		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
3531 		/* Check if NPA address is already allocated for this importer.
3532 		 * If yes, then set the corresponding bit in the importers_bitmap,
3533 		 * set the generation count and send the NPA-RSP back to the remote GPU.
3534 		 */
3535 		if (npa_addr_entry->npa_addr) {
3536 			npa_addr = npa_addr_entry->npa_addr;
3537 			set_bit(remote_acc_id, exp_xa_node->importers_bitmap);
3538 			importer_entry->generation_count = gen_count;
3539 			mutex_unlock(&exp_xa_node->node_lock);
3540 			dev_dbg(adev->dev,
3541 				"NPA-REQ: NPA:%llx size:%llx handle:%llx:%llx\n",
3542 				npa_addr, size, handle.handle_hi, handle.handle_lo);
3543 
3544 			goto send_npa_rsp;
3545 		}
3546 		mutex_unlock(&exp_xa_node->node_lock);
3547 
3548 		range_start = 0;
3549 		range_end = 0;
3550 	}
3551 
3552 	mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL);
3553 	if (!mm_node) {
3554 		dev_warn(adev->dev,
3555 			"NPA-REQ: mm_node alloc failed handle:%llx:%llx\n",
3556 			handle.handle_hi, handle.handle_lo);
3557 		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
3558 		goto mem_alloc_fail;
3559 	}
3560 
3561 	/* Allocate NPA address */
3562 	r = amdgpu_ualink_npa_alloc_va(adev, mm_node, 0, range_start,
3563 					range_end, size);
3564 	if (r) {
3565 		dev_warn(adev->dev,
3566 			"NPA-REQ: NPA addr alloc failed handle:%llx:%llx\n",
3567 			handle.handle_hi, handle.handle_lo);
3568 		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
3569 		goto npa_alloc_fail;
3570 	}
3571 	npa_addr = mm_node->start;
3572 
3573 	pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 0);
3574 	dev_dbg(adev->dev,
3575 		"NPA-REQ: Allocated NPA:%llx size:%llx PTE:%llx handle:%llx:%llx\n",
3576 		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
3577 
3578 	/* Map the NPA address into NPA VM*/
3579 	r = amdgpu_ualink_map_npa_addr(adev, npa_addr, size, bo, 0, pte_flags);
3580 	if (r) {
3581 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3582 		dev_warn(adev->dev,
3583 			"NPA-REQ: NPA addr (%llx) map failed handle:%llx:%llx\n",
3584 			npa_addr, handle.handle_hi, handle.handle_lo);
3585 		goto map_npa_fail;
3586 	}
3587 
3588 	dev_dbg(adev->dev,
3589 		"NPA-REQ: Mapped NPA:%llx size:%llx pte:%llx handle:%llx:%llx\n",
3590 		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
3591 send_npa_rsp:
3592 	/* Send NPA-RSP back to the remote GPU */
3593 	r = amdgpu_ualink_send_npa_rsp_msg(adev, remote_acc_id, handle,
3594 					   STRIP_NPA(npa_addr), size);
3595 	if (r) {
3596 		dev_warn(adev->dev,
3597 			"NPA-REQ: send NPA-RSP failed remote:%u handle:%llx:%llx\n",
3598 			remote_acc_id, handle.handle_hi, handle.handle_lo);
3599 		send_npa_fail = false;
3600 		goto send_npa_rsp_fail;
3601 	}
3602 
3603 	dev_dbg(adev->dev,
3604 		"NPA-REQ: Sent NPA-RSP with NPA:%llx size:%llx handle:%llx:%llx\n",
3605 		npa_addr, size, handle.handle_hi, handle.handle_lo);
3606 
3607 	/* If this is the first time we are setting the bit for this importer,
3608 	 * then store the NPA address, mm_node and generation count.
3609 	 */
3610 	mutex_lock(&exp_xa_node->node_lock);
3611 	if (!test_and_set_bit(remote_acc_id, exp_xa_node->importers_bitmap)) {
3612 		npa_addr_entry->npa_addr = npa_addr;
3613 		npa_addr_entry->mm_node = mm_node;
3614 		importer_entry->generation_count = gen_count;
3615 	}
3616 	mutex_unlock(&exp_xa_node->node_lock);
3617 
3618 	dev_dbg(adev->dev,
3619 		"NPA-REQ: BO pin_count:%d, importers:%d, handle:%llx:%llx\n",
3620 		bo->tbo.pin_count, bitmap_weight(exp_xa_node->importers_bitmap,
3621 		AMDGPU_UALINK_ACCEL_MAX), handle.handle_hi, handle.handle_lo);
3622 	WARN_ON(bo->tbo.pin_count < bitmap_weight(exp_xa_node->importers_bitmap,
3623 						   AMDGPU_UALINK_ACCEL_MAX));
3624 
3625 	/* Add this node to the exported handles list for the remote GPU,
3626 	 * but only if the node is still in exp_xa. If revoke already erased
3627 	 * it, skip the list_add to avoid a dangling list entry. The cleanup
3628 	 * worker is guaranteed to run after we drop our ref, so it will see
3629 	 * this importer in the bitmap and send NPA-REVOKE.
3630 	 */
3631 	xa_lock(&adev->ualink.exp_xa);
3632 	if (xa_load(&adev->ualink.exp_xa, exp_xa_node->handle.handle_lo) == exp_xa_node)
3633 		list_add(&importer_entry->list, &adev->ualink.exp_handles_list[remote_acc_id]);
3634 	xa_unlock(&adev->ualink.exp_xa);
3635 
3636 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3637 
3638 	return;
3639 
3640 send_npa_rsp_fail:
3641 	mutex_lock(&exp_xa_node->node_lock);
3642 	clear_bit(remote_acc_id, exp_xa_node->importers_bitmap);
3643 	mutex_unlock(&exp_xa_node->node_lock);
3644 	if (mm_node)
3645 		amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
3646 
3647 map_npa_fail:
3648 	if (mm_node)
3649 		amdgpu_ualink_npa_free_va(adev, mm_node);
3650 
3651 npa_alloc_fail:
3652 	kfree(mm_node);
3653 mem_alloc_fail:
3654 npa_duplicate_fail:
3655 	r = amdgpu_bo_reserve(bo, true);
3656 	if (likely(!r)) {
3657 		amdgpu_bo_unpin(bo);
3658 		amdgpu_bo_unreserve(bo);
3659 	} else {
3660 		dev_warn(adev->dev,
3661 			"NPA-REQ: BO reserve to unpin failed for handle:%llx:%llx\n",
3662 			handle.handle_hi, handle.handle_lo);
3663 	}
3664 
3665 bo_pin_fail:
3666 bo_reserve_fail:
3667 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3668 
3669 handle_invalid_fail:
3670 	if (send_npa_fail) {
3671 		r = amdgpu_ualink_send_npa_fail_msg(adev, remote_acc_id,
3672 						    handle, fail_reason);
3673 		if (r)
3674 			dev_warn(adev->dev,
3675 				"NPA-REQ: send NPA-FAIL failed remote:%u handle:%llx:%llx\n",
3676 				remote_acc_id, handle.handle_hi, handle.handle_lo);
3677 	}
3678 	return;
3679 conn_setup:
3680 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3681 	if (r)
3682 		dev_warn(adev->dev,
3683 			"NPA-REQ: connection setup failed with remote AccId:%u\n",
3684 			remote_acc_id);
3685 }
3686 
3687 static int amdgpu_ualink_translate_npa_fail_reason(struct amdgpu_device *adev,
3688 						   u32 fail_reason)
3689 {
3690 	switch (fail_reason) {
3691 	case AMDGPU_UALINK_NPA_FAIL_NOSPACE:
3692 		return -ENOSPC;
3693 	case AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE:
3694 	case AMDGPU_UALINK_NPA_FAIL_DUPLICATE:
3695 	case AMDGPU_UALINK_NPA_FAIL_ERROR:
3696 		return -EINVAL;
3697 	default:
3698 		dev_err(adev->dev,
3699 			"IMPORT: invalid NPA-FAIL reason:%u\n",
3700 			fail_reason);
3701 		return -EINVAL;
3702 	}
3703 }
3704 
3705 static int amdgpu_ualink_do_import_handle(struct amdgpu_device *adev,
3706 				struct amdgpu_ualink_imp_xa_node *imp_xa_node,
3707 				u32 remote_acc_id)
3708 {
3709 	struct amdgpu_ualink_handle handle = imp_xa_node->handle;
3710 	u32 generation_count;
3711 	int r;
3712 
3713 	/* First check if the connection is setup with the
3714 	 * remote GPU. If yes, then initiate the NPA protocol to
3715 	 * get the NPA address.
3716 	 * If not, then initiate the HELLO protocol to first setup
3717 	 * the connection and once the connection is setup, then
3718 	 * initiate the NPA protocol.
3719 	 */
3720 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3721 	if (r) {
3722 		if (r != -EAGAIN)
3723 			dev_warn(adev->dev,
3724 			"IMPORT: connection setup failed with remote AccId:%u\n",
3725 			remote_acc_id);
3726 		return r;
3727 	}
3728 
3729 	/* Send NPA_REQ message */
3730 	r = amdgpu_ualink_send_npa_req_msg(adev, remote_acc_id, handle);
3731 	if (r) {
3732 		dev_warn(adev->dev,
3733 			"IMPORT: NPA-REQ send failed to remote AccId:%u\n",
3734 			remote_acc_id);
3735 		goto reset_conn;
3736 	}
3737 
3738 	/* Wait for the NPA_RSP to come back */
3739 	r = wait_for_completion_interruptible_timeout(&imp_xa_node->npa_done,
3740 				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
3741 	if (r == -ERESTARTSYS) {
3742 		dev_err_ratelimited(adev->dev,
3743 			"IMPORT: NPA-RSP wait interrupted by signal\n");
3744 		return r;
3745 	} else if (r == 0) {
3746 		dev_warn(adev->dev,
3747 			"IMPORT: NPA-RSP timeout from remote AccId:%u\n",
3748 			remote_acc_id);
3749 		r = -ETIMEDOUT;
3750 		goto reset_conn;
3751 	}
3752 
3753 	/* If the NPA addr/size isn't filled with valid values, then either
3754 	 * we got a NPA_FAIL or something bad happened. In either case, we
3755 	 * return the error back to user-space.
3756 	 */
3757 	if (imp_xa_node->fail_reason) {
3758 		dev_warn(adev->dev,
3759 			"IMPORT: NPA-REQ failed with fail_reason:%d handle:%llx:%llx\n",
3760 			imp_xa_node->fail_reason, handle.handle_hi, handle.handle_lo);
3761 		return amdgpu_ualink_translate_npa_fail_reason(adev,
3762 						imp_xa_node->fail_reason);
3763 	}
3764 
3765 	if (!imp_xa_node->npa_addr || !imp_xa_node->size) {
3766 		dev_warn(adev->dev,
3767 			"IMPORT: invalid npa:%llx or size:%llx\n",
3768 			imp_xa_node->npa_addr, imp_xa_node->size);
3769 		imp_xa_node->npa_addr = 0;
3770 		imp_xa_node->size = 0;
3771 		return -EINVAL;
3772 	}
3773 
3774 	r = amdgpu_ualink_map_npa_to_dmabuf(adev, imp_xa_node);
3775 	if (r) {
3776 		dev_warn(adev->dev,
3777 			"IMPORT: dmabuf creation failed npa:%llx size:%llx\n",
3778 			imp_xa_node->npa_addr, imp_xa_node->size);
3779 		imp_xa_node->npa_addr = 0;
3780 		imp_xa_node->size = 0;
3781 		return r;
3782 	}
3783 
3784 	/* Add this node to the imported handles list for the remote GPU,
3785 	 * unless the exporter revoked the handle while the import was in
3786 	 * flight. The dmabuf is released with the last node reference.
3787 	 */
3788 	xa_lock(&adev->ualink.imp_xa);
3789 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_TEARDOWN) {
3790 		xa_unlock(&adev->ualink.imp_xa);
3791 		dev_dbg(adev->dev,
3792 			"IMPORT: handle:%llx:%llx revoked during import\n",
3793 			handle.handle_hi, handle.handle_lo);
3794 		return -EINVAL;
3795 	}
3796 	list_add(&imp_xa_node->list, &adev->ualink.imp_handles_list[remote_acc_id]);
3797 	WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_READY);
3798 	xa_unlock(&adev->ualink.imp_xa);
3799 
3800 	return 0;
3801 
3802 reset_conn:
3803 	dev_dbg(adev->dev,
3804 		"IMPORT: Resetting connection for remote:%u\n", remote_acc_id);
3805 	generation_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
3806 	amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
3807 					      AMDGPU_UALINK_CONN_NOT_READY,
3808 					      generation_count);
3809 	return r;
3810 }
3811 
3812 static int amdgpu_ualink_local_import(struct amdgpu_device *adev,
3813 				      u32 remote_acc_id,
3814 				      struct amdgpu_ualink_handle *handle,
3815 				      int *fd_out)
3816 {
3817 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3818 	struct amdgpu_device *peer_adev;
3819 	int fd;
3820 
3821 	mutex_lock(&mgpu_info.mutex);
3822 	peer_adev = find_peer_adev(remote_acc_id);
3823 	mutex_unlock(&mgpu_info.mutex);
3824 	if (!peer_adev) {
3825 		dev_err(adev->dev,
3826 			"IMPORT LOCAL: peer adev not found for AccId:%u\n",
3827 			remote_acc_id);
3828 		return -ENODEV;
3829 	}
3830 
3831 	xa_lock(&peer_adev->ualink.exp_xa);
3832 	exp_xa_node = xa_load(&peer_adev->ualink.exp_xa, handle->handle_lo);
3833 	if (!exp_xa_node ||
3834 	    exp_xa_node->handle.handle_hi != handle->handle_hi ||
3835 	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
3836 		xa_unlock(&peer_adev->ualink.exp_xa);
3837 		dev_err(adev->dev,
3838 			"IMPORT LOCAL: handle:%llx:%llx not found in peer exp_xa\n",
3839 			handle->handle_hi, handle->handle_lo);
3840 		return -EINVAL;
3841 	}
3842 	xa_unlock(&peer_adev->ualink.exp_xa);
3843 
3844 	get_dma_buf(exp_xa_node->dmabuf);
3845 	/* Get a new fd for the DMABuf */
3846 	fd = dma_buf_fd(exp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
3847 	if (fd < 0) {
3848 		dev_err(adev->dev,
3849 			"IMPORT LOCAL: dma-buf fd failed handle:%llx:%llx\n",
3850 			handle->handle_hi, handle->handle_lo);
3851 		dma_buf_put(exp_xa_node->dmabuf);
3852 		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3853 		return fd;
3854 	}
3855 
3856 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3857 	*fd_out = fd;
3858 	return 0;
3859 }
3860 
3861 int amdgpu_ualink_import_handle(struct drm_device *dev,
3862 				const struct amdgpu_ualink_handle *ualink_handle,
3863 				int *fd_out)
3864 {
3865 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3866 	struct amdgpu_device *adev = drm_to_adev(dev);
3867 	struct amdgpu_ualink_handle handle = *ualink_handle;
3868 	u32 remote_acc_id, node_state;
3869 	int r = 0, fd;
3870 
3871 	remote_acc_id = (handle.handle_lo &
3872 			AMDGPU_UALINK_HANDLE_ACCID_MASK);
3873 
3874 	if (remote_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
3875 		dev_err(adev->dev,
3876 			"IMPORT: invalid remote AccId:%u\n", remote_acc_id);
3877 		return -EINVAL;
3878 	}
3879 
3880 	if (amdgpu_ualink_is_local_accel(adev, remote_acc_id))
3881 		return amdgpu_ualink_local_import(adev, remote_acc_id,
3882 						  &handle, fd_out);
3883 
3884 	xa_lock(&adev->ualink.imp_xa);
3885 	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
3886 
3887 	if (imp_xa_node) {
3888 		/* If node state is Not_ready/Pending, then some other
3889 		 * thread is already trying the NPA protocol for the same
3890 		 * ualink handle. Back off and let the thread finish.
3891 		 * If the node state is Teardown, then it means this
3892 		 * node is about to be removed. So let user-space know
3893 		 * that this handle is invalid.
3894 		 */
3895 		node_state = READ_ONCE(imp_xa_node->node_state);
3896 		if (node_state == AMDGPU_UALINK_NODE_NOT_READY ||
3897 		    node_state == AMDGPU_UALINK_NODE_PENDING) {
3898 			xa_unlock(&adev->ualink.imp_xa);
3899 			r = -EAGAIN;
3900 			goto out;
3901 		} else if (node_state == AMDGPU_UALINK_NODE_TEARDOWN) {
3902 			xa_unlock(&adev->ualink.imp_xa);
3903 			r = -EINVAL;
3904 			goto out;
3905 		}
3906 
3907 		/* Increase the refcount while we are processing the request */
3908 		r = amdgpu_ualink_imp_xa_entry_get(imp_xa_node) ? 0 : -EINVAL;
3909 		xa_unlock(&adev->ualink.imp_xa);
3910 
3911 		/* If the refcount has become 0 but the entry is not yet
3912 		 * removed from the Xarray, then return error to user-space.
3913 		 */
3914 		if (r)
3915 			goto out;
3916 	} else {
3917 		xa_unlock(&adev->ualink.imp_xa);
3918 		/* if the partial handle doesn't exist in the Importer xarray then
3919 		 * initiate the NPA protocol and generate the DMABuf corresponding
3920 		 * to the NPA address.
3921 		 * First store the entry in the Xarray.
3922 		 */
3923 		imp_xa_node = kzalloc(sizeof(*imp_xa_node), GFP_KERNEL);
3924 		if (!imp_xa_node) {
3925 			r = -ENOMEM;
3926 			goto out;
3927 		}
3928 		imp_xa_node->adev = adev;
3929 		imp_xa_node->node_state = AMDGPU_UALINK_NODE_NOT_READY;
3930 		imp_xa_node->handle = handle;
3931 		init_completion(&imp_xa_node->npa_done);
3932 		INIT_LIST_HEAD(&imp_xa_node->list);
3933 		kref_init(&imp_xa_node->refcount);
3934 
3935 		/* Take an extra reference to store in the Xarray. The error
3936 		 * handling paths will drop both these references, while a
3937 		 * successful path will drop only one reference to the Xarray
3938 		 * entry.
3939 		 */
3940 		amdgpu_ualink_imp_xa_entry_get(imp_xa_node);
3941 
3942 		/* Check if another thread created a node for the same handle while
3943 		 * we were trying to create and initialize the node.
3944 		 */
3945 		r = xa_insert(&adev->ualink.imp_xa, handle.handle_lo,
3946 			      imp_xa_node, GFP_KERNEL);
3947 		if (r) {
3948 			kfree(imp_xa_node);
3949 			/* -EBUSY means another thread raced us and inserted a
3950 			 * node for the same handle. Ask user-space to retry.
3951 			 */
3952 			if (r == -EBUSY)
3953 				r = -EAGAIN;
3954 			else
3955 				dev_err(adev->dev,
3956 					"IMPORT: XA insert failed for handle:%llx:%llx err:%d\n",
3957 					handle.handle_hi, handle.handle_lo, r);
3958 			goto out;
3959 		}
3960 
3961 		r = amdgpu_ualink_do_import_handle(adev, imp_xa_node, remote_acc_id);
3962 
3963 		/* If error is returned, then cleanup the xarray entry before returning
3964 		 * the error back to user-space
3965 		 */
3966 		if (r) {
3967 			amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
3968 			if (r != -EAGAIN)
3969 				dev_err(adev->dev,
3970 					"IMPORT: XA import failed for handle:%llx:%llx\n",
3971 					handle.handle_hi, handle.handle_lo);
3972 			goto cleanup;
3973 		}
3974 	}
3975 
3976 	/* dma_buf_fd consumes a reference and assigns it to the fd.
3977 	 * Therefore take an extra reference to be consumed. It will be
3978 	 * released when user mode closes the fd.
3979 	 */
3980 	get_dma_buf(imp_xa_node->dmabuf);
3981 
3982 	fd = dma_buf_fd(imp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
3983 	if (fd >= 0) {
3984 		*fd_out = fd;
3985 	} else {
3986 		dma_buf_put(imp_xa_node->dmabuf);
3987 		r = fd;
3988 		dev_err(adev->dev,
3989 			"IMPORT: dma-buf fd creation failed handle:%llx:%llx\n",
3990 			handle.handle_hi, handle.handle_lo);
3991 	}
3992 
3993 cleanup:
3994 	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
3995 out:
3996 	return r;
3997 }
3998 
3999 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp,
4000 				u32 gem_handle,
4001 				struct amdgpu_ualink_handle *handle_out)
4002 {
4003 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
4004 	struct amdgpu_ualink_handle handle;
4005 	struct drm_gem_object *gobj;
4006 	struct amdgpu_device *adev;
4007 	struct amdgpu_bo *robj;
4008 	int r = 0, i;
4009 
4010 	gobj = drm_gem_object_lookup(filp, gem_handle);
4011 	if (!gobj)
4012 		return -ENOENT;
4013 
4014 	robj = gem_to_amdgpu_bo(gobj);
4015 	adev = amdgpu_ttm_adev(robj->tbo.bdev);
4016 
4017 	if (!(robj->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
4018 		dev_err(adev->dev, "Only VRAM BOs can be exported\n");
4019 		r = -EOPNOTSUPP;
4020 		goto out;
4021 	}
4022 
4023 	if (!robj->ualink_handle_lo) {
4024 		/* If no ualink handle generated for BO, then generate one and
4025 		 * add it to the exporter Xarray.
4026 		 */
4027 		exp_xa_node = kzalloc(sizeof(*exp_xa_node), GFP_KERNEL);
4028 		if (!exp_xa_node) {
4029 			dev_err(adev->dev, "Failed to allocate exp_xa_node\n");
4030 			r = -ENOMEM;
4031 			goto out;
4032 		}
4033 
4034 		amdgpu_bo_ref(robj);
4035 		exp_xa_node->bo = robj;
4036 		init_completion(&exp_xa_node->npa_done);
4037 		bitmap_zero(exp_xa_node->importers_bitmap,
4038 			    AMDGPU_UALINK_ACCEL_MAX);
4039 		bitmap_zero(exp_xa_node->npa_release_bitmap,
4040 			AMDGPU_UALINK_ACCEL_MAX);
4041 		kref_init(&exp_xa_node->refcount);
4042 		mutex_init(&exp_xa_node->node_lock);
4043 		INIT_WORK(&exp_xa_node->cleanup_work,
4044 			  amdgpu_ualink_exp_cleanup_worker);
4045 		for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4046 			INIT_LIST_HEAD(&exp_xa_node->importer_entries[i].list);
4047 			exp_xa_node->importer_entries[i].parent = exp_xa_node;
4048 		}
4049 		/* DMABuf handle for local import of fabric handles */
4050 		exp_xa_node->dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, filp,
4051 						gem_handle, DRM_CLOEXEC | DRM_RDWR);
4052 		if (IS_ERR(exp_xa_node->dmabuf)) {
4053 			r = PTR_ERR(exp_xa_node->dmabuf);
4054 			dev_err(adev->dev, "Failed to generate DMABuf for the BO\n");
4055 			goto err_unref_bo;
4056 		}
4057 
4058 		xa_lock(&adev->ualink.exp_xa);
4059 		amdgpu_generate_ualink_handle(adev, &handle);
4060 		exp_xa_node->handle = handle;
4061 		r = __xa_insert(&adev->ualink.exp_xa, handle.handle_lo,
4062 				exp_xa_node, GFP_KERNEL);
4063 		xa_unlock(&adev->ualink.exp_xa);
4064 		if (r) {
4065 			dev_err(adev->dev, "Failed to insert exp_xa_node into XA: %d\n", r);
4066 			goto err_put_dmabuf;
4067 		}
4068 
4069 		r = amdgpu_bo_reserve(robj, false);
4070 		if (r)
4071 			goto err_erase_xa;
4072 
4073 		if (robj->ualink_handle_lo) {
4074 			amdgpu_bo_unreserve(robj);
4075 			r = -EAGAIN;
4076 			goto err_erase_xa;
4077 		}
4078 
4079 		robj->ualink_handle_lo = handle.handle_lo;
4080 		amdgpu_bo_unreserve(robj);
4081 
4082 		/* Return the generated handle back to the caller */
4083 		*handle_out = handle;
4084 	} else {
4085 		handle_out->handle_lo = robj->ualink_handle_lo;
4086 
4087 		/* Do a sanity check to ensure the handle exists in the XA */
4088 		xa_lock(&adev->ualink.exp_xa);
4089 		exp_xa_node = xa_load(&adev->ualink.exp_xa,
4090 				      robj->ualink_handle_lo);
4091 		xa_unlock(&adev->ualink.exp_xa);
4092 		WARN(!exp_xa_node, "Exp XA: Handle_Lo: %llx not found",
4093 		     robj->ualink_handle_lo);
4094 		if (exp_xa_node)
4095 			handle_out->handle_hi = exp_xa_node->handle.handle_hi;
4096 	}
4097 
4098 	drm_gem_object_put(gobj);
4099 	return 0;
4100 
4101 err_erase_xa:
4102 	xa_erase(&adev->ualink.exp_xa, handle.handle_lo);
4103 err_put_dmabuf:
4104 	dma_buf_put(exp_xa_node->dmabuf);
4105 err_unref_bo:
4106 	amdgpu_bo_unref(&exp_xa_node->bo);
4107 	kfree(exp_xa_node);
4108 out:
4109 	drm_gem_object_put(gobj);
4110 	return r;
4111 }
4112 
4113 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data,
4114 				   struct drm_file *filp)
4115 {
4116 	union drm_amdgpu_ualink_handle *args = data;
4117 	struct amdgpu_device *adev = drm_to_adev(dev);
4118 	struct amdgpu_ualink_handle handle = {};
4119 	u32 gem_handle;
4120 	int r, fd = -1;
4121 
4122 	/* The ioctl is registered for every amdgpu device, but the UALink
4123 	 * software state only exists on ASICs that instantiate the UALink IP
4124 	 * block. Bail out if it is absent, and short-circuit before the
4125 	 * accel_state dereference below.
4126 	 */
4127 	if (!adev->ualink.info ||
4128 	    adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
4129 		dev_dbg(adev->dev,
4130 			"ualink device not available or not in active state\n");
4131 		return -EOPNOTSUPP;
4132 	}
4133 
4134 	/* The input and output members of the ioctl argument alias each other.
4135 	 * Latch every input field before invoking the handlers, and only write
4136 	 * the output fields afterwards.
4137 	 */
4138 	switch (args->in.op) {
4139 	case DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT:
4140 		gem_handle = args->in.gem_handle;
4141 		r = amdgpu_ualink_export_handle(dev, filp, gem_handle, &handle);
4142 		if (!r) {
4143 			args->out.export_ualink_handle[0] = handle.handle_lo;
4144 			args->out.export_ualink_handle[1] = handle.handle_hi;
4145 		}
4146 		break;
4147 	case DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT:
4148 		handle.handle_lo = args->in.import_ualink_handle[0];
4149 		handle.handle_hi = args->in.import_ualink_handle[1];
4150 		r = amdgpu_ualink_import_handle(dev, &handle, &fd);
4151 		if (!r)
4152 			args->out.import_dmabuf_handle = fd;
4153 		break;
4154 	default:
4155 		r = -EINVAL;
4156 		break;
4157 	}
4158 
4159 	return r;
4160 }
4161 
4162 int amdgpu_ualink_manager_start(struct amdgpu_device *adev)
4163 {
4164 	int i, r;
4165 
4166 	r = amdgpu_vm_init(adev, &adev->ualink.npa_vm, 0);
4167 	if (r)
4168 		goto out;
4169 
4170 	/* For pinning page tables and using CPU for page table updates. */
4171 	r = amdgpu_vm_make_npa(adev, &adev->ualink.npa_vm);
4172 	if (r)
4173 		goto uninit_vm;
4174 
4175 	adev->ualink.npa_wq = alloc_workqueue("NPA WQ", WQ_UNBOUND, 0);
4176 	if (unlikely(!adev->ualink.npa_wq)) {
4177 		dev_err(adev->dev, "Failed to allocate NPA WQ\n");
4178 		r = -ENOMEM;
4179 		goto uninit_vm;
4180 	}
4181 
4182 	/* Map this VM to NPA VMID */
4183 	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid,
4184 			amdgpu_gmc_pd_addr(adev->ualink.npa_vm.root.bo));
4185 
4186 	xa_init(&adev->ualink.exp_xa);
4187 	xa_init(&adev->ualink.imp_xa);
4188 	xa_init(&adev->ualink.handle_invalid_xa);
4189 
4190 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4191 		init_completion(&adev->ualink.conn_state[i].hello_done);
4192 		mutex_init(&adev->ualink.conn_state[i].lock);
4193 		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
4194 		INIT_LIST_HEAD(&adev->ualink.exp_handles_list[i]);
4195 		INIT_LIST_HEAD(&adev->ualink.imp_handles_list[i]);
4196 	}
4197 
4198 	amdgpu_ualink_npa_mm_init(adev);
4199 
4200 	return 0;
4201 
4202 uninit_vm:
4203 	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
4204 out:
4205 	return r;
4206 }
4207 
4208 /* Free every importer entry left in imp_xa at manager stop. */
4209 static void amdgpu_ualink_teardown_imp_xa_entries(struct amdgpu_device *adev)
4210 {
4211 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
4212 	unsigned long index;
4213 
4214 	xa_for_each(&adev->ualink.imp_xa, index, imp_xa_node) {
4215 		list_del_init(&imp_xa_node->list);
4216 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
4217 		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
4218 	}
4219 }
4220 
4221 /* Free a single exporter node at manager stop. This is the message-free
4222  * portion of amdgpu_ualink_exp_cleanup_worker().
4223  */
4224 static void amdgpu_ualink_teardown_exp_xa_node(struct amdgpu_device *adev,
4225 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
4226 {
4227 	DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
4228 	struct amdgpu_bo *bo = exp_xa_node->bo;
4229 	u32 remote_acc_id;
4230 
4231 	bitmap_copy(importers_bitmap, exp_xa_node->importers_bitmap,
4232 		    AMDGPU_UALINK_ACCEL_MAX);
4233 
4234 	dev_dbg(adev->dev,
4235 		"EXP-STOP: handle:%llx:%llx importers bitmap: %*pbl\n",
4236 		exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
4237 		AMDGPU_UALINK_ACCEL_MAX, importers_bitmap);
4238 
4239 	if (!bitmap_empty(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)) {
4240 		amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
4241 
4242 		/* Unlink importers and unpin the BO once per importer. */
4243 		if (likely(!amdgpu_bo_reserve(bo, true))) {
4244 			bo->ualink_handle_lo = 0ULL;
4245 			for_each_set_bit(remote_acc_id, importers_bitmap,
4246 					 AMDGPU_UALINK_ACCEL_MAX) {
4247 				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
4248 				amdgpu_bo_unpin(bo);
4249 			}
4250 			amdgpu_bo_unreserve(bo);
4251 		} else {
4252 			dev_warn(adev->dev,
4253 				"EXP-STOP: BO reserve to unpin failed handle:%llx:%llx\n",
4254 				exp_xa_node->handle.handle_hi,
4255 				exp_xa_node->handle.handle_lo);
4256 			for_each_set_bit(remote_acc_id, importers_bitmap,
4257 					 AMDGPU_UALINK_ACCEL_MAX)
4258 				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
4259 		}
4260 
4261 		amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, importers_bitmap);
4262 	}
4263 
4264 	/* Release the export dma-buf and drop the BO ref. */
4265 	dma_buf_put(exp_xa_node->dmabuf);
4266 	amdgpu_bo_unref(&bo);
4267 	exp_xa_node->bo = NULL;
4268 
4269 	mutex_destroy(&exp_xa_node->node_lock);
4270 	kfree(exp_xa_node);
4271 }
4272 
4273 /* Free every exporter entry left in exp_xa at manager stop. These were
4274  * exported but never revoked, so no cleanup worker was queued for them.
4275  */
4276 static void amdgpu_ualink_teardown_exp_xa_entries(struct amdgpu_device *adev)
4277 {
4278 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
4279 	unsigned long index;
4280 
4281 	xa_for_each(&adev->ualink.exp_xa, index, exp_xa_node) {
4282 		xa_erase(&adev->ualink.exp_xa, index);
4283 		amdgpu_ualink_teardown_exp_xa_node(adev, exp_xa_node);
4284 	}
4285 }
4286 
4287 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev)
4288 {
4289 	int i;
4290 
4291 	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 0);
4292 
4293 	/* Mark connections down so the drained workers and the teardown below
4294 	 * take the message-free path in amdgpu_ualink_check_conn_ready().
4295 	 */
4296 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4297 		mutex_lock(&adev->ualink.conn_state[i].lock);
4298 		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
4299 		mutex_unlock(&adev->ualink.conn_state[i].lock);
4300 	}
4301 
4302 	/* Drain in-flight exporter cleanup work so revoked nodes in
4303 	 * handle_invalid_xa free themselves and empty that xarray.
4304 	 */
4305 	drain_workqueue(adev->ualink.npa_wq);
4306 
4307 	/* Free live entries before the NPA allocator is torn down, so NPA
4308 	 * addresses can still be unmapped and freed.
4309 	 */
4310 	amdgpu_ualink_teardown_imp_xa_entries(adev);
4311 	amdgpu_ualink_teardown_exp_xa_entries(adev);
4312 
4313 	amdgpu_ualink_npa_mm_fini(adev);
4314 
4315 	/* All three xarrays are empty by now. */
4316 	xa_destroy(&adev->ualink.exp_xa);
4317 	xa_destroy(&adev->ualink.imp_xa);
4318 	xa_destroy(&adev->ualink.handle_invalid_xa);
4319 
4320 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++)
4321 		mutex_destroy(&adev->ualink.conn_state[i].lock);
4322 
4323 	destroy_workqueue(adev->ualink.npa_wq);
4324 	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
4325 }
4326 
4327 /*
4328  * UALink remote interrupt and shootdown
4329  */
4330 
4331 /* UALINK ring buffer size, same for both remote shootdown and interrupt ring */
4332 #define AMDGPU_UALINK_RB_SIZE	4096
4333 
4334 #define AMDGPU_UALINK_METADATA_HEADER	0x4E485446
4335 
4336 /* 2MB NPA start address for 2MB page mapping */
4337 #define AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET SZ_2M
4338 
4339 struct amdgpu_ualink_metadata {
4340 	u32 header;
4341 
4342 	/*
4343 	 * "ring entries" as unit. So the ring-size-in-bytes could be calculated
4344 	 * as entry-size * 2^RBsize. This would make the minimum size a single
4345 	 * entry and the maximum size 32786 entries.
4346 	 *
4347 	 * u32 rb_size:4;
4348 	 * u32 reserved0:4;
4349 	 * u32 vmid:4;
4350 	 * u32 reserved1:20;
4351 	 */
4352 	u32 rb_size;
4353 
4354 	/* ring buffer base address for remote interrupt */
4355 	u64 ri_rb;
4356 
4357 	/* tlb invalidate ring buffer base address for remote shootdown */
4358 	u64 tlb_inv_rb;
4359 
4360 	/* remote interrupt Tail pointer, write pointer address */
4361 	u64 tailptr_ri;
4362 
4363 	/* TLB invalidate ring buffer's Tail pointer, write pointer address */
4364 	u64 tailptr_tlb_inv;
4365 };
4366 
4367 struct amdgpu_ualink_wb {
4368 	/* last finished command seq number */
4369 	u32 data;
4370 
4371 	/* command complete error code */
4372 	u32 status;
4373 
4374 	/* ring rptr updated by FW */
4375 	u64 rptr;
4376 };
4377 
4378 /*
4379  * For remote interrupt and shootdown
4380  */
4381 struct amdgpu_ualink_ring {
4382 	u32 rb_size;
4383 
4384 	/* writeback data */
4385 	u32 seq;
4386 
4387 	/* local copy */
4388 	u64 wptr, rptr;
4389 
4390 	/* NPA gart mapping for SDMA */
4391 	u64 rb_npa_gart;
4392 	u64 wptr_npa_gart;
4393 	u64 doorbell_npa_gart;
4394 
4395 	/* gart mapping node */
4396 	struct drm_mm_node mm_node_rb;
4397 	struct drm_mm_node mm_node_wptr;
4398 	struct drm_mm_node mm_node_doorbell;
4399 
4400 	/* seq, complete status write back cpu address */
4401 	struct amdgpu_ualink_wb *wb_cpu;
4402 
4403 	/* true if fw write back address updated successfully */
4404 	bool ready;
4405 };
4406 
4407 struct amdgpu_ualink_peer {
4408 	/* remote interrupt and shootdown uses same SDMA entity */
4409 	struct mutex lock;
4410 
4411 	/* SDMA engine to send remote command via NPA */
4412 	struct drm_sched_entity entity;
4413 
4414 	/* to select different DXS ports, cycles through different values */
4415 	u32 dxs_port;
4416 
4417 	struct amdgpu_ualink_ring interrupt;
4418 	struct amdgpu_ualink_ring shootdown;
4419 };
4420 
4421 struct amdgpu_ualink_remote {
4422 	/* ualink metadata passed to MPNHT FW */
4423 	struct amdgpu_bo		*metadata_bo;
4424 	u64				metadata_gpu_addr;
4425 	void				*metadata_cpu_addr;
4426 
4427 	/* ualink ring, tlb ring buffer, wptr */
4428 	struct amdgpu_bo		*ring_bo;
4429 	u64				rb_gpu_addr;
4430 	void				*rb_cpu_addr;
4431 
4432 	/* ualink rptr, wb data, statuss for address alias mode */
4433 	struct amdgpu_bo		*rptr_bo;
4434 	u64				rptr_gpu_addr;
4435 	void				*rptr_cpu_addr;
4436 	u64				rptr_npa;
4437 
4438 	/* address alias mode alloc npa address for shared wb */
4439 	struct drm_mm_node		rptr_mm_node;
4440 
4441 	/*
4442 	 * Owned snapshot of the vPod's active accelerator bitmap, taken at
4443 	 * sw_init time. Must NOT alias info->vpod.active_accel_bits, which
4444 	 * psp_ual_query_info() refreshes on every commit: aliasing it would let
4445 	 * teardown unmap a different set than setup mapped, leaking drm_mm nodes
4446 	 * in the shared GTT manager.
4447 	 */
4448 	DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
4449 	u32				num_accel;
4450 
4451 	/* remote GPUs ring buffer, read, write pointer local copy and gart mapping */
4452 	struct amdgpu_ualink_peer	peer[AMDGPU_UALINK_ACCEL_MAX];
4453 
4454 	/* use lsdma write to NPA address for remote interrupt */
4455 	bool			       use_lsdma;
4456 };
4457 
4458 static inline struct amdgpu_ualink_remote *to_remote(struct amdgpu_device *adev)
4459 {
4460 	return adev->ualink.remote;
4461 }
4462 
4463 /*
4464  * Returns true if the vPod membership most recently reported by firmware
4465  * (info->vpod.active_accel_bits, read back via psp_ual_query_info() on the
4466  * committing GPU) differs from the owned snapshot captured when this
4467  * accelerator was last activated (remote->active_accel_bits). Used to
4468  * decide whether an accelerator must be (re)built to match the new member
4469  * set. A NULL remote means the accelerator has never been activated (no
4470  * snapshot yet), which also counts as "changed" so it gets brought up.
4471  */
4472 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev)
4473 {
4474 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4475 
4476 	if (!remote)
4477 		return true;
4478 
4479 	return !bitmap_equal(remote->active_accel_bits,
4480 			     adev->ualink.info->vpod.active_accel_bits,
4481 			     AMDGPU_UALINK_ACCEL_MAX);
4482 }
4483 
4484 static inline u32 ualink_accel_id(struct amdgpu_device *adev)
4485 {
4486 	return adev->ualink.info->ppod.accel_id;
4487 }
4488 
4489 static inline enum amdgpu_ualink_addr_mode ualink_addr_mode(struct amdgpu_device *adev)
4490 {
4491 	return adev->ualink.info->vpod.addr_mode;
4492 }
4493 
4494 /*
4495  * address mode and NPA address for ring wptr, rptr, wb status, wb data
4496  *
4497  * source identification address mode
4498  *
4499  * we need to export different tail pointers for different remote GPU accel_id,
4500  * hence we need to allocate a whole page for each remote GPU separately.
4501  *
4502  * rptr, wb data, status share the whole page with wptr.
4503  *
4504  *     1 page reserved NPA address for 1 active remote GPU, max 255 pages
4505  *     starting address: wptr NPA address + accel_id * 4K
4506  *     offset:
4507  *         remote interrupt ring wptr 0
4508  *         remote shootdown ring wptr 8
4509  *         remote interrupt wb status 16
4510  *         remote interrupt wb data   20
4511  *         remote interrupt ring rptr 24
4512  *         remote shootdown wb status 32
4513  *         remote shootdown wb data   36
4514  *         remote shootdown ring rptr 40
4515  *
4516  * source aliasing address mode
4517  *
4518  * wptr for different remote GPU can share same reserved 1 page NPA address page.
4519  *
4520  * to save reserved NPA address space, rptr, wb data, status for different
4521  * remote GPU share same allocated 2 pages NPA address.
4522  *
4523  *     1 page reserved NPA address for wptr of all active remote GPUs
4524  *     offset:
4525  *         remote interrupt ring wptr accel_id * 16
4526  *         remote shootdown ring wptr accel_id * 16 + 8
4527  *
4528  *     2 page allocated NPA address for rptr, writeback of all active remote GPUs
4529  *     starting address: at accel_id * 32 for each remote GPU
4530  *     offset:
4531  *         remote interrupt wb status 0
4532  *         remote interrupt wb data   4
4533  *         remote interrupt ring rptr 8
4534  *         remote shootdown wb status 16
4535  *         remote shootdown wb data   20
4536  *         remote shootdown ring rptr 24
4537  */
4538 static inline u32 ualink_wptr_size(struct amdgpu_device *adev)
4539 {
4540 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4541 		return AMDGPU_GPU_PAGE_SIZE;
4542 	else
4543 		return 0;	/* wptr uses the ring buffer for itself */
4544 }
4545 
4546 static inline u32 ualink_wb_size(struct amdgpu_device *adev)
4547 {
4548 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4549 		return 0;	/* share same page with wptr */
4550 	else
4551 		return 2 * sizeof(struct amdgpu_ualink_wb);
4552 }
4553 
4554 static inline u32 ualink_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
4555 {
4556 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4557 		return 0;	/* one separate page per GPU */
4558 	else
4559 		return accel_id * 2 * sizeof(u64);	/* two pointers per GPU */
4560 }
4561 
4562 static inline u32 ualink_tlb_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
4563 {
4564 	return ualink_wptr_offset(adev, accel_id) + sizeof(u64);
4565 }
4566 
4567 static inline u32 ualink_wb_offset(struct amdgpu_device *adev, u32 accel_id)
4568 {
4569 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4570 		return 2 * sizeof(u64);	/* after two wptr pointers */
4571 	else
4572 		return accel_id * ualink_wb_size(adev);
4573 }
4574 
4575 static inline u32 ualink_tlb_wb_offset(struct amdgpu_device *adev, u32 accel_id)
4576 {
4577 	return ualink_wb_offset(adev, accel_id) + sizeof(struct amdgpu_ualink_wb);
4578 }
4579 
4580 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, u32 flush_type)
4581 {
4582 	u32 bit;
4583 
4584 	bit = AMDGPU_MMHUB0_START;
4585 
4586 	for_each_set_bit_from(bit, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
4587 		amdgpu_gmc_flush_gpu_tlb(adev, adev->vm_manager.npa_vmid,
4588 					bit, flush_type);
4589 }
4590 
4591 /**
4592  * amdgpu_ualink_npa_vm_map_range - Map a range in the NPA VM
4593  * @adev: amdgpu device pointer
4594  * @bo: buffer object backing the mapping
4595  * @pte_flags: page table entry flags
4596  * @offset: offset into the buffer object in bytes
4597  * @size_in_pages: size of the range to map in pages
4598  * @npa_in_pages: NPA target address in pages
4599  *
4600  * Maps a buffer object range into the NPA VM page table at the specified
4601  * NPA address.
4602  *
4603  * Return: 0 on success, negative error code on failure
4604  */
4605 static int amdgpu_ualink_npa_vm_map_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
4606 				   u64 pte_flags, u64 offset, u64 size_in_pages,
4607 				   u64 npa_in_pages)
4608 {
4609 	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
4610 	int r;
4611 
4612 	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
4613 		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
4614 		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4615 
4616 	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
4617 				   false, NULL, npa_in_pages,
4618 				   npa_in_pages + size_in_pages - 1,
4619 				   pte_flags, offset, adev->vm_manager.vram_base_offset,
4620 				   bo->tbo.resource, NULL, &npa_vm->last_update);
4621 	if (r)
4622 		dev_dbg(adev->dev, "failed %d to map npa 0x%llx to NPA VM\n", r,
4623 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4624 	return r;
4625 }
4626 
4627 /**
4628  * amdgpu_npa_vm_unmap_range - Unmap a range from the NPA VM
4629  * @adev: amdgpu device pointer
4630  * @bo: buffer object backing the mapping
4631  * @pte_flags: page table entry flags
4632  * @offset: offset into the buffer object in bytes
4633  * @size_in_pages: size of the range to unmap in pages
4634  * @npa_in_pages: NPA target address in pages
4635  *
4636  * Removes a previously established mapping from the NPA VM page table.
4637  *
4638  * Return: 0 on success, negative error code on failure
4639  */
4640 static int amdgpu_ualink_npa_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
4641 				     u64 pte_flags, u64 offset, u64 size_in_pages,
4642 				     u64 npa_in_pages)
4643 {
4644 	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
4645 	int r;
4646 
4647 	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
4648 		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
4649 		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4650 
4651 	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
4652 				   false, NULL, npa_in_pages,
4653 				   npa_in_pages + size_in_pages - 1,
4654 				   pte_flags, offset, 0, bo->tbo.resource, NULL,
4655 				   &npa_vm->last_update);
4656 	if (r)
4657 		dev_dbg(adev->dev, "failed %d to unmap npa 0x%llx from NPA VM\n", r,
4658 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4659 	return r;
4660 }
4661 
4662 /*
4663  * Reserved NPA space for remote shootdown and interrupt ring buffer,
4664  * write pointers, read pointers and writeback buffers
4665  *
4666  * NPA ring type: 0 = shootdown, 1 = interrupt, 2 = tailptr (wptr)
4667  */
4668 enum ring_buffer_type {
4669 	RB_TYPE_TLB_INV			= 0,
4670 	RB_TYPE_REMOTE_INTERRUPT	= 1,
4671 	RB_TYPE_TAILPTR			= 2
4672 };
4673 
4674 /**
4675  * amdgpu_ualink_npa_addr - Get reserved NPA address for ring buffer
4676  * @adev: amdgpu device pointer
4677  * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
4678  * @src_accel_id: Source accelerator ID
4679  * @dst_accel_id: Destination accelerator ID
4680  *
4681  * Return: NPA address for the specified ring buffer type and GPUs
4682  */
4683 static u64 amdgpu_ualink_npa_addr(struct amdgpu_device *adev, u32 type,
4684 				  u32 src_accel_id, u32 dst_accel_id)
4685 {
4686 	u32 addr_mode = ualink_addr_mode(adev);
4687 	u64 npa;
4688 
4689 	WARN_ON_ONCE(src_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
4690 		     dst_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
4691 		     type > RB_TYPE_TAILPTR ||
4692 		     (addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT &&
4693 		     addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS));
4694 
4695 	switch (addr_mode) {
4696 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT:
4697 		npa = (u64)src_accel_id << 41 | type << 12;
4698 		break;
4699 
4700 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS:
4701 	default:
4702 		if (type == RB_TYPE_TAILPTR)
4703 			npa = (u64)dst_accel_id << 13;
4704 		else
4705 			npa = (u64)src_accel_id << 13 | type << 12;
4706 
4707 		/*
4708 		 * In order to avoid address conflicts between source-identification
4709 		 * and source-aliasing mode, add 2MB to the buffer addresses in
4710 		 * source-aliasing mode. This way in a misconfigured cluster, a GPU
4711 		 * configured in the wrong address mode will access unmapped NPA addresses
4712 		 * rather than the wrong buffer mappings.
4713 		 */
4714 		npa += AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET;
4715 		break;
4716 	}
4717 
4718 	dev_dbg(adev->dev, "addr mode %d from accel %d to accel %d type %d NPA 0x%llx\n",
4719 		addr_mode, src_accel_id, dst_accel_id, type, npa);
4720 
4721 	return npa;
4722 }
4723 
4724 /**
4725  * amdgpu_ualink_gart_npa_addr - Get GART-mapped NPA address for ring buffer
4726  * @adev: amdgpu device pointer
4727  * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
4728  * @src_accel_id: Source accelerator ID
4729  * @dst_accel_id: Destination accelerator ID
4730  *
4731  * Computes the NPA address suitable for GART mapping by adding dst_accid_id to
4732  * NPA address to access remote by SDMA.
4733 
4734  * clears the source GPU ID bits from the NPA address which is set for
4735  * address-identification mode.
4736  *
4737  * Return: NPA address adjusted for GART mapping
4738  */
4739 
4740 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT		41
4741 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK		GENMASK_ULL(50, 41)
4742 
4743 static u64 amdgpu_ualink_gart_npa_addr(struct amdgpu_device *adev, u32 type,
4744 					u32 src_accel_id, u32 dst_accel_id)
4745 {
4746 	u64 npa;
4747 
4748 	npa = amdgpu_ualink_npa_addr(adev, type, src_accel_id, dst_accel_id);
4749 
4750 	/* wiping out source accelerator id */
4751 	npa &= ~AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK;
4752 
4753 	return npa | ((u64)dst_accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT);
4754 }
4755 
4756 /**
4757  * amdgpu_ualink_reserve_npa_vm_and_bos - Reserve the NPA VM page directory and BOs
4758  * @adev: amdgpu device pointer
4759  * @bos: array of buffer objects to reserve
4760  * @n_bos: number of entries in @bos
4761  * @exec: drm_exec context to initialize and use for locking
4762  * @interruptible: true to use the interruptible dma_resv_lock
4763  *
4764  * Initializes @exec and uses it to lock all BOs in @bos together with the
4765  * NPA VM page directory, retrying on contention. On failure, @exec is
4766  * finalized and the error code is returned.
4767  *
4768  * Returns: 0 on success, negative error code on failure.
4769  */
4770 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
4771 						struct amdgpu_bo *bos[], u32 n_bos,
4772 						struct drm_exec *exec,
4773 						bool interruptible)
4774 {
4775 	u32 flags = DRM_EXEC_IGNORE_DUPLICATES;
4776 	int i, r = 0;
4777 
4778 	if (interruptible)
4779 		flags |= DRM_EXEC_INTERRUPTIBLE_WAIT;
4780 
4781 	dev_dbg(adev->dev, "reserve NPA vm and %d bos\n", n_bos);
4782 
4783 	drm_exec_init(exec, flags, 0);
4784 
4785 	drm_exec_until_all_locked(exec) {
4786 		for (i = 0; i < n_bos; i++) {
4787 			r = drm_exec_lock_obj(exec, &bos[i]->tbo.base);
4788 			drm_exec_retry_on_contention(exec);
4789 			if (unlikely(r))
4790 				goto out;
4791 		}
4792 
4793 		r = amdgpu_vm_lock_pd(&adev->ualink.npa_vm, exec, 0);
4794 		drm_exec_retry_on_contention(exec);
4795 		if (unlikely(r))
4796 			goto out;
4797 	}
4798 
4799 out:
4800 	if (r)
4801 		drm_exec_fini(exec);
4802 	return r;
4803 
4804 }
4805 
4806 /**
4807  * amdgpu_ualink_unreserve_npa_vm_and_bos - Release the NPA VM page directory and BOs
4808  * @adev: amdgpu device pointer
4809  * @exec: drm_exec context previously initialized by
4810  *        amdgpu_ualink_reserve_npa_vm_and_bos()
4811  *
4812  * Finalizes @exec, releasing all locks on the NPA VM page directory and
4813  * the associated BOs acquired during reservation.
4814  */
4815 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
4816 						   struct drm_exec *exec)
4817 {
4818 	dev_dbg(adev->dev, "unreserve NPA vm and bos\n");
4819 	drm_exec_fini(exec);
4820 }
4821 
4822 /**
4823  * amdgpu_ualink_metadata_npa_unmapping - Tear down NPA address mappings
4824  * @adev: amdgpu device pointer
4825  *
4826  * Unmaps all NPA address mappings from the NPA VM for ring buffers,
4827  * write pointers, and read pointers. Waits for outstanding DMA fences
4828  * and flushes the TLB.
4829  */
4830 static void amdgpu_ualink_metadata_npa_unmapping(struct amdgpu_device *adev)
4831 {
4832 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4833 	u64 timeout = msecs_to_jiffies(2000);
4834 	u32 dst_accel_id = ualink_accel_id(adev);
4835 	struct amdgpu_bo *bos[2];
4836 	u32 n_bos;
4837 	struct dma_fence *fence;
4838 	struct drm_exec exec;
4839 	u32 rptr_size, rptr_size_in_pages;
4840 	u32 rb_size, rb_size_in_pages;
4841 	u64 pte_flags = adev->gmc.noretry_flags;
4842 	u64 npa;
4843 	int r;
4844 
4845 	if (!remote->ring_bo)
4846 		return;
4847 
4848 	rb_size = AMDGPU_UALINK_RB_SIZE;
4849 	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
4850 
4851 	bos[0] = remote->ring_bo;
4852 	n_bos = 1;
4853 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4854 		bos[1] = remote->rptr_bo;
4855 		n_bos = 2;
4856 	}
4857 
4858 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
4859 	if (unlikely(r))
4860 		return;
4861 
4862 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4863 		/* rptr npa mapping, up to allocated 2 pages npa address */
4864 		rptr_size = ualink_wb_size(adev);
4865 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
4866 		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
4867 
4868 		npa = remote->rptr_npa;
4869 
4870 		amdgpu_ualink_npa_vm_unmap_range(adev, remote->rptr_bo,
4871 						 pte_flags, 0, rptr_size_in_pages,
4872 						 npa >> AMDGPU_GPU_PAGE_SHIFT);
4873 		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
4874 
4875 		/*
4876 		 * unmap wptr, remote interrupt, shootdown ring.
4877 		 * wptr page is part of the single 2MB mapping for remote GPUs
4878 		 * interrupt and shootdown ring buffer
4879 		 */
4880 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
4881 					     0, dst_accel_id);
4882 		amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, pte_flags,
4883 						 0,
4884 						 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
4885 						 npa >> AMDGPU_GPU_PAGE_SHIFT);
4886 	} else {
4887 		u32 wptr_offset = 2 * rb_size * remote->num_accel;
4888 		u32 idx = 0;
4889 		u32 accel_id;
4890 
4891 		/* source identification mode */
4892 		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
4893 			if (accel_id == dst_accel_id)
4894 				continue;
4895 
4896 			/* remote interrupt ring npa mapping */
4897 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
4898 						     accel_id, dst_accel_id);
4899 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4900 							 pte_flags, idx * 2 * rb_size,
4901 							 rb_size_in_pages,
4902 							 npa >> AMDGPU_GPU_PAGE_SHIFT);
4903 
4904 			/* remote shootdown ring npa mapping */
4905 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
4906 						     dst_accel_id);
4907 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4908 							 pte_flags, (idx * 2 + 1) * rb_size,
4909 							 rb_size_in_pages,
4910 							 npa >> AMDGPU_GPU_PAGE_SHIFT);
4911 
4912 			/* wptr, rptr npa mapping */
4913 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
4914 						      dst_accel_id);
4915 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4916 						  pte_flags, wptr_offset + idx * PAGE_SIZE,
4917 						  1, npa >> AMDGPU_GPU_PAGE_SHIFT);
4918 			idx++;
4919 		}
4920 	}
4921 
4922 	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
4923 	if (r) {
4924 		dev_dbg(adev->dev, "failed %d to update directories\n", r);
4925 		goto out_unreserve;
4926 	}
4927 
4928 	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
4929 	if (fence) {
4930 		r = dma_fence_wait_timeout(fence, true, timeout);
4931 		dma_fence_put(fence);
4932 		if (r <= 0)
4933 			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
4934 	}
4935 
4936 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
4937 out_unreserve:
4938 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
4939 }
4940 
4941 /**
4942  * amdgpu_ualink_metadata_npa_mapping - Setup NPA address mapping in VM
4943  * @adev: amdgpu device pointer
4944  *
4945  * Maps NPA addresses to GPA for metadata ring buffers, write pointers,
4946  * on NPA VMID 15.
4947  *
4948  * Return: 0 on success, negative error code on failure
4949  */
4950 static int amdgpu_ualink_metadata_npa_mapping(struct amdgpu_device *adev)
4951 {
4952 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4953 	u32 dst_accel_id = ualink_accel_id(adev);
4954 	u64 timeout = msecs_to_jiffies(2000);
4955 	struct amdgpu_bo *bos[2];
4956 	u32 n_bos;
4957 	struct dma_fence *fence;
4958 	struct drm_exec exec;
4959 	u32 rb_size, rb_size_in_pages;
4960 	u64 npa, npa_in_pages, pte_flags;
4961 	int r;
4962 
4963 	rb_size = AMDGPU_UALINK_RB_SIZE;
4964 	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
4965 
4966 	bos[0] = remote->ring_bo;
4967 	n_bos = 1;
4968 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4969 		bos[1] = remote->rptr_bo;
4970 		n_bos = 2;
4971 	}
4972 
4973 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
4974 	if (unlikely(r))
4975 		return r;
4976 
4977 	pte_flags = amdgpu_ttm_tt_pte_flags(adev, remote->ring_bo->tbo.ttm,
4978 					    remote->ring_bo->tbo.resource);
4979 	dev_dbg(adev->dev, "init pte_flags 0x%llx\n", pte_flags);
4980 
4981 	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, remote->ring_bo,
4982 			      AMDGPU_VM_MTYPE_DEFAULT, &pte_flags);
4983 	dev_dbg(adev->dev, "after get coherent pte_flags 0x%llx\n", pte_flags);
4984 
4985 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4986 		u32 rptr_size, rptr_size_in_pages;
4987 
4988 		/* rptr npa mapping, up to allocated 2 pages npa address */
4989 		rptr_size = ualink_wb_size(adev);
4990 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
4991 		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
4992 
4993 		r = amdgpu_ualink_npa_alloc_va(adev, &remote->rptr_mm_node,
4994 					       0, 0, 0,
4995 					       rptr_size_in_pages);
4996 		if (r)
4997 			goto out;
4998 
4999 		npa_in_pages = remote->rptr_mm_node.start;
5000 
5001 		dev_dbg(adev->dev, "source aliasing rptr alloc 0x%llx and map to npa vm\n",
5002 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
5003 
5004 		r = amdgpu_ualink_npa_vm_map_range(adev, remote->rptr_bo, pte_flags, 0,
5005 						   rptr_size_in_pages,
5006 						   npa_in_pages);
5007 		if (r)
5008 			goto error_npa_mapping;
5009 
5010 		remote->rptr_npa = npa_in_pages << AMDGPU_GPU_PAGE_SHIFT;
5011 
5012 		/* wptr, remote interrupt, shootdown ring, single big 2MB mapping */
5013 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
5014 					     0, dst_accel_id);
5015 		r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5016 						   0,
5017 						   2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
5018 						   npa >> AMDGPU_GPU_PAGE_SHIFT);
5019 		if (r)
5020 			goto error_npa_mapping;
5021 
5022 	} else {
5023 		u32 wptr_offset = 2 * rb_size * remote->num_accel;
5024 		u32 accel_id, idx = 0;
5025 
5026 		/* Source identification mode */
5027 		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5028 			if (accel_id == dst_accel_id)
5029 				continue;
5030 
5031 			/* remote interrupt ring npa mapping */
5032 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
5033 						      accel_id, dst_accel_id);
5034 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5035 							   idx * 2 * rb_size,
5036 							   rb_size_in_pages,
5037 							   npa >> AMDGPU_GPU_PAGE_SHIFT);
5038 			if (r)
5039 				goto error_npa_mapping;
5040 
5041 			/* remote shootdown ring npa mapping */
5042 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
5043 						     dst_accel_id);
5044 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5045 							   (idx * 2 + 1) * rb_size,
5046 							   rb_size_in_pages,
5047 							   npa >> AMDGPU_GPU_PAGE_SHIFT);
5048 			if (r)
5049 				goto error_npa_mapping;
5050 
5051 			/* wptr, rptr npa mapping */
5052 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
5053 						     dst_accel_id);
5054 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo,
5055 							   pte_flags,
5056 							   wptr_offset + idx * PAGE_SIZE,
5057 							   1, npa >> AMDGPU_GPU_PAGE_SHIFT);
5058 			if (r)
5059 				goto error_npa_mapping;
5060 
5061 			idx++;
5062 		}
5063 	}
5064 
5065 	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
5066 	if (r) {
5067 		dev_dbg(adev->dev, "failed %d to update directories\n", r);
5068 		goto error_npa_mapping;
5069 	}
5070 
5071 	/* TODO: only wait the last fence, then flush TLB */
5072 	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
5073 	if (fence) {
5074 		r = dma_fence_wait_timeout(fence, true, timeout);
5075 		dma_fence_put(fence);
5076 		if (r <= 0)
5077 			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
5078 	}
5079 
5080 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
5081 
5082 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
5083 	return 0;
5084 
5085 error_npa_mapping:
5086 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
5087 		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
5088 	if (r)
5089 		dev_dbg(adev->dev, "failed %d to map NPA vm\n", r);
5090 
5091 out:
5092 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
5093 
5094 	return r;
5095 }
5096 
5097 /**
5098  * amdgpu_ualink_sdma_entities_init - Initialize SDMA job scheduler entities
5099  * @adev: amdgpu device pointer
5100  *
5101  * Sets up SDMA job scheduler entities for remote interrupt and shootdown.
5102  * Each entity uses one SDMA scheduler/ring.
5103  *
5104  * Return: 0 on success, negative error code on failure
5105  */
5106 static int amdgpu_ualink_sdma_entities_init(struct amdgpu_device *adev)
5107 {
5108 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5109 	struct drm_gpu_scheduler *sched;
5110 	u32 accel_id;
5111 	int i = 0, r;
5112 
5113 	dev_dbg(adev->dev, "enter\n");
5114 
5115 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5116 		sched = &adev->sdma.instance[accel_id % adev->sdma.num_instances].ring.sched;
5117 
5118 		r = drm_sched_entity_init(&remote->peer[accel_id].entity, DRM_SCHED_PRIORITY_HIGH,
5119 					  &sched, 1, NULL);
5120 		if (r)
5121 			goto out_free;
5122 		i++;
5123 	}
5124 
5125 	remote->use_lsdma = true;
5126 
5127 	dev_dbg(adev->dev, "exit\n");
5128 	return 0;
5129 
5130 out_free:
5131 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5132 		if (i--)
5133 			drm_sched_entity_destroy(&remote->peer[accel_id].entity);
5134 		else
5135 			break;
5136 	}
5137 
5138 	return r;
5139 }
5140 
5141 static void amdgpu_ualink_sdma_entities_fini(struct amdgpu_device *adev)
5142 {
5143 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5144 	u32 accel_id;
5145 
5146 	dev_dbg(adev->dev, "enter\n");
5147 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX)
5148 		drm_sched_entity_destroy(&remote->peer[accel_id].entity);
5149 	dev_dbg(adev->dev, "exit\n");
5150 }
5151 
5152 /**
5153  * amdgpu_ualink_gart_map - Allocate GART entry and map NPA address
5154  * @adev: amdgpu device pointer
5155  * @npages: Number of pages to map
5156  * @npa: NPA address to map
5157  * @mm_node: DRM memory manager node for GART allocation
5158  * @pte_flags: the GART mapping flags
5159  *
5160  * Allocates GART entries and sets up NPA address mapping without TTM BO.
5161  *
5162  * Return: 0 on success, negative error code on failure
5163  */
5164 static int amdgpu_ualink_gart_map(struct amdgpu_device *adev, u64 npages,
5165 				   u64 npa, struct drm_mm_node *mm_node,
5166 				   u64 pte_flags)
5167 {
5168 	struct ttm_resource_manager *man =
5169 				ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
5170 	struct amdgpu_gtt_mgr *mgr =
5171 				container_of(man, struct amdgpu_gtt_mgr, manager);
5172 	dma_addr_t *dma_addr;
5173 	int i, r;
5174 
5175 	dev_dbg(adev->dev, "npa 0x%llx npages 0x%llx\n", npa, npages);
5176 
5177 	if (npages > 1) {
5178 		dma_addr = kmalloc_array(npages, sizeof(*dma_addr), GFP_KERNEL);
5179 		if (!dma_addr)
5180 			return -ENOMEM;
5181 
5182 		for (i = 0; i < npages; i++)
5183 			dma_addr[i] = npa + i * AMDGPU_GPU_PAGE_SIZE;
5184 	} else {
5185 		dma_addr = (dma_addr_t *)&npa;
5186 	}
5187 
5188 	r = amdgpu_gtt_mgr_alloc_entries(mgr, mm_node, npages, DRM_MM_INSERT_BEST);
5189 	if (r)
5190 		goto out;
5191 	amdgpu_gart_bind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, dma_addr,
5192 			 pte_flags);
5193 
5194 out:
5195 	dev_dbg(adev->dev, "npa 0x%llx mapped to 0x%llx npages 0x%llx r=%d\n",
5196 		npa, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, r);
5197 
5198 	if (npages > 1)
5199 		kfree(dma_addr);
5200 	return r;
5201 }
5202 
5203 /**
5204  * amdgpu_ualink_gart_unmap - Unmap and free GART entry
5205  * @adev: amdgpu device pointer
5206  * @npages: Number of pages to unmap
5207  * @mm_node: DRM memory manager node for GART allocation
5208  *
5209  * Unbinds GART mapping and frees allocated entries.
5210  */
5211 static void amdgpu_ualink_gart_unmap(struct amdgpu_device *adev, u64 npages,
5212 				      struct drm_mm_node *mm_node)
5213 {
5214 	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
5215 	struct amdgpu_gtt_mgr *mgr =
5216 				container_of(man, struct amdgpu_gtt_mgr, manager);
5217 
5218 	dev_dbg(adev->dev, "0x%llx npages 0x%llx\n", mm_node->start << AMDGPU_GPU_PAGE_SHIFT,
5219 		npages);
5220 
5221 	if (!drm_mm_node_allocated(mm_node))
5222 		return;
5223 	amdgpu_gart_unbind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages);
5224 	amdgpu_gtt_mgr_free_entries(mgr, mm_node);
5225 }
5226 
5227 /**
5228  * amdgpu_ualink_metadata_fini - Clean up ualink metadata structures
5229  * @adev: amdgpu device pointer
5230  *
5231  * Unmaps NPA addresses and frees all allocated buffers for metadata,
5232  * ring buffers, and pointers.
5233  */
5234 static void amdgpu_ualink_metadata_fini(struct amdgpu_device *adev)
5235 {
5236 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5237 
5238 	dev_dbg(adev->dev, "accel_id %u\n", ualink_accel_id(adev));
5239 	amdgpu_bo_free_kernel(&remote->rptr_bo, &remote->rptr_gpu_addr,
5240 				      &remote->rptr_cpu_addr);
5241 	amdgpu_bo_free_kernel(&remote->ring_bo, &remote->rb_gpu_addr,
5242 			      &remote->rb_cpu_addr);
5243 	amdgpu_bo_free_kernel(&remote->metadata_bo, &remote->metadata_gpu_addr,
5244 			      &remote->metadata_cpu_addr);
5245 }
5246 
5247 /**
5248  * amdgpu_ualink_metadata_init - Initialize ualink metadata structures
5249  * @adev: amdgpu device pointer
5250  *
5251  * Allocates and initializes metadata structures, ring buffers, and write/read
5252  * pointers for multi-GPU ualink. Communicates with firmware to load metadata.
5253  *
5254  * Return: 0 on success, negative error code on failure
5255  */
5256 static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev)
5257 {
5258 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5259 	struct amdgpu_ualink_metadata *metadata;
5260 	u32 size, rb_size, wptr_size, rptr_size, metadata_size;
5261 	u64 rb_gpu_addr, wptr_gpu_addr;
5262 	u32 status, accel_id;
5263 	int r;
5264 
5265 	bitmap_copy(remote->active_accel_bits,
5266 		    adev->ualink.info->vpod.active_accel_bits,
5267 		    AMDGPU_UALINK_ACCEL_MAX);
5268 	dev_dbg(adev->dev, "%d active accelerators config in vpod\n",
5269 		bitmap_weight(remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX));
5270 
5271 	/*
5272 	 * allocate metadata entries and ring buffer for all remote GPUs,
5273 	 * to get 2MB page ring buffer NPA mapping for remote access.
5274 	 */
5275 	remote->num_accel = AMDGPU_UALINK_ACCEL_MAX;
5276 
5277 	status = adev->ualink.msg_ctl->check_status(adev);
5278 	if (status != AMDGPU_NHT_FW_ST_PREINIT &&
5279 	    status != AMDGPU_NHT_FW_ST_HALT) {
5280 		dev_dbg(adev->dev, "fw status 0x%x not preinit or halt\n", status);
5281 		return -ENODEV;
5282 	}
5283 
5284 	dev_dbg(adev->dev, "accel_id %u addr_mode %d fw status 0x%x\n",
5285 		ualink_accel_id(adev), ualink_addr_mode(adev), status);
5286 
5287 	/* Alloc metadata structure for all GPUs */
5288 	metadata_size = sizeof(struct amdgpu_ualink_metadata) * AMDGPU_UALINK_ACCEL_MAX;
5289 	metadata_size = AMDGPU_GPU_PAGE_ALIGN(metadata_size);
5290 
5291 	dev_dbg(adev->dev, "metadata size 0x%x\n", metadata_size);
5292 
5293 	/* pinned system memory */
5294 	r = amdgpu_bo_create_kernel(adev, metadata_size, PAGE_SIZE,
5295 				    AMDGPU_GEM_DOMAIN_GTT,
5296 				    &remote->metadata_bo, &remote->metadata_gpu_addr,
5297 				    &remote->metadata_cpu_addr);
5298 	if (r)
5299 		goto out;
5300 
5301 	memset(remote->metadata_cpu_addr, 0, metadata_size);
5302 	metadata = remote->metadata_cpu_addr;
5303 
5304 	dev_dbg(adev->dev, "metadata gpu address 0x%llx\n", remote->metadata_gpu_addr);
5305 
5306 	/* Allocate ring buffers, wptr for remote interrupt and shootdown */
5307 	rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
5308 	wptr_size = ualink_wptr_size(adev);
5309 	size = (rb_size + wptr_size) * remote->num_accel;
5310 	size = AMDGPU_GPU_PAGE_ALIGN(size);
5311 
5312 	dev_dbg(adev->dev, "rb_size 0x%x wptr_size 0x%x total alloc size 0x%x\n",
5313 		rb_size, wptr_size, size);
5314 
5315 	/* pinned VRAM */
5316 	r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
5317 				    AMDGPU_GEM_DOMAIN_VRAM,
5318 				    &remote->ring_bo,
5319 				    &remote->rb_gpu_addr,
5320 				    &remote->rb_cpu_addr);
5321 	if (r)
5322 		goto out;
5323 
5324 	memset(remote->rb_cpu_addr, 0, size);
5325 
5326 	dev_dbg(adev->dev, "rb gpu addr 0x%llx cpu addr 0x%p vram_start 0x%llx vram_base 0x%llx\n",
5327 		remote->rb_gpu_addr, remote->rb_cpu_addr, adev->gmc.vram_start,
5328 		adev->vm_manager.vram_base_offset);
5329 
5330 	/*
5331 	 * address aliasing mode, shared wptr pagee left is not enough for wb data,
5332 	 * wb status and rptr, alloc another BO, and then alloc npa address and map
5333 	 * to npa vm, for remote to access.
5334 	 *
5335 	 * No gart mapping required for wb data, status and rptr because this is
5336 	 * updated by firmware.
5337 	 */
5338 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
5339 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(ualink_wb_size(adev) * remote->num_accel);
5340 		dev_dbg(adev->dev, "source aliasing mode rptr_size 0x%x\n", rptr_size);
5341 
5342 		/* pinned VRAM */
5343 		r = amdgpu_bo_create_kernel(adev, rptr_size, PAGE_SIZE,
5344 					    AMDGPU_GEM_DOMAIN_VRAM,
5345 					    &remote->rptr_bo,
5346 					    &remote->rptr_gpu_addr,
5347 					    &remote->rptr_cpu_addr);
5348 		if (r)
5349 			goto out;
5350 
5351 		memset(remote->rptr_cpu_addr, 0, rptr_size);
5352 
5353 		dev_dbg(adev->dev, "source aliasing rptr gpu addr 0x%llx cpu addr 0x%p\n",
5354 			remote->rptr_gpu_addr, remote->rptr_cpu_addr);
5355 	}
5356 
5357 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
5358 		wptr_gpu_addr = remote->rb_gpu_addr + rb_size * ualink_accel_id(adev);
5359 
5360 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5361 		dev_dbg(adev->dev, "init for accel_id %u\n", accel_id);
5362 
5363 		if (accel_id == ualink_accel_id(adev))
5364 			continue;
5365 
5366 		rb_gpu_addr = remote->rb_gpu_addr + rb_size * accel_id;
5367 
5368 		if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
5369 			wptr_gpu_addr = remote->rb_gpu_addr + rb_size * remote->num_accel;
5370 			wptr_gpu_addr += wptr_size * accel_id;
5371 		}
5372 
5373 		metadata[accel_id].header = AMDGPU_UALINK_METADATA_HEADER;
5374 		metadata[accel_id].rb_size = fls(AMDGPU_UALINK_RB_SIZE / 64) - 1;
5375 
5376 		/*
5377 		 * remote shootdown type is 0, remote command type is 1
5378 		 * with 2MB ring buffer mapping, remote shootdown ring NPA is before interrupt
5379 		 */
5380 		metadata[accel_id].ri_rb = rb_gpu_addr + AMDGPU_UALINK_RB_SIZE;
5381 		metadata[accel_id].tlb_inv_rb = rb_gpu_addr;
5382 
5383 		metadata[accel_id].tailptr_ri = wptr_gpu_addr +
5384 						ualink_wptr_offset(adev, accel_id);
5385 		metadata[accel_id].tailptr_tlb_inv = wptr_gpu_addr +
5386 						     ualink_tlb_wptr_offset(adev, accel_id);
5387 
5388 		dev_dbg(adev->dev, "init from accel_id %u to accel_id %u, rb_size 0x%x\n",
5389 			accel_id, ualink_accel_id(adev), metadata[accel_id].rb_size);
5390 		dev_dbg(adev->dev, "rb 0x%llx tlb rb 0x%llx\n",
5391 			metadata[accel_id].ri_rb, metadata[accel_id].tlb_inv_rb);
5392 		dev_dbg(adev->dev, "rb wptr at 0x%llx tlb wptr at 0x%llx\n",
5393 			metadata[accel_id].tailptr_ri, metadata[accel_id].tailptr_tlb_inv);
5394 	}
5395 
5396 	r = adev->ualink.msg_ctl->send_metadata(adev,
5397 						    remote->metadata_gpu_addr,
5398 						    AMDGPU_UALINK_ACCEL_MAX << 8);
5399 
5400 out:
5401 	if (r)
5402 		amdgpu_ualink_metadata_fini(adev);
5403 
5404 	dev_dbg(adev->dev, "ret 0x%x\n", r);
5405 	return r;
5406 }
5407 
5408 /*
5409  * nHT Firmware Error code
5410  *
5411  * Success				0x0
5412  * FIFO overflow			0x1
5413  * Invalid RB command			0x2
5414  * Timeout				0x3
5415  * Invalid metadata entry		0x4
5416  * RB overflow				0x5
5417  * Invalid wptr address			0x6
5418  * Interrupt cookie send failure	0x7
5419  * Invalid RB address			0x8
5420  * Invalid writeback address		0x9
5421  */
5422 static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status)
5423 {
5424 	const char *fw_err_code_msg[] = {
5425 		"Unknown FW error status",
5426 		"FIFO overflow",		/* error code 1 */
5427 		"Invalid RB command",
5428 		"Timeout",
5429 		"Invalid metadata entry",
5430 		"RB overflow",
5431 		"Invalid wptr address",
5432 		"Interrupt cookie send failure",
5433 		"Invalid RB address",
5434 		"Invalid writeback address"	/* error code 9 */
5435 		};
5436 
5437 	if (status >= ARRAY_SIZE(fw_err_code_msg))
5438 		status = 0;
5439 
5440 	dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]);
5441 	return;
5442 }
5443 
5444 /**
5445  * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion
5446  * @remote_accel_id: remote accelator id to wait for reply
5447  * @adev: amdgpu device pointer
5448  * @wb_cpu: CPU virtual address of writeback buffer
5449  * @seq: Sequence number to wait for
5450  *
5451  * Polls the writeback buffer waiting for the sequence number to reach or
5452  * exceed the expected value. The writeback buffer contains status and data
5453  * fields updated by firmware to indicate completion and error conditions.
5454  *
5455  * Return: 0 on success
5456  *         -ETIME on timeout
5457  *         -ECOMM on firmware return error status
5458  */
5459 static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev,
5460 					      u32 remote_accel_id,
5461 					      struct amdgpu_ualink_wb *wb_cpu,
5462 					      u32 seq)
5463 {
5464 	/* 2 seconds timeout, long enough for FW to reply */
5465 	ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC);
5466 	u32 status, data;
5467 	u64 rptr;
5468 
5469 	while (true) {
5470 		data = READ_ONCE(wb_cpu->data);
5471 		if (data >= seq) {
5472 			status = READ_ONCE(wb_cpu->status);
5473 			rptr = READ_ONCE(wb_cpu->rptr);
5474 			break;
5475 		}
5476 
5477 		if (ktime_after(ktime_get(), timeout)) {
5478 			dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id);
5479 			return -ETIME;
5480 		}
5481 
5482 		usleep_range(10, 50);
5483 	}
5484 
5485 	if (status) {
5486 		dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n",
5487 			remote_accel_id, status, data, seq, rptr);
5488 
5489 		amdgpu_ualink_remote_error(adev, status);
5490 		return -ECOMM;
5491 	}
5492 
5493 	dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n",
5494 		remote_accel_id, seq, data, rptr);
5495 	return 0;
5496 }
5497 
5498 #define AMDGPU_UALINK_REMOTE_OP_TLB_INV		0x1
5499 #define AMDGPU_UALINK_REMOTE_OP_INT		0x2
5500 #define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB	0x3
5501 
5502 static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data,
5503 					 u32 dw0,  u32 dw1, u32 dw2, u32 dw3)
5504 {
5505 	u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT;
5506 	u32 *cpu_addr = *cpu_addr_p;
5507 	u32 flush_type = dw0;
5508 	u32 size_in_pages = dw3;
5509 	u32 cmd;
5510 
5511 	cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV;
5512 	cmd |= 1 << 28;	/* headptr update */
5513 	cmd |= 1 << 29;	/* wb enable */
5514 
5515 	*cpu_addr++ = cmd;
5516 	*cpu_addr++ = wb_data;
5517 	*cpu_addr++ = 0;
5518 	*cpu_addr++ = 0;
5519 	*cpu_addr++ = 0;
5520 	*cpu_addr++ = 0;
5521 	*cpu_addr++ = 0; /* metadata */
5522 	*cpu_addr++ = 0; /* metadata */
5523 
5524 	pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages);
5525 
5526 	/* Always shootdown everything with full address size s-field coding */
5527 	addr = GENMASK_ULL(51, 11);
5528 
5529 	*cpu_addr++ = lower_32_bits(addr) | flush_type;
5530 	*cpu_addr++ = upper_32_bits(addr);
5531 
5532 	*cpu_addr++ = 0;
5533 	*cpu_addr++ = 0;
5534 	*cpu_addr++ = 0;
5535 	*cpu_addr++ = 0;
5536 	*cpu_addr++ = 0;
5537 	*cpu_addr++ = 0;
5538 	*cpu_addr_p = cpu_addr;
5539 }
5540 
5541 static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data,
5542 					 u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5543 {
5544 	u32 *cpu_addr = *cpu_addr_p;
5545 	u32 cmd;
5546 
5547 	cmd = AMDGPU_UALINK_REMOTE_OP_INT;
5548 	cmd |= 1 << 28;	/* headptr_update */
5549 	cmd |= 1 << 29; /* wb enable */
5550 
5551 	*cpu_addr++ = cmd;
5552 	*cpu_addr++ = wb_data;
5553 	*cpu_addr++ = 0;
5554 	*cpu_addr++ = 0;
5555 	*cpu_addr++ = 0;
5556 	*cpu_addr++ = 0;
5557 	*cpu_addr++ = 0; /* metadata */
5558 	*cpu_addr++ = 0; /* metadata */
5559 	/* updated by f/w, overwrite PASID with GPU ID */
5560 	*cpu_addr++ = 0;
5561 	*cpu_addr++ = 0;
5562 	*cpu_addr++ = 0;
5563 	*cpu_addr++ = 0;
5564 	/* IH ContextID 4 dwords */
5565 	*cpu_addr++ = dw0;
5566 	*cpu_addr++ = dw1;
5567 	*cpu_addr++ = dw2;
5568 	*cpu_addr++ = dw3;
5569 	*cpu_addr_p = cpu_addr;
5570 }
5571 
5572 static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data,
5573 					      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5574 {
5575 	u32 *cpu_addr = *cpu_addr_p;
5576 	u32 cmd;
5577 
5578 	cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB;
5579 	cmd |= 1 << 28;	/* headptr update */
5580 	cmd |= 1 << 29;	/* wb enable */
5581 
5582 	*cpu_addr++ = cmd;
5583 	*cpu_addr++ = wb_data;
5584 	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb_npa */
5585 	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb_npa */
5586 	*cpu_addr++ = 0;
5587 	*cpu_addr++ = 0;
5588 	*cpu_addr++ = 0; /* metadata */
5589 	*cpu_addr++ = 0; /* metadata */
5590 	*cpu_addr++ = 0;
5591 	*cpu_addr++ = 0;
5592 	*cpu_addr++ = 0;
5593 	*cpu_addr++ = 0;
5594 	*cpu_addr++ = 0;
5595 	*cpu_addr++ = 0;
5596 	*cpu_addr++ = 0;
5597 	*cpu_addr++ = 0;
5598 	*cpu_addr_p = cpu_addr;
5599 }
5600 
5601 /*
5602  * Function proto to generate packet using above different emit_ functions
5603  */
5604 typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0,
5605 				   u32 dw1, u32 dw2, u32 dw3);
5606 
5607 /*
5608  * Refresh rptr from the writeback and check the ring has room for one more
5609  * command. The caller must hold the peer lock.
5610  *
5611  * Return 0 on success or a negative error code.
5612  */
5613 static int amdgpu_ualink_ring_check_space(struct amdgpu_device *adev,
5614 					  u32 remote_accel_id,
5615 					  struct amdgpu_ualink_ring *ring,
5616 					  struct amdgpu_ualink_wb *wb_cpu)
5617 {
5618 	ring->rptr = READ_ONCE(wb_cpu->rptr);
5619 
5620 	if (WARN_ON_ONCE(ring->rptr > ring->wptr)) {
5621 		dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n",
5622 			remote_accel_id, ring->wptr, ring->rptr);
5623 		return -EFAULT;
5624 	}
5625 
5626 	if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) {
5627 		dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n",
5628 			remote_accel_id, ring->wptr, ring->rptr);
5629 		return -ENOSPC;
5630 	}
5631 
5632 	return 0;
5633 }
5634 
5635 static int amdgpu_ualink_send_command(struct amdgpu_device *adev,
5636 				      u32 remote_accel_id,
5637 				      struct amdgpu_ualink_ring *ring,
5638 				      struct amdgpu_ualink_wb *wb_cpu,
5639 				      ualink_emit_packet emit_func,
5640 				      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5641 {
5642 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5643 	struct amdgpu_ualink_peer *peer;
5644 	struct amdgpu_ring *sdma_ring;
5645 	struct dma_fence *fence;
5646 	struct amdgpu_job *job;
5647 	struct amdgpu_ib *ib;
5648 	u32 seq, ndw, ndw_copy_cmd, rem;
5649 	u64 doorbell_npa_gart;
5650 	u32 *src_cpu;
5651 	u64 src;
5652 	int r;
5653 
5654 	/* Refuse sends unless active */
5655 	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
5656 		return -ESHUTDOWN;
5657 
5658 	peer = &remote->peer[remote_accel_id];
5659 
5660 	r = amdgpu_ualink_ring_check_space(adev, remote_accel_id, ring, wb_cpu);
5661 	if (r)
5662 		goto out;
5663 
5664 	/*
5665 	 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell
5666 	 *
5667 	 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to
5668 	 * ensure the ordering of 3 sdma copy commands execution.
5669 	 */
5670 
5671 	/* buffer after sdma commands, starting at 8 dwords boundary */
5672 	ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8);
5673 
5674 	 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */
5675 	ndw = ndw_copy_cmd + 16 + 2 + 1;
5676 	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
5677 				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
5678 				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
5679 	if (r)
5680 		goto out;
5681 
5682 	ib = &job->ibs[0];
5683 	src = ib->gpu_addr + ndw_copy_cmd * 4;
5684 	src_cpu = ib->ptr + ndw_copy_cmd;
5685 
5686 	seq = ++ring->seq;
5687 
5688 	div_u64_rem(ring->wptr, ring->rb_size, &rem);
5689 	dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n",
5690 		src, ring->rb_npa_gart + rem * 64,
5691 		ring->wptr_npa_gart, ring->doorbell_npa_gart, seq);
5692 
5693 	dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr);
5694 
5695 	/* remote command packet */
5696 	emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3);
5697 
5698 	/* wptr value */
5699 	*src_cpu++ = lower_32_bits(ring->wptr + 1);
5700 	*src_cpu++ = upper_32_bits(ring->wptr + 1);
5701 
5702 	/* doorbell value */
5703 	*src_cpu++ = lower_32_bits(ring->wptr + 1);
5704 
5705 	/*
5706 	 * Add an offset to the doorbell address that cycles through different
5707 	 * values of bit 11:8 to use different DXS ports.
5708 	 */
5709 	peer->dxs_port = (peer->dxs_port + 1) & 0xF;
5710 	doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8);
5711 	dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n",
5712 		peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart);
5713 
5714 	dev_dbg(adev->dev, "use %s\n", remote->use_lsdma ? "lsdma" : "sdma");
5715 
5716 	if (remote->use_lsdma) {
5717 		div_u64_rem(ring->wptr, ring->rb_size, &rem);
5718 		r = amdgpu_lsdma_copy_mem(adev, src,
5719 					  ring->rb_npa_gart + rem * 64,
5720 					  64);
5721 		if (!r)
5722 			r = amdgpu_lsdma_copy_mem(adev, src + 64,
5723 						  ring->wptr_npa_gart, 8);
5724 		if (!r)
5725 			r = amdgpu_lsdma_copy_mem(adev, src + 72,
5726 						  doorbell_npa_gart, 4);
5727 
5728 		amdgpu_job_free(job);
5729 
5730 		/* Copy failed: command never reached the remote, so no
5731 		 * writeback will arrive. Return now instead of polling the
5732 		 * completion for the full remote timeout.
5733 		 */
5734 		if (r) {
5735 			dev_dbg(adev->dev,
5736 				"remote %u lsdma copy failed (r %d), skip completion wait\n",
5737 				remote_accel_id, r);
5738 			return r;
5739 		}
5740 
5741 		goto out_wait_complete;
5742 	}
5743 
5744 	sdma_ring = &adev->sdma.instance[0].ring;
5745 
5746 	/*
5747 	 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands
5748 	 * execution doesn't overlap, doorbell update is after data and wptr update
5749 	 * finished.
5750 	 */
5751 	div_u64_rem(ring->wptr, ring->rb_size, &rem);
5752 	amdgpu_emit_copy_buffer(adev, ib, src,
5753 				ring->rb_npa_gart + rem * 64,
5754 				64, 0);
5755 
5756 	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
5757 
5758 	amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0);
5759 
5760 	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
5761 
5762 	amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0);
5763 
5764 	amdgpu_ring_pad_ib(sdma_ring, ib);
5765 	WARN_ON(ib->length_dw > ndw_copy_cmd);
5766 
5767 	fence = amdgpu_job_submit(job);
5768 	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
5769 	dma_fence_put(fence);
5770 	if (r <= 0) {
5771 		dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
5772 			remote_accel_id, r);
5773 		if (r == 0)
5774 			r = -ETIME;
5775 		goto out;
5776 	}
5777 
5778 out_wait_complete:
5779 	/*
5780 	 * poll remote completion writeback data
5781 	 */
5782 	r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq);
5783 
5784 	/*
5785 	 * Advance wptr unless the remote timed out, even if the FW returned an
5786 	 * error status. On timeout the caller resets the connection, which
5787 	 * resyncs wptr from the remote in amdgpu_ualink_setup_connection.
5788 	 */
5789 	if (r != -ETIME)
5790 		ring->wptr++;
5791 
5792 out:
5793 	dev_dbg(adev->dev, "ret r = %d\n", r);
5794 	return r;
5795 }
5796 
5797 static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
5798 				      u32 remote_accel_id,
5799 				      struct amdgpu_ualink_wb **wb_cpu,
5800 				      u64 *wb_npa, u32 type)
5801 {
5802 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5803 	u32 accel_id = ualink_accel_id(adev);
5804 	u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
5805 	uintptr_t wb;
5806 	u64 npa;
5807 	u32 offset = 0;
5808 
5809 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
5810 		wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel;
5811 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR,
5812 					     remote_accel_id, accel_id);
5813 	} else {
5814 		wb = (uintptr_t)remote->rptr_cpu_addr;
5815 		npa = remote->rptr_npa;
5816 		npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT;
5817 	}
5818 
5819 	if (type == RB_TYPE_TLB_INV)
5820 		offset = ualink_tlb_wb_offset(adev, remote_accel_id);
5821 	else if (type == RB_TYPE_REMOTE_INTERRUPT)
5822 		offset = ualink_wb_offset(adev, remote_accel_id);
5823 
5824 	*wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset);
5825 	if (wb_npa)
5826 		*wb_npa = npa + offset;
5827 
5828 	dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id,
5829 		remote_accel_id, npa + offset);
5830 }
5831 
5832 struct amdgpu_ualink_wptr_fence_cb {
5833 	struct dma_fence_cb base;
5834 	u64 *wptr_cpu;
5835 	u64 result;
5836 };
5837 
5838 static void amdgpu_ualink_read_wptr_fence_cb(struct dma_fence *fence,
5839 					     struct dma_fence_cb *cb)
5840 {
5841 	struct amdgpu_ualink_wptr_fence_cb *wptr_cb;
5842 
5843 	wptr_cb = container_of(cb, typeof(*wptr_cb), base);
5844 	wptr_cb->result = *wptr_cb->wptr_cpu;
5845 }
5846 
5847 static int amdgpu_ualink_read_remote_wptr(struct amdgpu_device *adev,
5848 					  u32 remote_accel_id,
5849 					  struct amdgpu_ualink_ring *ring,
5850 					  u64 *wptr)
5851 {
5852 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5853 	struct amdgpu_ualink_peer *peer = &remote->peer[remote_accel_id];
5854 	struct amdgpu_ualink_wptr_fence_cb wptr_cb = { };
5855 	struct amdgpu_ring *sdma_ring;
5856 	struct dma_fence *fence;
5857 	struct amdgpu_job *job;
5858 	struct amdgpu_ib *ib;
5859 	u32 ndw, ndw_copy_cmd;
5860 	u64 wptr_gpu, *wptr_cpu;
5861 	int r;
5862 
5863 	*wptr = 0;
5864 
5865 	/* Refuse remote access unless active */
5866 	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
5867 		return -ESHUTDOWN;
5868 
5869 	/* 1 sdma copy command to read wptr */
5870 	ndw_copy_cmd = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
5871 
5872 	/* wptr 2 dwords */
5873 	ndw = ndw_copy_cmd + 2;
5874 	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
5875 				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
5876 				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
5877 	if (r)
5878 		return r;
5879 
5880 	ib = &job->ibs[0];
5881 	wptr_gpu = ib->gpu_addr + ndw_copy_cmd * 4;
5882 	wptr_cpu = (u64 *)(ib->ptr + ndw_copy_cmd);
5883 
5884 	dev_dbg(adev->dev, "read wptr from remote %u npa gart wptr 0x%llx use %s\n",
5885 		remote_accel_id, ring->wptr_npa_gart,
5886 		remote->use_lsdma ? "lsdma" : "sdma");
5887 
5888 	if (remote->use_lsdma) {
5889 		r = amdgpu_lsdma_copy_mem(adev, ring->wptr_npa_gart, wptr_gpu, 8);
5890 		if (!r)
5891 			*wptr = *wptr_cpu;
5892 		amdgpu_job_free(job);
5893 		goto out;
5894 	}
5895 
5896 	wptr_cb.wptr_cpu = wptr_cpu;
5897 
5898 	amdgpu_emit_copy_buffer(adev, ib, ring->wptr_npa_gart, wptr_gpu, 8, 0);
5899 
5900 	sdma_ring = &adev->sdma.instance[0].ring;
5901 	amdgpu_ring_pad_ib(sdma_ring, ib);
5902 	WARN_ON(ib->length_dw > ndw_copy_cmd);
5903 
5904 	fence = amdgpu_job_submit(job);
5905 
5906 	r = dma_fence_add_callback(fence, &wptr_cb.base,
5907 				   amdgpu_ualink_read_wptr_fence_cb);
5908 	if (r == -ENOENT) {
5909 		/* Fence already signaled; the callback won't run, do it inline. */
5910 		amdgpu_ualink_read_wptr_fence_cb(fence, &wptr_cb.base);
5911 		r = 0;
5912 		goto out_put_fence;
5913 	} else if (r) {
5914 		/* -EINVAL: NULL fence or func, not expected here. */
5915 		goto out_put_fence;
5916 	}
5917 
5918 	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
5919 	if (r > 0) {
5920 		r = 0;	/* read wptr successfully */
5921 		goto out_put_fence;
5922 	}
5923 
5924 	dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
5925 		remote_accel_id, r);
5926 
5927 	if (r == 0)
5928 		r = -ETIME;
5929 
5930 	/*
5931 	 * Timed out: the callback is still armed and wptr_cb is on our stack.
5932 	 * Cancel it so it can't fire after we return and write into the freed
5933 	 * stack frame.
5934 	 */
5935 	dma_fence_remove_callback(fence, &wptr_cb.base);
5936 
5937 out_put_fence:
5938 	if (!r)
5939 		*wptr = wptr_cb.result;
5940 	dma_fence_put(fence);
5941 
5942 out:
5943 	dev_dbg(adev->dev, "remote %u wptr 0x%llx return %d\n",
5944 		remote_accel_id, *wptr, r);
5945 	return r;
5946 }
5947 
5948 /**
5949  * amdgpu_ualink_ring_resync_ptrs - sync local ring pointers from remote
5950  * @adev: amdgpu device pointer
5951  * @remote_accel_id: remote accelerator ID
5952  *
5953  * After a reboot on either side the two ends disagree on wptr. Read the remote
5954  * wptr of both peer rings and adopt it as the local wptr and rptr.
5955  *
5956  * Return 0 if both rings already match, 1 if they were resynced, or a negative
5957  * error code if a remote read failed, leaving the rings untouched.
5958  */
5959 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
5960 					  u32 remote_accel_id)
5961 {
5962 	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
5963 	struct amdgpu_ualink_wb *int_wb, *tlb_wb;
5964 	u64 int_wptr, tlb_wptr;
5965 	int r;
5966 
5967 	/* ring->wptr and ring->rptr are owned by the send path. */
5968 	guard(mutex)(&peer->lock);
5969 
5970 	/* Read both before updating either, so a failure changes nothing. */
5971 	r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
5972 					   &peer->interrupt, &int_wptr);
5973 	if (!r)
5974 		r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
5975 						   &peer->shootdown, &tlb_wptr);
5976 	if (r) {
5977 		dev_dbg(adev->dev, "accel_id %u read remote %u wptr failed %d\n",
5978 			ualink_accel_id(adev), remote_accel_id, r);
5979 		return r;
5980 	}
5981 
5982 	if (peer->interrupt.wptr == int_wptr && peer->shootdown.wptr == tlb_wptr) {
5983 		dev_dbg(adev->dev, "accel_id %u wptr 0x%llx 0x%llx is same as remote %u\n",
5984 			ualink_accel_id(adev), int_wptr, tlb_wptr, remote_accel_id);
5985 		return 0;
5986 	}
5987 
5988 	dev_dbg(adev->dev,
5989 		"accel_id %u wptr 0x%llx 0x%llx sync to remote %u wptr 0x%llx 0x%llx\n",
5990 		ualink_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr,
5991 		remote_accel_id, int_wptr, tlb_wptr);
5992 
5993 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL,
5994 				  RB_TYPE_REMOTE_INTERRUPT);
5995 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL,
5996 				  RB_TYPE_TLB_INV);
5997 
5998 	peer->interrupt.wptr = int_wptr;
5999 	peer->interrupt.rptr = int_wptr;
6000 	peer->shootdown.wptr = tlb_wptr;
6001 	peer->shootdown.rptr = tlb_wptr;
6002 
6003 	/* Drop the stale writeback rptr, it disagrees with the resynced wptr. */
6004 	WRITE_ONCE(int_wb->rptr, int_wptr);
6005 	WRITE_ONCE(tlb_wb->rptr, tlb_wptr);
6006 	return 1;
6007 }
6008 
6009 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
6010 					   u32 remote_accel_id)
6011 {
6012 	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
6013 
6014 	dev_dbg(adev->dev, "remote accel_id %u\n", remote_accel_id);
6015 
6016 	guard(mutex)(&peer->lock);
6017 
6018 	peer->interrupt.ready = false;
6019 	peer->shootdown.ready = false;
6020 }
6021 
6022 static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev,
6023 					   u32 remote_accel_id, u32 ring_type)
6024 {
6025 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6026 	struct amdgpu_ualink_peer *peer;
6027 	struct amdgpu_ualink_ring *ring;
6028 	struct amdgpu_ualink_wb *wb_cpu;
6029 	u64 wb_npa;
6030 	int r;
6031 
6032 	peer = &remote->peer[remote_accel_id];
6033 	if (ring_type == RB_TYPE_REMOTE_INTERRUPT)
6034 		ring = &peer->interrupt;
6035 	else if (ring_type == RB_TYPE_TLB_INV)
6036 		ring = &peer->shootdown;
6037 	else
6038 		return -EINVAL;
6039 
6040 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa,
6041 				  ring_type);
6042 
6043 	r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu,
6044 				       amdgpu_ualink_emit_update_wb_addr,
6045 				       upper_32_bits(wb_npa),
6046 				       lower_32_bits(wb_npa),
6047 				       0, 0);
6048 	if (!r) {
6049 		ring->wb_cpu = wb_cpu;
6050 		ring->ready = true;
6051 	}
6052 	return r;
6053 }
6054 
6055 /**
6056  * amdgpu_ualink_remote_shootdown - Send a remote TLB invalidation request
6057  * @adev: amdgpu device pointer
6058  * @remote_accel_id: accelerator ID of the remote GPU to shootdown
6059  * @addr: page-aligned address to invalidate
6060  * @size_in_pages: number of pages to invalidate, 0 to invalidate entire TLB
6061  * @flush_type: type of TLB flush to perform
6062  *
6063  * Sends a remote TLB shootdown command to the specified peer GPU via the
6064  * UALink ring buffer. The address and size are translated using the S-field
6065  * encoding before being written to the ring.
6066  *
6067  * Return: 0 on success, negative error code on failure
6068  */
6069 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
6070 				   u32 remote_accel_id, u64 addr,
6071 				   u32 size_in_pages, u32 flush_type)
6072 {
6073 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6074 	struct amdgpu_ualink_peer *peer;
6075 	struct amdgpu_ualink_ring *ring;
6076 	int r;
6077 
6078 	dev_dbg(adev->dev, "skip remote shootdown to remote_accel_id %u\n",
6079 		remote_accel_id);
6080 	return 0;
6081 
6082 	peer = &remote->peer[remote_accel_id];
6083 	ring = &peer->shootdown;
6084 
6085 	guard(mutex)(&peer->lock);
6086 
6087 	if (!ring->ready) {
6088 		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
6089 		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
6090 						    RB_TYPE_TLB_INV);
6091 		if (r)
6092 			return r;
6093 	}
6094 
6095 	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
6096 					  ring->wb_cpu,
6097 					  amdgpu_ualink_emit_shootdown,
6098 					  flush_type, upper_32_bits(addr),
6099 					  lower_32_bits(addr), size_in_pages);
6100 }
6101 
6102 /**
6103  * amdgpu_ualink_remote_interrupt - Send a remote interrupt to a peer GPU
6104  * @adev: amdgpu device pointer
6105  * @remote_accel_id: accelerator ID of the remote GPU to interrupt
6106  * @dw0: IH context ID dword 0
6107  * @dw1: IH context ID dword 1
6108  * @dw2: IH context ID dword 2
6109  * @dw3: IH context ID dword 3
6110  *
6111  * Sends a remote interrupt command to the specified peer GPU via the
6112  * UALink ring buffer. The four context ID dwords are delivered to the
6113  * remote GPU's interrupt handler.
6114  *
6115  * Return: 0 on success, negative error code on failure
6116  */
6117 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
6118 				   u32 remote_accel_id, u32 dw0, u32 dw1,
6119 				   u32 dw2, u32 dw3)
6120 {
6121 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6122 	struct amdgpu_ualink_peer *peer;
6123 	struct amdgpu_ualink_ring *ring;
6124 	int r;
6125 
6126 	peer = &remote->peer[remote_accel_id];
6127 	ring = &peer->interrupt;
6128 
6129 	guard(mutex)(&peer->lock);
6130 
6131 	if (!ring->ready) {
6132 		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
6133 		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
6134 						    RB_TYPE_REMOTE_INTERRUPT);
6135 		if (r)
6136 			return r;
6137 	}
6138 
6139 	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
6140 					  ring->wb_cpu,
6141 					  amdgpu_ualink_emit_interrupt,
6142 					  dw0, dw1, dw2, dw3);
6143 }
6144 
6145 /**
6146  * amdgpu_ualink_peer_remote_init - Initialize remote peer GPU connections
6147  * @adev: amdgpu device pointer
6148  *
6149  * Sets up ring buffers, GART mappings, and control structures for
6150  * communicating with remote GPUs in the fabric.
6151  *
6152  * Return: 0 on success, negative error code on failure
6153  */
6154 static int amdgpu_ualink_peer_remote_init(struct amdgpu_device *adev)
6155 {
6156 	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
6157 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6158 	struct amdgpu_bo *bo = remote->ring_bo;
6159 	struct amdgpu_ualink_peer *peer;
6160 	struct amdgpu_ualink_ring *interrupt;
6161 	struct amdgpu_ualink_ring *shootdown;
6162 	u32 src_accel_id = ualink_accel_id(adev);
6163 	u32 dst_accel_id;
6164 	u64 npa, flags;
6165 	int r = 0;
6166 
6167 	/* NPA mapping PTE flags VSCT = 0011 */
6168 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, bo->tbo.resource);
6169 	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | AMDGPU_PTE_BUS_ATOMICS;
6170 	flags &= ~AMDGPU_PTE_VALID;
6171 	flags &= ~AMDGPU_PTE_SYSTEM;
6172 	/* PTE.X=0 turn off RPC checks for RBs, wptr and doorbell NPA address */
6173 	flags &= ~AMDGPU_PTE_EXECUTABLE;
6174 
6175 	dev_dbg(adev->dev, "src_accel_id %u gart mapping flags 0x%llx\n",
6176 		src_accel_id, flags);
6177 
6178 	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
6179 		if (dst_accel_id == src_accel_id)
6180 			continue;
6181 
6182 		dev_dbg(adev->dev, "peer_remote to dst_accel_id %u\n", dst_accel_id);
6183 
6184 		peer = &remote->peer[dst_accel_id];
6185 		mutex_init(&peer->lock);
6186 
6187 		interrupt = &peer->interrupt;
6188 		shootdown = &peer->shootdown;
6189 
6190 		/* ring buffer npa gart mapping */
6191 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
6192 						   src_accel_id, dst_accel_id);
6193 		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &interrupt->mm_node_rb, flags);
6194 		if (r)
6195 			break;
6196 		interrupt->rb_npa_gart = adev->gmc.gart_start +
6197 					 (interrupt->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
6198 
6199 		dev_dbg(adev->dev, "rb npa 0x%llx mapped on gart 0x%llx\n",
6200 			npa, interrupt->rb_npa_gart);
6201 
6202 		/* tlb invalidate ring buffer npa gart mapping */
6203 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TLB_INV, src_accel_id,
6204 						  dst_accel_id);
6205 		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &shootdown->mm_node_rb, flags);
6206 		if (r)
6207 			break;
6208 		shootdown->rb_npa_gart = adev->gmc.gart_start +
6209 					 (shootdown->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
6210 
6211 		dev_dbg(adev->dev, "tlb rb npa 0x%llx mapped on gart 0x%llx\n",
6212 			npa, shootdown->rb_npa_gart);
6213 
6214 		/* wptr npa gart mapping */
6215 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TAILPTR, src_accel_id,
6216 						  dst_accel_id);
6217 		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_wptr, flags);
6218 		if (r)
6219 			break;
6220 
6221 		interrupt->wptr_npa_gart = adev->gmc.gart_start +
6222 					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
6223 					    + ualink_wptr_offset(adev, src_accel_id);
6224 		shootdown->wptr_npa_gart = adev->gmc.gart_start +
6225 					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
6226 					    + ualink_tlb_wptr_offset(adev, src_accel_id);
6227 
6228 		dev_dbg(adev->dev, "rb wptr npa 0x%llx on gart 0x%llx, tlb wptr on gart 0x%llx\n",
6229 			npa, interrupt->wptr_npa_gart, shootdown->wptr_npa_gart);
6230 
6231 		/*
6232 		 * doorbell npa gart mapping, doorbell NPA address coding
6233 		 *
6234 		 * ReqAddr[51:40] = 0xFFF
6235 		 * ReqAddr[39:30] = GPU accel_id
6236 		 * ReqAddr[29:12] = 18'h00000
6237 		 * ReqAddr[11:8]  = Random value
6238 		 * ReqAddr[7:6]   = PMI Index
6239 		 *     0 - PMI0 Remote Interrupt
6240 		 *     1 - PMI1 Remote TLB Shootdown
6241 		 * ReqAddr[5:0]   = 6'h0
6242 		 */
6243 		npa = 0xFFFULL << 40 | (u64)dst_accel_id << 30;
6244 		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_doorbell, flags);
6245 		if (r)
6246 			break;
6247 
6248 		interrupt->doorbell_npa_gart = adev->gmc.gart_start +
6249 			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT);
6250 
6251 		dev_dbg(adev->dev, "rb doorbell npa 0x%llx mapped on gart 0x%llx\n",
6252 			npa, interrupt->doorbell_npa_gart);
6253 
6254 		shootdown->doorbell_npa_gart = adev->gmc.gart_start+
6255 			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT) + (1 << 6);
6256 
6257 		dev_dbg(adev->dev, "tlb doorbell npa 0x%llx mapped on gart 0x%llx\n",
6258 			npa, shootdown->doorbell_npa_gart);
6259 
6260 		interrupt->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
6261 		interrupt->wptr = 0;
6262 		interrupt->rptr = 0;
6263 		interrupt->seq = 0;
6264 
6265 		shootdown->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
6266 		shootdown->wptr = 0;
6267 		shootdown->rptr = 0;
6268 		shootdown->seq = 0;
6269 	}
6270 
6271 	dev_dbg(adev->dev, "peer remote init done r=%d\n", r);
6272 	return r;
6273 }
6274 
6275 /**
6276  * amdgpu_ualink_peer_remote_fini - Clean up remote peer GPU connections
6277  * @adev: amdgpu device pointer
6278  *
6279  * Unmaps GART entries and frees resources for remote GPU communication.
6280  */
6281 static void amdgpu_ualink_peer_remote_fini(struct amdgpu_device *adev)
6282 {
6283 	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
6284 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6285 	u32 src_accel_id = ualink_accel_id(adev);
6286 	u32 dst_accel_id;
6287 	struct amdgpu_ualink_peer *peer;
6288 	struct amdgpu_ualink_ring *ring;
6289 
6290 	dev_dbg(adev->dev, "src_accel_id %u\n", src_accel_id);
6291 
6292 	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
6293 		dev_dbg(adev->dev, "dst_accel_id %u\n", dst_accel_id);
6294 		if (dst_accel_id == src_accel_id)
6295 			continue;
6296 
6297 		peer = &remote->peer[dst_accel_id];
6298 		mutex_destroy(&peer->lock);
6299 
6300 		ring = &peer->interrupt;
6301 		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
6302 		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_wptr);
6303 		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_doorbell);
6304 
6305 		ring = &peer->shootdown;
6306 		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
6307 	}
6308 }
6309 
6310 /**
6311  * amdgpu_ualink_sw_init - Initialize UALink software resources
6312  * @adev: amdgpu device pointer
6313  *
6314  * Performs full UALink software initialization including metadata allocation,
6315  * NPA address mapping, SDMA scheduler entity setup, and remote peer GPU
6316  * connection initialization. On failure, all previously initialized resources
6317  * are cleaned up.
6318  *
6319  * Return: 0 on success, negative error code on failure
6320  */
6321 int amdgpu_ualink_sw_init(struct amdgpu_device *adev)
6322 {
6323 	struct amdgpu_ualink_remote *remote;
6324 	int r;
6325 
6326 	remote = kzalloc(sizeof(*remote), GFP_KERNEL);
6327 	if (!remote)
6328 		return -ENOMEM;
6329 	adev->ualink.remote = remote;
6330 
6331 	r = amdgpu_ualink_metadata_init(adev);
6332 	if (r)
6333 		goto out;
6334 
6335 	r = amdgpu_ualink_metadata_npa_mapping(adev);
6336 	if (r)
6337 		goto out_metadata_fini;
6338 
6339 	r = amdgpu_ualink_sdma_entities_init(adev);
6340 	if (r)
6341 		goto out_npa_unmap;
6342 
6343 	r = amdgpu_ualink_peer_remote_init(adev);
6344 	if (r)
6345 		goto out_sdma_entities_fini;
6346 
6347 	dev_dbg(adev->dev, "ualink sw init succeed\n");
6348 	return 0;
6349 
6350 out_sdma_entities_fini:
6351 	amdgpu_ualink_sdma_entities_fini(adev);
6352 out_npa_unmap:
6353 	amdgpu_ualink_metadata_npa_unmapping(adev);
6354 out_metadata_fini:
6355 	amdgpu_ualink_metadata_fini(adev);
6356 out:
6357 	kfree(adev->ualink.remote);
6358 	adev->ualink.remote = NULL;
6359 	dev_dbg(adev->dev, "ualink sw init failed %d\n", r);
6360 	return r;
6361 }
6362 
6363 /**
6364  * amdgpu_ualink_sw_fini - Tear down UALink software resources
6365  * @adev: amdgpu device pointer
6366  *
6367  * Cleans up all UALink software resources in reverse order of initialization:
6368  * remote peer connections, SDMA entities, NPA mappings, and metadata.
6369  */
6370 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev)
6371 {
6372 	dev_dbg(adev->dev, "halt accel_id %u addr_mode %d\n", ualink_accel_id(adev),
6373 		ualink_addr_mode(adev));
6374 
6375 	adev->ualink.msg_ctl->send_halt(adev);
6376 
6377 	amdgpu_ualink_peer_remote_fini(adev);
6378 	amdgpu_ualink_sdma_entities_fini(adev);
6379 	amdgpu_ualink_metadata_npa_unmapping(adev);
6380 	amdgpu_ualink_metadata_fini(adev);
6381 	kfree(adev->ualink.remote);
6382 	adev->ualink.remote = NULL;
6383 }
6384 
6385 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id)
6386 {
6387 	return amdgpu_ualink_send_hello_msg(adev, remote_accel_id);
6388 }
6389 
6390 /*
6391  * The low bits of handle_lo carry the message-header field on the wire.
6392  * Strip those bits and splice in @acc_id to recover the
6393  * fully-qualified handle_lo value.
6394  */
6395 static inline u64 amdgpu_ualink_reassemble_handle(u64 handle_lo, u32 acc_id)
6396 {
6397 	return (handle_lo & ~AMDGPU_UALINK_MESSAGE_HEADER_MASK) |
6398 	       (acc_id & AMDGPU_UALINK_HANDLE_ACCID_MASK);
6399 }
6400 
6401 static int amdgpu_ualink_process_irq(struct amdgpu_device *adev,
6402 				     struct amdgpu_irq_src *source,
6403 				     struct amdgpu_iv_entry *entry)
6404 {
6405 	u32 sender_acc_id, receiver_acc_id, msg_type, src_acc_id;
6406 	enum amdgpu_ualink_accel_state accel_state;
6407 	struct amdgpu_ualink_handle handle;
6408 	u32 size, npa_addr, fail_reason;
6409 	u32 dw0, dw1, dw2, dw3;
6410 	u32 local_acc_id;
6411 	int handled = 1;
6412 
6413 	if (unlikely(adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)) {
6414 		dev_dbg(adev->dev,
6415 			"UALink manager not initialized, dropping irq\n");
6416 		return handled;
6417 	}
6418 
6419 	/* Only handle interrupts while ACTIVE. Otherwise (e.g. during teardown)
6420 	 * consume stale ring entries but drop them, so they don't touch state
6421 	 * being freed.
6422 	 */
6423 	accel_state = adev->ualink.info->accel_state;
6424 	if (accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
6425 		dev_dbg(adev->dev,
6426 			"Dropping stale UALink interrupt, accel_state %d\n",
6427 			accel_state);
6428 		return handled;
6429 	}
6430 
6431 	dev_dbg(adev->dev, "%s client_id 0x%x src_id 0x%x ih\n",
6432 		entry->ih == &adev->irq.ih ? "ring" : "ualink soft ring",
6433 		entry->client_id, entry->src_id);
6434 
6435 	/* ContextID 4 dwords */
6436 	src_acc_id = entry->pasid;
6437 	dw0 = entry->src_data[0];
6438 	dw1 = entry->src_data[1];
6439 	dw2 = entry->src_data[2];
6440 	dw3 = entry->src_data[3];
6441 
6442 	/* Copy IH entry into ualink soft ring. */
6443 	if (entry->ih == &adev->irq.ih) {
6444 		dev_dbg(adev->dev, "src accel_id %u context id 0x%x 0x%x 0x%x 0x%x\n",
6445 			src_acc_id, dw0, dw1, dw2, dw3);
6446 		dev_dbg(adev->dev, "delegate to ualink irq soft ring\n");
6447 		amdgpu_irq_ualink_delegate(adev, entry, 8);
6448 		return handled;
6449 	}
6450 
6451 	msg_type = dw0 & AMDGPU_UALINK_MESSAGE_HEADER_MASK;
6452 	local_acc_id = adev->ualink.info->ppod.accel_id;
6453 	dev_dbg(adev->dev, "Got MSG: remote acc_id %u msg_type %u\n",
6454 		src_acc_id, msg_type);
6455 
6456 	/* Debug hook: if the bit corresponding to this msg_type is set in
6457 	 * drop_msg_bitmap, drop this single packet and clear the bit so that
6458 	 * any subsequent incoming messages are processed normally. Used to
6459 	 * exercise the connection reset/recovery paths via debugfs.
6460 	 */
6461 	if (msg_type < BITS_PER_LONG &&
6462 	    test_and_clear_bit(msg_type, &adev->ualink.drop_msg_bitmap)) {
6463 		dev_warn(adev->dev,
6464 			 "DROP MSG (debugfs): src acc_id %u msg_type %u dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
6465 			 src_acc_id, msg_type, dw0, dw1, dw2, dw3);
6466 		return handled;
6467 	}
6468 
6469 	switch (msg_type) {
6470 	case AMDGPU_UALINK_HELLO_MSG:
6471 		receiver_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT) &
6472 					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
6473 		sender_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT) &
6474 					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
6475 		dev_dbg(adev->dev,
6476 			"Got HELLO MSG: src acc_id %u receiver_acc_id %u sender_acc_id %u\n",
6477 			src_acc_id, receiver_acc_id, sender_acc_id);
6478 		amdgpu_ualink_process_hello_msg(adev, receiver_acc_id, sender_acc_id,
6479 						src_acc_id);
6480 		break;
6481 	case AMDGPU_UALINK_HELLO_ACK_MSG:
6482 		dev_dbg(adev->dev,
6483 			"Got HELLO-ACK MSG: remote acc_id %u\n", src_acc_id);
6484 		amdgpu_ualink_process_hello_ack_msg(adev, src_acc_id);
6485 		break;
6486 	case AMDGPU_UALINK_NPA_REQ_MSG:
6487 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6488 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6489 								   local_acc_id);
6490 		dev_dbg(adev->dev,
6491 			"Got NPA-REQ MSG: remote acc_id %u handle %llx:%llx\n",
6492 			src_acc_id, handle.handle_hi, handle.handle_lo);
6493 		amdgpu_ualink_process_npa_req_msg(adev, src_acc_id, handle);
6494 		break;
6495 	case AMDGPU_UALINK_NPA_RSP_MSG:
6496 		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
6497 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6498 								   src_acc_id);
6499 		size = entry->src_data[2];
6500 		npa_addr = entry->src_data[3];
6501 		dev_dbg(adev->dev,
6502 			"Got NPA-RSP MSG: remote acc_id %u handle_lo %llx size 0x%x npa_addr 0x%x\n",
6503 			src_acc_id, handle.handle_lo, size, npa_addr);
6504 		amdgpu_ualink_process_npa_rsp_msg(adev, src_acc_id,
6505 						  handle.handle_lo,
6506 						  npa_addr, size);
6507 		break;
6508 	case AMDGPU_UALINK_NPA_FAIL_MSG:
6509 		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
6510 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6511 								   src_acc_id);
6512 		fail_reason = entry->src_data[2] &
6513 				AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK;
6514 		dev_dbg(adev->dev,
6515 			"Got NPA-FAIL MSG: remote acc_id %u handle_lo %llx fail_reason %u\n",
6516 			src_acc_id, handle.handle_lo, fail_reason);
6517 		amdgpu_ualink_process_npa_fail_msg(adev, src_acc_id,
6518 						   handle.handle_lo,
6519 						   fail_reason);
6520 		break;
6521 	case AMDGPU_UALINK_NPA_REVOKE_MSG:
6522 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6523 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6524 								   src_acc_id);
6525 		dev_dbg(adev->dev,
6526 			"Got NPA-REVOKE MSG: remote acc_id %u handle %llx:%llx\n",
6527 			src_acc_id, handle.handle_hi, handle.handle_lo);
6528 		amdgpu_ualink_process_npa_revoke_msg(adev, src_acc_id, handle);
6529 		break;
6530 	case AMDGPU_UALINK_NPA_RELEASE_MSG:
6531 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6532 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6533 								   local_acc_id);
6534 		dev_dbg(adev->dev,
6535 			"Got NPA-RELEASE MSG: remote acc_id %u handle %llx:%llx\n",
6536 			src_acc_id, handle.handle_hi, handle.handle_lo);
6537 		amdgpu_ualink_process_npa_release_msg(adev, src_acc_id, handle);
6538 		break;
6539 	default:
6540 		dev_err(adev->dev, "Unknown message type (%u)\n", msg_type);
6541 		break;
6542 	}
6543 
6544 	return handled;
6545 }
6546 
6547 static int amdgpu_ualink_set_irq_state(struct amdgpu_device *adev,
6548 					struct amdgpu_irq_src *source,
6549 					u32 type,
6550 					enum amdgpu_interrupt_state state)
6551 {
6552 	/*
6553 	 * Don't set register to enable/disable nHT controller interrupt.
6554 	 *
6555 	 * F/W running on MP2, which can always send cookie to IH block to
6556 	 * interrupt driver.
6557 	 */
6558 	dev_dbg(adev->dev, "ualink interrupt %s\n",
6559 		state == AMDGPU_IRQ_STATE_ENABLE ? "enable" : "disable");
6560 	return 0;
6561 }
6562 
6563 static const struct amdgpu_irq_src_funcs ualink_irq_funcs = {
6564 	.set = amdgpu_ualink_set_irq_state,
6565 	.process = amdgpu_ualink_process_irq,
6566 };
6567 
6568 /**
6569  * amdgpu_ualink_init_interrupt - register the UALink IRQ source
6570  * @adev: amdgpu device pointer
6571  * @client_id: IH client ID for the interrupt source
6572  * @src_id: source ID for the interrupt source
6573  *
6574  * Registers the UALink interrupt source with the IH subsystem.
6575  *
6576  * Return: 0 on success, negative error code on failure
6577  */
6578 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev,
6579 				 unsigned int client_id, unsigned int src_id)
6580 {
6581 	int r;
6582 
6583 	dev_dbg(adev->dev, "init ualink irq client_id 0x%x src_id 0x%x\n",
6584 		client_id, src_id);
6585 
6586 	adev->ualink.irq.num_types = 1;
6587 	adev->ualink.irq.funcs = &ualink_irq_funcs;
6588 
6589 	r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->ualink.irq);
6590 	return r;
6591 }
6592 
6593