xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2026 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/xarray.h>
26 #include <drm/drm_mm.h>
27 #include <linux/random.h>
28 #include "amdgpu.h"
29 #include "amdgpu_sdma.h"
30 #include "amdgpu_ualink.h"
31 #include "amdgpu_xgmi.h"
32 #include "amdgpu_dma_buf.h"
33 #include "psp_gfx_if.h"
34 #include <linux/math.h>
35 #include <linux/sysfs.h>
36 #include <linux/string.h>
37 
38 static void deactivate_accelerator(struct amdgpu_device *adev);
39 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
40 				u32 remote_accel_id, u32 dw0, u32 dw1,
41 				u32 dw2, u32 dw3);
42 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev,
43 				    u32 flush_type);
44 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
45 						struct amdgpu_bo *bos[], u32 n_bos,
46 						struct drm_exec *exec,
47 						bool interruptible);
48 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
49 						   struct drm_exec *exec);
50 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
51 						  u32 remote_accel_id, u32 state,
52 						  u32 generation_count);
53 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo);
54 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
55 					  u32 remote_accel_id, u64 addr,
56 					  u32 size_in_pages, u32 flush_type);
57 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev);
58 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev);
59 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
60 					   u32 remote_accel_id);
61 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
62 					  u32 remote_accel_id);
63 
64 #define STRIP_NPA(addr)						\
65 	(((u64)(addr) & ~AMDGPU_UALINK_NPA_ADDR_GPUID_MASK))
66 
67 #define GENERATE_NPA(addr, remote_acc_id)			\
68 		((u64)(((u64)(addr)) |				\
69 		 ((u64)(remote_acc_id) << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT)))
70 
71 static const struct drm_client_funcs ualink_client_funcs = {
72 	.unregister	= drm_client_release,
73 };
74 
75 static int amdgpu_ualink_drm_client_create(struct amdgpu_device *adev)
76 {
77 	int ret;
78 
79 	ret = drm_client_init(&adev->ddev, &adev->ualink.client, "ualink",
80 			      &ualink_client_funcs);
81 	if (ret) {
82 		dev_err(adev->dev, "Failed to init UALink DRM client: %d\n",
83 			ret);
84 		return ret;
85 	}
86 
87 	drm_client_register(&adev->ualink.client);
88 
89 	return 0;
90 }
91 
92 static void amdgpu_ualink_object_fini(struct amdgpu_device *adev)
93 {
94 	if (!adev->ualink.info)
95 		return;
96 
97 	kobject_put(&adev->ualink.stations->kobj);
98 	kobject_put(&adev->ualink.config->kobj);
99 	kobject_put(&adev->ualink.setup->kobj);
100 	kobject_put(&adev->ualink.info->kobj);
101 	adev->ualink.stations = NULL;
102 	adev->ualink.config = NULL;
103 	adev->ualink.setup = NULL;
104 	adev->ualink.info = NULL;
105 }
106 
107 static bool __check_ppod_info(struct amdgpu_device *adev,
108 			      const struct amdgpu_ualink_info *info)
109 {
110 	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
111 
112 	if (ppod->size <= 0 || ppod->size > AMDGPU_UALINK_ACCEL_MAX) {
113 		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
114 			ppod->size, AMDGPU_UALINK_ACCEL_MAX);
115 		return false;
116 	}
117 	if (ppod->accel_id >= ppod->size) {
118 		dev_dbg(adev->dev,
119 			"Accelerator ID %u greater or equal pPod size %u\n",
120 			ppod->accel_id, ppod->size);
121 		return false;
122 	}
123 
124 	return true;
125 }
126 
127 static bool __check_vpod_info(struct amdgpu_device *adev,
128 			      const struct amdgpu_ualink_info *info)
129 {
130 	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
131 	const struct amdgpu_ualink_vpod_info *vpod = &info->vpod;
132 	unsigned int weight;
133 
134 	if (vpod->size == 0 || vpod->size > ppod->size) {
135 		dev_warn(adev->dev,
136 			 "UALINK: vPod size %u out of range [1..%u]\n",
137 			 vpod->size, ppod->size);
138 		return false;
139 	}
140 
141 	switch (vpod->addr_mode) {
142 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS:
143 		break;
144 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT:
145 		dev_warn(adev->dev,
146 			 "UALINK: source-identification addr mode is not supported\n");
147 		return false;
148 	default:
149 		dev_warn(adev->dev, "UALINK: invalid addr mode %u\n",
150 			 vpod->addr_mode);
151 		return false;
152 	}
153 
154 	weight = bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
155 	if (weight != vpod->size) {
156 		dev_warn(adev->dev,
157 			 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n",
158 			 vpod->size, weight);
159 		return false;
160 	}
161 	if (!test_bit(ppod->accel_id, vpod->active_accel_bits)) {
162 		dev_warn(adev->dev,
163 			 "UALINK: accelerator ID %u not listed in vpod_active_accels\n",
164 			 ppod->accel_id);
165 		return false;
166 	}
167 
168 	return true;
169 }
170 
171 static void
172 amdgpu_ualink_info_set_accel_state(struct amdgpu_device *adev,
173 				   struct amdgpu_ualink_info *info,
174 				   enum psp_gfx_ual_config_state cfg_state)
175 {
176 	enum amdgpu_ualink_accel_state cur;
177 	enum amdgpu_ualink_accel_state target;
178 	bool ppod_validated;
179 	bool vpod_validated;
180 
181 	if (!info)
182 		return;
183 
184 	cur = info->accel_state;
185 
186 	ppod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED &&
187 			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
188 	vpod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED &&
189 			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
190 
191 	switch (cfg_state) {
192 	case UAL_CFG_IDLE:
193 		return;
194 	case UAL_CFG_PPOD:
195 		target = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
196 		break;
197 	case UAL_CFG_VPOD:
198 	case UAL_CFG_STATION:
199 		target = AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
200 		break;
201 	case UAL_CFG_COMPLETE:
202 		target = AMDGPU_UALINK_ACCEL_STATE_READY;
203 		break;
204 	default:
205 		dev_dbg(adev->dev, "invalid configuration state %u", cfg_state);
206 		return;
207 	}
208 
209 	/* ppod stage: should be part of a ppod first */
210 	if (!ppod_validated && !__check_ppod_info(adev, info)) {
211 		if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
212 			info->accel_state =
213 				AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
214 		} else {
215 			info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
216 			dev_err(adev->dev,
217 				"ppod configuration is invalid, setting to error state");
218 		}
219 		return;
220 	}
221 	if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
222 		if (!ppod_validated)
223 			info->accel_state =
224 				AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
225 		return;
226 	}
227 
228 	/* A vpod_id of 0 means the GPU is not part of any vPod */
229 	if (info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
230 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
231 		return;
232 	}
233 
234 	/* vpod stage: required to reach vpod_configured or ready */
235 	if (!vpod_validated && !__check_vpod_info(adev, info)) {
236 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
237 		dev_err(adev->dev,
238 			"vpod configuration is invalid, setting to error state");
239 		return;
240 	}
241 	if (target == AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
242 		if (!vpod_validated)
243 			info->accel_state =
244 				AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
245 		return;
246 	}
247 
248 	/* complete stage: advance to ready unless already ready/active */
249 	if (cur < AMDGPU_UALINK_ACCEL_STATE_READY ||
250 	    cur > AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
251 		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_READY;
252 }
253 
254 static int amdgpu_ualink_update_vpod_config(struct amdgpu_device *adev)
255 {
256 	/* TBD: Do updates/cleanup based on updated vpod configuration */
257 	return 0;
258 }
259 
260 static int amdgpu_ualink_update_accel_state(
261 	struct amdgpu_device *adev, enum amdgpu_ualink_accel_state prev_state,
262 	u32 prev_vpod_id, enum psp_gfx_ual_config_state cfg_state)
263 {
264 	dev_info(adev->dev,
265 		 "UALINK: update_accel_state: prev_state=%d prev_vpod_id=%u new_vpod_id=%u cfg_state=%d\n",
266 		 prev_state, prev_vpod_id, adev->ualink.info->vpod.id,
267 		 cfg_state);
268 
269 	/* If the device is already active and its vpod_id is unchanged, the
270 	 * update does not affect vpod membership. Skip the local vpod
271 	 * integrity check and re-activation.
272 	 */
273 	if (prev_state == AMDGPU_UALINK_ACCEL_STATE_ACTIVE &&
274 	    adev->ualink.info->vpod.id == prev_vpod_id) {
275 		amdgpu_ualink_update_vpod_config(adev);
276 
277 		/* Same vpod_id, but the member set may have changed (vPod grown
278 		 * or shrunk while this GPU stayed ACTIVE). If so, run the local
279 		 * vpod activation path: once every local peer has committed the
280 		 * new config (integrity passes), it bounces the affected ACTIVE
281 		 * peers to rebuild links/GART for the new member set.
282 		 */
283 		if (amdgpu_ualink_vpod_membership_changed(adev)) {
284 			dev_info(adev->dev,
285 				 "UALINK: update_accel_state: ACTIVE vpod_id=%u membership changed, reconfiguring\n",
286 				 adev->ualink.info->vpod.id);
287 			scoped_guard(mutex, &mgpu_info.mutex)
288 				__amdgpu_ualink_activate_vpod_locked(adev);
289 		} else {
290 			dev_info(adev->dev,
291 				 "UALINK: update_accel_state: already ACTIVE, vpod_id/membership unchanged\n");
292 		}
293 		return 0;
294 	}
295 
296 	/* A new vpod_id of 0 means this GPU was removed from the vPod. */
297 	if (adev->ualink.info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
298 		dev_info(adev->dev,
299 			 "UALINK: update_accel_state: vpod_id=0, removing accelerator from vPod\n");
300 		amdgpu_ualink_update_vpod_config(adev);
301 		scoped_guard(mutex, &mgpu_info.mutex)
302 			deactivate_accelerator(adev);
303 		return 0;
304 	}
305 
306 	/* GPU joining a new vpod should be with invalid vpod id*/
307 	scoped_guard(mutex, &mgpu_info.mutex) {
308 		if (prev_vpod_id == AMDGPU_UALINK_VPOD_ID_INVALID) {
309 			dev_info(adev->dev,
310 				 "UALINK: update_accel_state: joining vpod_id=%u\n",
311 				 adev->ualink.info->vpod.id);
312 			amdgpu_ualink_info_set_accel_state(
313 				adev, adev->ualink.info, cfg_state);
314 			__amdgpu_ualink_activate_vpod_locked(adev);
315 		} else {
316 			/* GPU should first get removal which will set invalid vpod_id
317 			* and then join a new vpod
318 			*/
319 			dev_err(adev->dev,
320 				"Invalid vpod transition from %u to %u\n",
321 				prev_vpod_id, adev->ualink.info->vpod.id);
322 			return -EINVAL;
323 		}
324 	}
325 
326 	return 0;
327 }
328 
329 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev)
330 {
331 	enum amdgpu_ualink_accel_state prev_state;
332 	enum psp_gfx_ual_config_state cfg_state;
333 	u32 prev_vpod_id;
334 	int r, qerr;
335 
336 	/* TBD: Stop ASP interrupts if driver faced an issue */
337 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
338 		u32 status;
339 
340 		dev_dbg(adev->dev,
341 			"UALink not initialized, skipping config update\n");
342 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
343 		return psp_ual_send_completion(
344 			&adev->psp, adev->ualink.psp_if_ver,
345 			PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, status);
346 	}
347 
348 	prev_state = adev->ualink.info->accel_state;
349 	prev_vpod_id = adev->ualink.info->vpod.id;
350 
351 	qerr = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
352 				  adev->ualink.info, &cfg_state);
353 
354 	/*TBD: find the right value of status to be sent to ASP*/
355 	r = psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
356 				    PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, 0);
357 	if (r || qerr)
358 		goto err;
359 
360 	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
361 					     cfg_state);
362 	if (r)
363 		goto err;
364 
365 	return 0;
366 
367 err:
368 	scoped_guard(mutex, &mgpu_info.mutex) {
369 		deactivate_accelerator(adev);
370 		adev->ualink.info->accel_state =
371 			AMDGPU_UALINK_ACCEL_STATE_ERROR;
372 	}
373 	dev_err(adev->dev,
374 		"UALink config update failed, setting to error state");
375 
376 	return r;
377 }
378 
379 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev)
380 {
381 	u32 status = 0;
382 
383 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
384 		dev_dbg(adev->dev,
385 			"UALink not initialized, skipping pause update\n");
386 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
387 		goto out;
388 	}
389 
390 	dev_dbg(adev->dev, "UALink pause command is not handled\n");
391 
392 out:
393 	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
394 				       PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID,
395 				       status);
396 }
397 
398 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev)
399 {
400 	u32 status = 0;
401 
402 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
403 		dev_dbg(adev->dev,
404 			"UALink not initialized, skipping pause update\n");
405 		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
406 		goto out;
407 	}
408 
409 	dev_dbg(adev->dev, "UALink resume command is not handled\n");
410 
411 out:
412 	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
413 				       PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID,
414 				       status);
415 }
416 
417 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block)
418 {
419 	struct amdgpu_device *adev = ip_block->adev;
420 	int r;
421 
422 	if (!adev->ualink.info)
423 		return 0;
424 
425 	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
426 	r = psp_ual_get_interface_version(&adev->psp, &adev->ualink.psp_if_ver);
427 	if (r) {
428 		adev->ualink.psp_if_ver = 0xffffffff;
429 		goto disable;
430 	}
431 	dev_info(adev->dev, "Found UALink interface version 0x%x\n",
432 		 adev->ualink.psp_if_ver);
433 
434 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_HW;
435 
436 	return 0;
437 disable:
438 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
439 	return 0;
440 }
441 
442 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block)
443 {
444 	struct amdgpu_device *adev = ip_block->adev;
445 
446 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
447 		return 0;
448 
449 	scoped_guard(mutex, &mgpu_info.mutex)
450 		deactivate_accelerator(adev);
451 
452 	return 0;
453 }
454 
455 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block)
456 {
457 	struct amdgpu_device *adev = ip_block->adev;
458 	enum psp_gfx_ual_config_state cfg_state;
459 	int r;
460 
461 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_HW)
462 		return 0;
463 
464 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
465 			       adev->ualink.info, &cfg_state);
466 	if (r)
467 		return r;
468 
469 	scoped_guard(mutex, &mgpu_info.mutex) {
470 		amdgpu_ualink_info_set_accel_state(adev, adev->ualink.info,
471 						   cfg_state);
472 		__amdgpu_ualink_activate_vpod_locked(adev);
473 	}
474 
475 	r = amdgpu_ualink_drm_client_create(adev);
476 	if (r) {
477 		dev_err(adev->dev, "Failed to create UALink DRM client: %d\n",
478 			r);
479 		goto error;
480 	}
481 
482 	/* Consider UALink initialized only at this stage */
483 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_COMPLETE;
484 
485 	return 0;
486 
487 error:
488 	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
489 	return r;
490 }
491 
492 /****************************************************************************
493  * UALink info and configuration in sysfs
494  *
495  * Using kobj_attribute and not device_attribute here because the UALink
496  * attributes are part of their own nested kobjects and not the amdgpu
497  * device itself or a subdevice.
498  ****************************************************************************/
499 
500 /* UALink configuration APIs are being deprecated before upstreaming and are
501  * not enabled by default. They are replaced by a unified in-band/side-band
502  * approach through the IFoE driver. Querying of the current configuration
503  * will continue to be supported in sysfs.
504  *
505  * Uncomment the following line to enable the deprecated configuration APIs
506  * for testing purposes only. The code to support these APIs will be removed
507  * in a future patch.
508  */
509 /* #define UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS */
510 
511 #define UALINK_VALUE_SHOW(prefix, name, field, format)			\
512 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
513 					       struct kobj_attribute *attr,\
514 					       char *buf)		\
515 {									\
516 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
517 									\
518 	return sysfs_emit(buf, format"\n", info->field);		\
519 }
520 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
521 #define UALINK_VALUE_STORE(prefix, name, field, type, base)		\
522 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
523 						struct kobj_attribute *attr,\
524 						const char *buf, size_t count)\
525 {									\
526 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
527 	int r;								\
528 									\
529 	r = kstrto##type(buf, base, &info->field);			\
530 	if (r < 0)							\
531 		return r;						\
532 	return count;							\
533 }
534 #endif
535 
536 static ssize_t show_idbits(unsigned long *bits, unsigned int nbits, char *buf)
537 {
538 	ssize_t len = 0;
539 	unsigned int id;
540 
541 	for_each_set_bit(id, bits, nbits)
542 		len += sysfs_emit_at(buf, len, "%u ", id);
543 	if (len)
544 		len--;
545 	return len + sysfs_emit_at(buf, len, "\n");
546 }
547 #define UALINK_IDBITS_SHOW(prefix, name, field)				\
548 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
549 					       struct kobj_attribute *attr,\
550 					       char *buf)		\
551 {									\
552 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
553 									\
554 	return show_idbits(info->field, sizeof(info->field)*8, buf);	\
555 }
556 
557 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
558 static ssize_t store_idbits(unsigned long *bits, unsigned int nbits,
559 			    const char *buf, size_t count)
560 {
561 	char *dup_buf, *str, *tok;
562 	ssize_t r;
563 	u32 id;
564 
565 	bitmap_zero(bits, nbits);
566 	if (!count)
567 		return 0;
568 
569 	dup_buf = kstrndup(buf, count, GFP_KERNEL);
570 	if (unlikely(!dup_buf))
571 		return -ENOMEM;
572 	str = dup_buf;
573 	do {
574 		str += strspn(str, " ");
575 		tok = strsep(&str, ", ");
576 		r = kstrtou32(tok, 10, &id);
577 		if (r < 0)
578 			goto err;
579 		if (id >= nbits) {
580 			r = -ERANGE;
581 			goto err;
582 		}
583 		set_bit(id, bits);
584 	} while (str);
585 	kfree(dup_buf);
586 
587 	return count;
588 
589 err:
590 	kfree(dup_buf);
591 	return r;
592 }
593 #define UALINK_IDBITS_STORE(prefix, name, field, nbits)			\
594 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
595 						struct kobj_attribute *attr,\
596 						const char *buf, size_t count)\
597 {									\
598 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
599 	DECLARE_BITMAP(bits, nbits);					\
600 	ssize_t r = store_idbits(bits, nbits, buf, count);		\
601 									\
602 	if (r >= 0)							\
603 		bitmap_copy(info->field, bits, nbits);			\
604 	return r;							\
605 }
606 #endif
607 
608 static ssize_t show_idarray(const u32 *array, u32 size, char *buf)
609 {
610 	ssize_t len = 0;
611 	unsigned int i;
612 
613 	for (i = 0; i < size; i++)
614 		len += sysfs_emit_at(buf, len, "%u ", array[i]);
615 	if (len)
616 		len--;
617 	return len + sysfs_emit_at(buf, len, "\n");
618 }
619 #define UALINK_IDARRAY_SHOW(prefix, name, field, size)			\
620 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
621 					       struct kobj_attribute *attr,\
622 					       char *buf)		\
623 {									\
624 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
625 									\
626 	return show_idarray(info->field, info->size, buf);		\
627 }
628 
629 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
630 static ssize_t store_idarray(u32 *array, u32 *size, u32 max_id, u32 max_size,
631 			     const char *buf, size_t count)
632 {
633 	char *dup_buf, *str, *tok;
634 	unsigned int index = 0;
635 	ssize_t r;
636 	u32 id;
637 
638 	*size = 0;
639 	if (!count)
640 		return 0;
641 
642 	dup_buf = kstrndup(buf, count, GFP_KERNEL);
643 	if (unlikely(!dup_buf))
644 		return -ENOMEM;
645 	str = dup_buf;
646 	do {
647 		str += strspn(str, " ");
648 		tok = strsep(&str, ", ");
649 		r = kstrtou32(tok, 10, &id);
650 		if (r < 0)
651 			goto err;
652 		if (index >= max_size) {
653 			r = -ENOSPC;
654 			goto err;
655 		}
656 		if (id >= max_id) {
657 			r = -ERANGE;
658 			goto err;
659 		}
660 		array[index++] = id;
661 	} while (str);
662 	kfree(dup_buf);
663 
664 	*size = index;
665 	return count;
666 
667 err:
668 	kfree(dup_buf);
669 	return r;
670 }
671 #define UALINK_IDARRAY_STORE(prefix, name, field, size, max_id)		\
672 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
673 						struct kobj_attribute *attr,\
674 						const char *buf, size_t count)\
675 {									\
676 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
677 	u32 new_array[ARRAY_SIZE(info->field)];				\
678 	u32 new_size;							\
679 	ssize_t r = store_idarray(new_array, &new_size, max_id,		\
680 				  ARRAY_SIZE(new_array), buf, count);	\
681 									\
682 	if (r >= 0) {							\
683 		memcpy(info->field, new_array, sizeof(new_array));	\
684 		info->size = new_size;					\
685 	}								\
686 	return r;							\
687 }
688 #endif
689 
690 #define UALINK_UUID_SHOW(prefix, name, field)				\
691 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
692 					       struct kobj_attribute *attr,\
693 					       char *buf)		\
694 {									\
695 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
696 									\
697 	return sysfs_emit(buf, "%pU\n", &info->field);			\
698 }
699 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
700 #define UALINK_UUID_STORE(prefix, name, field)				\
701 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
702 						struct kobj_attribute *attr,\
703 						const char *buf, size_t count)\
704 {									\
705 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
706 	int r;								\
707 									\
708 	r = uuid_parse(buf, &info->field);				\
709 	return r ? r : count;						\
710 }
711 #endif
712 
713 static ssize_t show_enum(u32 x, const char * const values[], unsigned int n,
714 			 char *buf)
715 {
716 	return sysfs_emit(buf, "%s\n", x < n && values[x] ? values[x] : "invalid");
717 }
718 #define UALINK_ENUM_SHOW(prefix, name, field)				\
719 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
720 					       struct kobj_attribute *attr,\
721 					       char *buf)		\
722 {									\
723 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
724 									\
725 	return show_enum(info->field, ualink_##name##_values,		\
726 			 ARRAY_SIZE(ualink_##name##_values), buf);	\
727 }
728 
729 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
730 static ssize_t store_enum(u32 *x, const char * const values[], unsigned int n,
731 			  const char *buf, size_t count)
732 {
733 	unsigned int i;
734 
735 	for (i = 0; i < n; i++) {
736 		if (values[i] && sysfs_streq(buf, values[i])) {
737 			*x = i;
738 			return count;
739 		}
740 	}
741 	return -EINVAL;
742 }
743 #define UALINK_ENUM_STORE(prefix, name, field)				\
744 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
745 						struct kobj_attribute *attr,\
746 						const char *buf, size_t count)\
747 {									\
748 	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
749 									\
750 	return store_enum(&info->field, ualink_##name##_values,		\
751 			 ARRAY_SIZE(ualink_##name##_values),		\
752 			 buf, count);					\
753 }
754 #endif
755 static const char * const ualink_link_type_values[] = {
756 	"UALoE", "UALink"
757 };
758 static const char * const ualink_addr_mode_values[] = {
759 	"source-aliasing", "source-identification"
760 };
761 static const char * const ualink_accel_state_values[] = {
762 	[AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED]	= "unconfigured",
763 	[AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED]	= "ppod_configured",
764 	[AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED]	= "vpod_configured",
765 	[AMDGPU_UALINK_ACCEL_STATE_READY]		= "ready",
766 	[AMDGPU_UALINK_ACCEL_STATE_ACTIVE]		= "active",
767 	[AMDGPU_UALINK_ACCEL_STATE_ERROR]		= "error",
768 };
769 static_assert(ARRAY_SIZE(ualink_accel_state_values) ==
770 	      AMDGPU_UALINK_ACCEL_STATE_MAX);
771 
772 UALINK_ENUM_SHOW(info, link_type, link_type);
773 UALINK_VALUE_SHOW(info, accel_id,  ppod.accel_id,  "%u");
774 UALINK_VALUE_SHOW(info, bandwidth, ppod.bandwidth, "%u");
775 UALINK_VALUE_SHOW(info, latency,   ppod.latency,   "%u");
776 UALINK_UUID_SHOW(info, ppod_id, ppod.id);
777 UALINK_VALUE_SHOW(info, ppod_size, ppod.size,      "%u");
778 UALINK_VALUE_SHOW(info, vpod_id,   vpod.id,        "%u");
779 UALINK_VALUE_SHOW(info, vpod_size, vpod.size,      "%u");
780 UALINK_IDBITS_SHOW(info, vpod_active_accels, vpod.active_accel_bits);
781 UALINK_ENUM_SHOW(info, addr_mode, vpod.addr_mode);
782 UALINK_ENUM_SHOW(info, accel_state, accel_state);
783 UALINK_IDARRAY_SHOW(info, local_accels, local_accels, n_local_accels);
784 
785 static struct amdgpu_device *ualink_xcp_kobj_to_adev(struct kobject *kobj)
786 {
787 	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
788 					      ualink.kobj);
789 
790 	return xcp->xcp_mgr->adev;
791 }
792 
793 #define UALINK_XCP_INFO_SHOW(name)					\
794 static ssize_t ualink_xcp_info_##name##_show(struct kobject *kobj,	\
795 		struct kobj_attribute *attr, char *buf)			\
796 {									\
797 	struct amdgpu_device *adev = ualink_xcp_kobj_to_adev(kobj);	\
798 									\
799 	return ualink_info_##name##_show(&adev->ualink.info->kobj,	\
800 					 attr, buf);			\
801 }
802 
803 UALINK_XCP_INFO_SHOW(link_type)
804 UALINK_XCP_INFO_SHOW(accel_id)
805 UALINK_XCP_INFO_SHOW(bandwidth)
806 UALINK_XCP_INFO_SHOW(latency)
807 UALINK_XCP_INFO_SHOW(ppod_id)
808 UALINK_XCP_INFO_SHOW(ppod_size)
809 UALINK_XCP_INFO_SHOW(vpod_id)
810 UALINK_XCP_INFO_SHOW(vpod_size)
811 UALINK_XCP_INFO_SHOW(vpod_active_accels)
812 UALINK_XCP_INFO_SHOW(addr_mode)
813 UALINK_XCP_INFO_SHOW(accel_state)
814 UALINK_XCP_INFO_SHOW(local_accels)
815 
816 #define UALINK_INFO_ATTR(name) __ATTR(name, 0444, ualink_info_##name##_show, NULL)
817 static struct kobj_attribute ualink_info_link_type = UALINK_INFO_ATTR(link_type);
818 static struct kobj_attribute ualink_info_accel_id  = UALINK_INFO_ATTR(accel_id);
819 static struct kobj_attribute ualink_info_bandwidth = UALINK_INFO_ATTR(bandwidth);
820 static struct kobj_attribute ualink_info_latency   = UALINK_INFO_ATTR(latency);
821 static struct kobj_attribute ualink_info_ppod_id   = UALINK_INFO_ATTR(ppod_id);
822 static struct kobj_attribute ualink_info_ppod_size = UALINK_INFO_ATTR(ppod_size);
823 static struct kobj_attribute ualink_info_vpod_id   = UALINK_INFO_ATTR(vpod_id);
824 static struct kobj_attribute ualink_info_vpod_size = UALINK_INFO_ATTR(vpod_size);
825 static struct kobj_attribute ualink_info_vpod_active_accels = UALINK_INFO_ATTR(vpod_active_accels);
826 static struct kobj_attribute ualink_info_addr_mode = UALINK_INFO_ATTR(addr_mode);
827 static struct kobj_attribute ualink_info_accel_state = UALINK_INFO_ATTR(accel_state);
828 static struct kobj_attribute ualink_info_local_accels = UALINK_INFO_ATTR(local_accels);
829 
830 static const struct attribute *ualink_info_attrs[] = {
831 	&ualink_info_link_type.attr,
832 	&ualink_info_accel_id.attr,
833 	&ualink_info_bandwidth.attr,
834 	&ualink_info_latency.attr,
835 	&ualink_info_ppod_id.attr,
836 	&ualink_info_ppod_size.attr,
837 	&ualink_info_vpod_id.attr,
838 	&ualink_info_vpod_size.attr,
839 	&ualink_info_vpod_active_accels.attr,
840 	&ualink_info_addr_mode.attr,
841 	&ualink_info_accel_state.attr,
842 	&ualink_info_local_accels.attr,
843 	NULL
844 };
845 
846 #define UALINK_XCP_INFO_ATTR(name) \
847 	__ATTR(name, 0444, ualink_xcp_info_##name##_show, NULL)
848 static struct kobj_attribute ualink_xcp_info_link_type = UALINK_XCP_INFO_ATTR(link_type);
849 static struct kobj_attribute ualink_xcp_info_accel_id  = UALINK_XCP_INFO_ATTR(accel_id);
850 static struct kobj_attribute ualink_xcp_info_bandwidth = UALINK_XCP_INFO_ATTR(bandwidth);
851 static struct kobj_attribute ualink_xcp_info_latency   = UALINK_XCP_INFO_ATTR(latency);
852 static struct kobj_attribute ualink_xcp_info_ppod_id   = UALINK_XCP_INFO_ATTR(ppod_id);
853 static struct kobj_attribute ualink_xcp_info_ppod_size = UALINK_XCP_INFO_ATTR(ppod_size);
854 static struct kobj_attribute ualink_xcp_info_vpod_id   = UALINK_XCP_INFO_ATTR(vpod_id);
855 static struct kobj_attribute ualink_xcp_info_vpod_size = UALINK_XCP_INFO_ATTR(vpod_size);
856 static struct kobj_attribute ualink_xcp_info_vpod_active_accels = UALINK_XCP_INFO_ATTR(vpod_active_accels);
857 static struct kobj_attribute ualink_xcp_info_addr_mode = UALINK_XCP_INFO_ATTR(addr_mode);
858 static struct kobj_attribute ualink_xcp_info_accel_state = UALINK_XCP_INFO_ATTR(accel_state);
859 static struct kobj_attribute ualink_xcp_info_local_accels = UALINK_XCP_INFO_ATTR(local_accels);
860 
861 static struct attribute *ualink_xcp_info_attrs[] = {
862 	&ualink_xcp_info_link_type.attr,
863 	&ualink_xcp_info_accel_id.attr,
864 	&ualink_xcp_info_bandwidth.attr,
865 	&ualink_xcp_info_latency.attr,
866 	&ualink_xcp_info_ppod_id.attr,
867 	&ualink_xcp_info_ppod_size.attr,
868 	&ualink_xcp_info_vpod_id.attr,
869 	&ualink_xcp_info_vpod_size.attr,
870 	&ualink_xcp_info_vpod_active_accels.attr,
871 	&ualink_xcp_info_addr_mode.attr,
872 	&ualink_xcp_info_accel_state.attr,
873 	&ualink_xcp_info_local_accels.attr,
874 	NULL
875 };
876 
877 static umode_t ualink_xcp_info_is_visible(struct kobject *kobj,
878 					  struct attribute *attr, int n)
879 {
880 	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
881 					      ualink.kobj);
882 
883 	if (!xcp->valid)
884 		return 0;
885 
886 	return attr->mode;
887 }
888 
889 static const struct attribute_group ualink_xcp_info_group = {
890 	.attrs = ualink_xcp_info_attrs,
891 	.is_visible = ualink_xcp_info_is_visible,
892 };
893 
894 static const struct kobj_type ualink_xcp_info_ktype = {
895 	.sysfs_ops = &kobj_sysfs_ops
896 };
897 
898 static void ualink_info_release(struct kobject *kobj)
899 {
900 	kfree(to_ualink_info(kobj));
901 }
902 
903 static const struct kobj_type ualink_info_ktype = {
904 	.release = ualink_info_release,
905 	.sysfs_ops = &kobj_sysfs_ops
906 };
907 
908 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
909 UALINK_VALUE_SHOW(ppod_setup, accel_id,  ppod.accel_id,  "%u");
910 UALINK_VALUE_SHOW(ppod_setup, bandwidth, ppod.bandwidth, "%u");
911 UALINK_VALUE_SHOW(ppod_setup, latency,   ppod.latency,   "%u");
912 UALINK_UUID_SHOW(ppod_setup, ppod_id, ppod.id);
913 UALINK_VALUE_SHOW(ppod_setup, ppod_size, ppod.size,      "%u");
914 UALINK_IDARRAY_SHOW(ppod_setup, local_accels, local_accels, n_local_accels);
915 
916 UALINK_VALUE_STORE(ppod_setup, accel_id,  ppod.accel_id,  u32, 10);
917 UALINK_VALUE_STORE(ppod_setup, bandwidth, ppod.bandwidth, u32, 10);
918 UALINK_VALUE_STORE(ppod_setup, latency,   ppod.latency,   u32, 10);
919 UALINK_UUID_STORE(ppod_setup, ppod_id,   ppod.id);
920 UALINK_VALUE_STORE(ppod_setup, ppod_size, ppod.size,      u32, 10);
921 UALINK_IDARRAY_STORE(ppod_setup, local_accels, local_accels, n_local_accels,
922 		     AMDGPU_UALINK_ACCEL_MAX);
923 
924 static enum amdgpu_ualink_accel_state
925 check_ppod_state(struct amdgpu_device *adev,
926 		 const struct amdgpu_ualink_ppod_setup *setup)
927 {
928 	if (setup->ppod.size <= 0 || setup->ppod.size > AMDGPU_UALINK_ACCEL_MAX) {
929 		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
930 			setup->ppod.size, AMDGPU_UALINK_ACCEL_MAX);
931 		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
932 	}
933 	if (setup->ppod.accel_id >= setup->ppod.size) {
934 		dev_dbg(adev->dev, "Accelerator ID %u greater or equal pPod size %u\n",
935 			setup->ppod.accel_id, setup->ppod.size);
936 		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
937 	}
938 	return AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
939 }
940 
941 static ssize_t ualink_ppod_setup_commit_store(struct kobject *kobj,
942 					      struct kobj_attribute *attr,
943 					      const char *buf, size_t count)
944 {
945 	struct amdgpu_ualink_ppod_setup *setup = to_ualink_ppod_setup(kobj);
946 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
947 	struct device *dev = kobj_to_dev(info->kobj.parent);
948 	struct drm_device *ddev = dev_get_drvdata(dev);
949 	struct amdgpu_device *adev = drm_to_adev(ddev);
950 	int r;
951 
952 	if (!sysfs_streq(buf, "true"))
953 		return -EINVAL;
954 
955 	r = psp_ual_set_ppod_config(&adev->psp, adev->ualink.psp_if_ver,
956 				    setup);
957 	if (r)
958 		return r;
959 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
960 	if (r)
961 		return r;
962 
963 	/*
964 	 * Hold mgpu_info.mutex to serialize with activate_local_vpod() in
965 	 * vpod_config_commit_store which also holds this lock. Without it,
966 	 * deactivate_accelerator() can race with activate_accelerator(),
967 	 * causing concurrent vm_fini / vm_init on the same NPA VM.
968 	 */
969 	mutex_lock(&mgpu_info.mutex);
970 	if (info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_READY)
971 		deactivate_accelerator(adev);
972 	info->accel_state = check_ppod_state(adev, setup);
973 	/* PPOD is expected to be configured first */
974 	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
975 	mutex_unlock(&mgpu_info.mutex);
976 
977 	/* TODO: If accel_state was ACTIVE, reset all connections */
978 
979 	return count;
980 }
981 
982 #define UALINK_PPOD_SETUP_ATTR(name) __ATTR(name, 0600,			\
983 		ualink_ppod_setup_##name##_show,			\
984 		ualink_ppod_setup_##name##_store)
985 static struct kobj_attribute ualink_ppod_setup_accel_id  = UALINK_PPOD_SETUP_ATTR(accel_id);
986 static struct kobj_attribute ualink_ppod_setup_bandwidth = UALINK_PPOD_SETUP_ATTR(bandwidth);
987 static struct kobj_attribute ualink_ppod_setup_latency   = UALINK_PPOD_SETUP_ATTR(latency);
988 static struct kobj_attribute ualink_ppod_setup_ppod_id   = UALINK_PPOD_SETUP_ATTR(ppod_id);
989 static struct kobj_attribute ualink_ppod_setup_ppod_size = UALINK_PPOD_SETUP_ATTR(ppod_size);
990 static struct kobj_attribute ualink_ppod_setup_local_accels = UALINK_PPOD_SETUP_ATTR(local_accels);
991 static struct kobj_attribute ualink_ppod_setup_commit = __ATTR(commit, 0200, NULL,
992 							       ualink_ppod_setup_commit_store);
993 
994 static const struct attribute *ualink_ppod_setup_attrs[] = {
995 	&ualink_ppod_setup_accel_id.attr,
996 	&ualink_ppod_setup_bandwidth.attr,
997 	&ualink_ppod_setup_latency.attr,
998 	&ualink_ppod_setup_ppod_id.attr,
999 	&ualink_ppod_setup_ppod_size.attr,
1000 	&ualink_ppod_setup_local_accels.attr,
1001 	&ualink_ppod_setup_commit.attr,
1002 	NULL
1003 };
1004 #endif
1005 
1006 static void ualink_ppod_setup_release(struct kobject *kobj)
1007 {
1008 	kfree(to_ualink_ppod_setup(kobj));
1009 }
1010 
1011 static const struct kobj_type ualink_ppod_setup_ktype = {
1012 	.release = ualink_ppod_setup_release,
1013 	.sysfs_ops = &kobj_sysfs_ops
1014 };
1015 
1016 static struct amdgpu_device *find_peer_adev(unsigned int accel_id)
1017 {
1018 	unsigned int i;
1019 
1020 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1021 		struct amdgpu_device *peer_adev = mgpu_info.gpu_ins[i].adev;
1022 
1023 		if (peer_adev->ualink.info &&
1024 		    peer_adev->ualink.info->ppod.accel_id == accel_id)
1025 			return peer_adev;
1026 	}
1027 
1028 	return NULL;
1029 }
1030 
1031 static bool amdgpu_ualink_is_local_accel(struct amdgpu_device *adev,
1032 					 u32 accel_id)
1033 {
1034 	struct amdgpu_ualink_info *info = adev->ualink.info;
1035 	unsigned int i;
1036 
1037 	if (!info)
1038 		return false;
1039 
1040 	for (i = 0; i < info->n_local_accels; i++) {
1041 		if (info->local_accels[i] == accel_id)
1042 			return true;
1043 	}
1044 	return false;
1045 }
1046 
1047 static void activate_accelerator(struct amdgpu_device *adev)
1048 {
1049 	int r;
1050 
1051 	if (adev->ualink.info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
1052 		return;
1053 
1054 	dev_info(adev->dev,
1055 		 "UALINK: activating accelerator accel_id=%u (accel_state=%d)\n",
1056 		 adev->ualink.info->ppod.accel_id,
1057 		 adev->ualink.info->accel_state);
1058 
1059 	/* Enable incoming NPA address translation with NPA VMID */
1060 	r = psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
1061 				   adev->vm_manager.npa_vmid, true);
1062 	if (r) {
1063 		dev_err(adev->dev, "Failed to set NPA config\n");
1064 		return;
1065 	}
1066 
1067 	r = amdgpu_ualink_manager_start(adev);
1068 	if (r) {
1069 		dev_err(adev->dev, "Failed to start UALink manager\n");
1070 		return;
1071 	}
1072 
1073 	r = amdgpu_ualink_sw_init(adev);
1074 	if (r) {
1075 		dev_err(adev->dev, "Failed to init UALink sw r=%d\n", r);
1076 		amdgpu_ualink_manager_stop(adev);
1077 		return;
1078 	}
1079 
1080 	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
1081 	dev_info(adev->dev,
1082 		 "UALINK: accelerator accel_id=%u is now ACTIVE\n",
1083 		 adev->ualink.info->ppod.accel_id);
1084 }
1085 
1086 static void deactivate_accelerator(struct amdgpu_device *adev)
1087 {
1088 	if (adev->ualink.info->accel_state < AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
1089 		return;
1090 
1091 	dev_info(adev->dev,
1092 		 "UALINK: deactivating accelerator accel_id=%u (accel_state=%d -> PPOD_CONFIGURED)\n",
1093 		 adev->ualink.info->ppod.accel_id,
1094 		 adev->ualink.info->accel_state);
1095 
1096 	/* Disable incoming NPA address translation with NPA VMID */
1097 	psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
1098 			       adev->vm_manager.npa_vmid, false);
1099 	/* ignore return value */
1100 	adev->ualink.info->accel_state =
1101 		AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
1102 
1103 	amdgpu_ualink_sw_fini(adev);
1104 	amdgpu_ualink_manager_stop(adev);
1105 }
1106 
1107 static void activate_local_vpod(struct amdgpu_device *adev)
1108 {
1109 	struct amdgpu_ualink_info *info = adev->ualink.info;
1110 	struct amdgpu_device *peer_adev;
1111 	unsigned int accel_id;
1112 	unsigned int i;
1113 
1114 	for (i = 0; i < info->n_local_accels; i++) {
1115 		accel_id = info->local_accels[i];
1116 
1117 		peer_adev = find_peer_adev(accel_id);
1118 		if (WARN_ON(!peer_adev || !peer_adev->ualink.info))
1119 			/* info->local_accels is corrupted? */
1120 			continue;
1121 
1122 		/* Bring the peer up to match the current vPod membership.
1123 		 *
1124 		 * A peer that is not yet ACTIVE (or whose ACTIVE membership
1125 		 * changed on a grow/shrink) has its remote metadata, links and
1126 		 * GART mappings built for a stale member set (or none at all).
1127 		 * Firmware only accepts a full metadata reload while halted, so
1128 		 * surgical per-peer deltas are not possible: fully bounce the
1129 		 * accelerator. deactivate_accelerator() is a no-op when the peer
1130 		 * is not ACTIVE, so this handles first-time bring-up too.
1131 		 * Unchanged ACTIVE peers are left untouched.
1132 		 */
1133 		if (amdgpu_ualink_vpod_membership_changed(peer_adev)) {
1134 			dev_info(peer_adev->dev,
1135 				 "UALINK: (re)configuring vpod for accel_id=%u\n",
1136 				 peer_adev->ualink.info->ppod.accel_id);
1137 			deactivate_accelerator(peer_adev);
1138 			activate_accelerator(peer_adev);
1139 		}
1140 	}
1141 }
1142 
1143 static inline bool __is_vpod_peer(struct amdgpu_ualink_info *info,
1144 				  struct amdgpu_ualink_info *peer_info)
1145 {
1146 	return peer_info->vpod.id == info->vpod.id &&
1147 	       uuid_equal(&peer_info->ppod.id, &info->ppod.id);
1148 }
1149 
1150 static int __check_local_vpod_integrity(struct amdgpu_device *adev)
1151 {
1152 	DECLARE_BITMAP(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
1153 	struct amdgpu_ualink_info *info = adev->ualink.info;
1154 	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
1155 	struct amdgpu_ualink_info *peer_info;
1156 	struct amdgpu_device *peer_adev;
1157 	unsigned int i, n_local_accels;
1158 	unsigned int accel_id;
1159 	/* Check that all local accelerators listed in vpod_active_accels have
1160 	 * matching pod IDs
1161 	 */
1162 	bitmap_zero(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
1163 	n_local_accels = 0;
1164 	__set_bit(info->ppod.accel_id, local_accel_ids);
1165 	local_accels[n_local_accels++] = info->ppod.accel_id;
1166 
1167 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1168 		peer_adev = mgpu_info.gpu_ins[i].adev;
1169 		if (peer_adev == adev || !peer_adev->ualink.info)
1170 			continue;
1171 
1172 		peer_info = peer_adev->ualink.info;
1173 		/* peer device ppod not configured */
1174 		if (peer_info->accel_state <
1175 		    AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
1176 			dev_info(adev->dev,
1177 				 "UALINK: integrity EAGAIN: peer accel_id=%u not ppod-configured (accel_state=%d)\n",
1178 				 peer_info->ppod.accel_id,
1179 				 peer_info->accel_state);
1180 			return -EAGAIN;
1181 		}
1182 
1183 		accel_id = peer_info->ppod.accel_id;
1184 		if (!test_bit(accel_id, info->vpod.active_accel_bits))
1185 			continue;
1186 		/* peer device vpod not configured */
1187 		if (peer_info->accel_state <
1188 		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
1189 			dev_info(adev->dev,
1190 				 "UALINK: integrity EAGAIN: vpod peer accel_id=%u not vpod-configured (accel_state=%d)\n",
1191 				 accel_id, peer_info->accel_state);
1192 			return -EAGAIN;
1193 		}
1194 
1195 		if (!uuid_equal(&peer_info->ppod.id, &info->ppod.id)) {
1196 			dev_warn(adev->dev,
1197 				 "UALINK: integrity fail: peer %u ppod_id doesn't match: %pU != %pU\n",
1198 				 peer_info->ppod.accel_id, &peer_info->ppod.id,
1199 				 &info->ppod.id);
1200 			return -EINVAL;
1201 		}
1202 
1203 		if (peer_info->ppod.size != info->ppod.size) {
1204 			dev_warn(adev->dev,
1205 				 "UALINK: integrity fail: peer %u ppod_size doesn't match: %u != %u\n",
1206 				 accel_id, peer_info->ppod.size,
1207 				 info->ppod.size);
1208 			return -EINVAL;
1209 		}
1210 
1211 		if (peer_info->vpod.id != info->vpod.id) {
1212 			dev_warn(adev->dev,
1213 				 "UALINK: integrity fail: peer %u vpod_id doesn't match: %u != %u\n",
1214 				 accel_id, peer_info->vpod.id, info->vpod.id);
1215 			return -EINVAL;
1216 		}
1217 		if (peer_info->vpod.size != info->vpod.size) {
1218 			dev_warn(adev->dev,
1219 				 "UALINK: integrity fail: peer %u vpod_size doesn't match: %u != %u\n",
1220 				 accel_id, peer_info->vpod.size,
1221 				 info->vpod.size);
1222 			return -EINVAL;
1223 		}
1224 		if (peer_info->vpod.addr_mode != info->vpod.addr_mode) {
1225 			dev_warn(adev->dev,
1226 				 "UALINK: integrity fail: peer %u addr_mode doesn't match: %u != %u\n",
1227 				 accel_id, peer_info->vpod.addr_mode,
1228 				 info->vpod.addr_mode);
1229 			return -EINVAL;
1230 		}
1231 		if (!bitmap_equal(peer_info->vpod.active_accel_bits,
1232 				  info->vpod.active_accel_bits,
1233 				  AMDGPU_UALINK_ACCEL_MAX)) {
1234 			dev_warn(adev->dev,
1235 				 "UALINK: integrity fail: peer %u vpod_active_accels don't match\n",
1236 				 accel_id);
1237 			return -EINVAL;
1238 		}
1239 
1240 		if (__test_and_set_bit(accel_id, local_accel_ids)) {
1241 			dev_warn(adev->dev,
1242 				 "UALINK: integrity fail: duplicate accel_id %u among local vpod peers\n",
1243 				 accel_id);
1244 			return -EINVAL;
1245 		}
1246 		local_accels[n_local_accels++] = accel_id;
1247 	}
1248 
1249 	for (i = 0; i < mgpu_info.num_gpu; i++) {
1250 		peer_adev = mgpu_info.gpu_ins[i].adev;
1251 		peer_info = peer_adev->ualink.info;
1252 
1253 		if (!peer_info)
1254 			continue;
1255 		if (peer_adev != adev && !__is_vpod_peer(info, peer_info))
1256 			continue;
1257 
1258 		peer_info->n_local_accels = n_local_accels;
1259 		memcpy(peer_info->local_accels, local_accels,
1260 		       sizeof(local_accels));
1261 	}
1262 
1263 	return 0;
1264 }
1265 
1266 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev)
1267 {
1268 	int ret;
1269 
1270 	if (adev->ualink.info->accel_state <
1271 		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED ||
1272 	    adev->ualink.info->accel_state ==
1273 		    AMDGPU_UALINK_ACCEL_STATE_ERROR)
1274 		return;
1275 
1276 	ret = __check_local_vpod_integrity(adev);
1277 	if (ret && ret != -EAGAIN) {
1278 		dev_err(adev->dev,
1279 			"Local vpod integrity check failed: %d\n", ret);
1280 		return;
1281 	}
1282 	if (ret == -EAGAIN) {
1283 		dev_info(adev->dev,
1284 			 "UALINK: activate deferred, waiting for local vpod peers to reach VPOD_CONFIGURED\n");
1285 		return;
1286 	}
1287 
1288 	dev_info(adev->dev,
1289 		 "UALINK: integrity OK, applying local vpod (%u local accels)\n",
1290 		 adev->ualink.info->n_local_accels);
1291 	activate_local_vpod(adev);
1292 }
1293 
1294 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1295 UALINK_VALUE_SHOW(vpod_config, vpod_id,   vpod.id,   "%u");
1296 UALINK_VALUE_SHOW(vpod_config, vpod_size, vpod.size, "%u");
1297 UALINK_IDBITS_SHOW(vpod_config, vpod_active_accels, vpod.active_accel_bits);
1298 UALINK_ENUM_SHOW(vpod_config, addr_mode, vpod.addr_mode);
1299 
1300 UALINK_VALUE_STORE(vpod_config, vpod_id,   vpod.id,   u32, 10);
1301 UALINK_VALUE_STORE(vpod_config, vpod_size, vpod.size, u32, 10);
1302 UALINK_IDBITS_STORE(vpod_config, vpod_active_accels, vpod.active_accel_bits,
1303 		    AMDGPU_UALINK_ACCEL_MAX);
1304 UALINK_ENUM_STORE(vpod_config, addr_mode, vpod.addr_mode);
1305 
1306 static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj,
1307 					       struct kobj_attribute *attr,
1308 					       const char *buf, size_t count)
1309 {
1310 	struct amdgpu_ualink_vpod_config *config = to_ualink_vpod_config(kobj);
1311 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
1312 	struct device *dev = kobj_to_dev(info->kobj.parent);
1313 	struct drm_device *ddev = dev_get_drvdata(dev);
1314 	struct amdgpu_device *adev = drm_to_adev(ddev);
1315 	enum amdgpu_ualink_accel_state prev_state;
1316 	u32 prev_vpod_id;
1317 	int r;
1318 
1319 	if (!sysfs_streq(buf, "true"))
1320 		return -EINVAL;
1321 
1322 	dev_dbg(adev->dev,
1323 		"UALINK: vpod-commit enter: accel_state=%d staged vpod_id=%u vpod_size=%u addr_mode=%u\n",
1324 		info->accel_state, config->vpod.id, config->vpod.size,
1325 		config->vpod.addr_mode);
1326 
1327 	if (info->accel_state < AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
1328 		dev_warn(adev->dev,
1329 			 "UALINK: vpod-commit rejected, ppod not configured (accel_state=%d)\n",
1330 			 info->accel_state);
1331 		return -EINVAL;
1332 	}
1333 
1334 	prev_state = info->accel_state;
1335 	prev_vpod_id = info->vpod.id;
1336 	r = psp_ual_set_vpod_config(&adev->psp, adev->ualink.psp_if_ver,
1337 				    config);
1338 	if (r) {
1339 		dev_warn(adev->dev,
1340 			 "UALINK: vpod-commit psp_ual_set_vpod_config failed: %d\n",
1341 			 r);
1342 		return r;
1343 	}
1344 
1345 	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
1346 	if (r) {
1347 		dev_warn(adev->dev,
1348 			 "UALINK: vpod-commit psp_ual_query_info failed: %d\n",
1349 			 r);
1350 		return r;
1351 	}
1352 
1353 	dev_dbg(adev->dev,
1354 		"UALINK: vpod-commit fw read-back: vpod_id=%u vpod_size=%u accel_id=%u ppod.size=%u addr_mode=%u\n",
1355 		info->vpod.id, info->vpod.size, info->ppod.accel_id,
1356 		info->ppod.size, info->vpod.addr_mode);
1357 
1358 	if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID &&
1359 	    !__check_vpod_info(adev, info)) {
1360 		dev_err(adev->dev,
1361 			"UALINK: vpod-commit __check_vpod_info() failed\n");
1362 		return -EINVAL;
1363 	}
1364 
1365 	/* The integrity check makes sure each new GPU is consistent with the
1366 	 * other GPUs already in the vPod. All known local GPUs can become
1367 	 * "ready" at the same time.
1368 	 *
1369 	 * Misconfiguration of one GPU does not reduce the state of other GPUs
1370 	 * already in the vPod. GPU is intentionally not put to ERROR state if
1371 	 * misconfiguration occurs.
1372 	 */
1373 	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
1374 					     UAL_CFG_VPOD);
1375 	if (r)
1376 		return r;
1377 
1378 	return count;
1379 }
1380 
1381 #define UALINK_VPOD_CONFIG_ATTR(name) __ATTR(name, 0600,		\
1382 		ualink_vpod_config_##name##_show,			\
1383 		ualink_vpod_config_##name##_store)
1384 static struct kobj_attribute ualink_vpod_config_vpod_id   = UALINK_VPOD_CONFIG_ATTR(vpod_id);
1385 static struct kobj_attribute ualink_vpod_config_vpod_size = UALINK_VPOD_CONFIG_ATTR(vpod_size);
1386 static struct kobj_attribute ualink_vpod_config_vpod_active_accels =
1387 						UALINK_VPOD_CONFIG_ATTR(vpod_active_accels);
1388 static struct kobj_attribute ualink_vpod_config_addr_mode = UALINK_VPOD_CONFIG_ATTR(addr_mode);
1389 static struct kobj_attribute ualink_vpod_config_commit = __ATTR(commit, 0200, NULL,
1390 								ualink_vpod_config_commit_store);
1391 
1392 static const struct attribute *ualink_vpod_config_attrs[] = {
1393 	&ualink_vpod_config_vpod_id.attr,
1394 	&ualink_vpod_config_vpod_size.attr,
1395 	&ualink_vpod_config_vpod_active_accels.attr,
1396 	&ualink_vpod_config_addr_mode.attr,
1397 	&ualink_vpod_config_commit.attr,
1398 	NULL
1399 };
1400 #endif
1401 
1402 static void ualink_vpod_config_release(struct kobject *kobj)
1403 {
1404 	kfree(to_ualink_vpod_config(kobj));
1405 }
1406 
1407 static const struct kobj_type ualink_vpod_config_ktype = {
1408 	.release = ualink_vpod_config_release,
1409 	.sysfs_ops = &kobj_sysfs_ops
1410 };
1411 
1412 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1413 UALINK_VALUE_SHOW(station_config, flags, flags, "0x%x");
1414 
1415 UALINK_VALUE_STORE(station_config, flags, flags, u8, 16);
1416 
1417 static size_t lane_bitmap_to_string(char *str,
1418 		const struct amdgpu_ualink_station_config *stations)
1419 {
1420 	size_t i;
1421 
1422 	for (i = 0; i < stations->n_stations; i++)
1423 		str[i] = stations->lane_en_bitmap[i] +
1424 			(stations->lane_en_bitmap[i] < 10 ? '0' : 'a' - 10);
1425 	str[i] = '\0';
1426 
1427 	return i;
1428 }
1429 static ssize_t
1430 ualink_station_config_lane_en_bitmap_show(struct kobject *kobj,
1431 					  struct kobj_attribute *attr,
1432 					  char *buf)
1433 {
1434 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1435 	char str[AMDGPU_UALINK_STATIONS_MAX + 1];
1436 
1437 	lane_bitmap_to_string(str, stations);
1438 	return sysfs_emit(buf, "%s\n", str);
1439 }
1440 static ssize_t
1441 ualink_station_config_lane_en_bitmap_store(struct kobject *kobj,
1442 					   struct kobj_attribute *attr,
1443 					   const char *buf, size_t count)
1444 {
1445 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1446 	u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX];
1447 	u32 n_stations = 0;
1448 	bool end = false;
1449 	size_t i;
1450 
1451 	for (i = 0; i < count; i++) {
1452 		/* Accept any number of \n in the end */
1453 		if (buf[i] == '\n') {
1454 			end = true;
1455 			continue;
1456 		}
1457 		/* Don't accept any more data after \n */
1458 		if (end)
1459 			return -EINVAL;
1460 		/* Don't accept more data than the array size */
1461 		if (i >= sizeof(lane_en_bitmap))
1462 			return -ENOSPC;
1463 		/* Accept hex digits */
1464 		if (buf[i] >= '0' && buf[i] <= '9')
1465 			lane_en_bitmap[n_stations++] = buf[i] - '0';
1466 		else if (buf[i] >= 'a' && buf[i] <= 'f')
1467 			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'a';
1468 		else if (buf[i] >= 'A' && buf[i] <= 'F')
1469 			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'A';
1470 		else
1471 			return -EINVAL;
1472 	}
1473 
1474 	memcpy(stations->lane_en_bitmap, lane_en_bitmap, sizeof(lane_en_bitmap));
1475 	stations->n_stations = n_stations;
1476 	return i;
1477 }
1478 
1479 static ssize_t ualink_station_config_commit_store(struct kobject *kobj,
1480 						  struct kobj_attribute *attr,
1481 						  const char *buf, size_t count)
1482 {
1483 	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
1484 	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
1485 	struct device *dev = kobj_to_dev(info->kobj.parent);
1486 	struct drm_device *ddev = dev_get_drvdata(dev);
1487 	struct amdgpu_device *adev = drm_to_adev(ddev);
1488 	int r;
1489 
1490 	/* DF reconfiguration does not interact with accelerator state */
1491 
1492 	if (!sysfs_streq(buf, "true"))
1493 		return -EINVAL;
1494 
1495 	r = psp_ual_set_station_config(&adev->psp, adev->ualink.psp_if_ver, stations);
1496 	if (r)
1497 		return r;
1498 
1499 	return count;
1500 }
1501 
1502 #define UALINK_STATION_CONFIG_ATTR(name) __ATTR(name, 0600,		\
1503 		ualink_station_config_##name##_show,			\
1504 		ualink_station_config_##name##_store)
1505 static struct kobj_attribute ualink_station_config_flags  = UALINK_STATION_CONFIG_ATTR(flags);
1506 static struct kobj_attribute ualink_station_config_lane_en_bitmap =
1507 							UALINK_STATION_CONFIG_ATTR(lane_en_bitmap);
1508 static struct kobj_attribute ualink_station_config_commit =
1509 				__ATTR(commit, 0200, NULL, ualink_station_config_commit_store);
1510 
1511 static const struct attribute *ualink_station_config_attrs[] = {
1512 	&ualink_station_config_flags.attr,
1513 	&ualink_station_config_lane_en_bitmap.attr,
1514 	&ualink_station_config_commit.attr,
1515 	NULL
1516 };
1517 #endif
1518 
1519 static void ualink_station_config_release(struct kobject *kobj)
1520 {
1521 	kfree(to_ualink_station_config(kobj));
1522 }
1523 
1524 static const struct kobj_type ualink_station_config_ktype = {
1525 	.release = ualink_station_config_release,
1526 	.sysfs_ops = &kobj_sysfs_ops
1527 };
1528 
1529 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block)
1530 {
1531 	struct amdgpu_ualink_station_config *stations;
1532 	struct amdgpu_ualink_vpod_config *vpod_config;
1533 	struct amdgpu_ualink_ppod_setup *ppod_setup;
1534 	struct amdgpu_device *adev = ip_block->adev;
1535 	struct amdgpu_ualink_info *info;
1536 
1537 	info = kzalloc_obj(*info);
1538 	ppod_setup = kzalloc_obj(*ppod_setup);
1539 	vpod_config = kzalloc_obj(*vpod_config);
1540 	stations = kzalloc_obj(*stations);
1541 	if (!info || !ppod_setup || !vpod_config || !stations) {
1542 		kfree(info);
1543 		kfree(ppod_setup);
1544 		kfree(vpod_config);
1545 		kfree(stations);
1546 		return -ENOMEM;
1547 	}
1548 
1549 	info->ppod.accel_id = 0xffffffff;
1550 	info->ppod.bandwidth = 0xffffffff;
1551 	info->ppod.latency = 0xffffffff;
1552 	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
1553 	info->vpod.addr_mode = AMDGPU_UALINK_ADDR_MODE_MAX;
1554 
1555 	/*
1556 	 * Initialize the kobjects here so their lifetime is tied to the UALink
1557 	 * manager software state. amdgpu_ualink_object_fini() drops the final
1558 	 * reference via kobject_put().
1559 	 */
1560 	kobject_init(&info->kobj, &ualink_info_ktype);
1561 	kobject_init(&ppod_setup->kobj, &ualink_ppod_setup_ktype);
1562 	kobject_init(&vpod_config->kobj, &ualink_vpod_config_ktype);
1563 	kobject_init(&stations->kobj, &ualink_station_config_ktype);
1564 
1565 	adev->ualink.info = info;
1566 	adev->ualink.setup = ppod_setup;
1567 	adev->ualink.config = vpod_config;
1568 	adev->ualink.stations = stations;
1569 
1570 	return 0;
1571 }
1572 
1573 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block)
1574 {
1575 	amdgpu_ualink_object_fini(ip_block->adev);
1576 	return 0;
1577 }
1578 
1579 static int ualink_kobj_add(struct kobject *kobj, struct kobject *parent,
1580 			   const char *name, const struct attribute **attrs)
1581 {
1582 	int r;
1583 
1584 	r = kobject_add(kobj, parent, "%s", name);
1585 	if (r)
1586 		return r;
1587 	r = sysfs_create_files(kobj, attrs);
1588 	if (r)
1589 		kobject_del(kobj);
1590 
1591 	return r;
1592 }
1593 
1594 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev);
1595 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev);
1596 
1597 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev)
1598 {
1599 	struct amdgpu_ualink_info *info = adev->ualink.info;
1600 	int r;
1601 
1602 	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
1603 		return 0;
1604 
1605 	/* ualink parent node */
1606 	r = ualink_kobj_add(&info->kobj, &adev->dev->kobj, "ualink",
1607 			    ualink_info_attrs);
1608 	if (r)
1609 		goto err;
1610 
1611 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
1612 	r = ualink_kobj_add(&adev->ualink.setup->kobj, &info->kobj, "setup",
1613 			    ualink_ppod_setup_attrs);
1614 	if (r)
1615 		goto err_info;
1616 
1617 	r = ualink_kobj_add(&adev->ualink.config->kobj, &info->kobj, "config",
1618 			    ualink_vpod_config_attrs);
1619 	if (r)
1620 		goto err_setup;
1621 
1622 	r = ualink_kobj_add(&adev->ualink.stations->kobj, &info->kobj,
1623 			    "stations", ualink_station_config_attrs);
1624 	if (r)
1625 		goto err_config;
1626 #else
1627 	r = kobject_add(&adev->ualink.setup->kobj, &info->kobj, "%s",
1628 			"setup");
1629 	if (r)
1630 		goto err_info;
1631 
1632 	r = kobject_add(&adev->ualink.config->kobj, &info->kobj, "%s",
1633 			"config");
1634 	if (r)
1635 		goto err_setup;
1636 
1637 	r = kobject_add(&adev->ualink.stations->kobj, &info->kobj, "%s",
1638 			"stations");
1639 	if (r)
1640 		goto err_config;
1641 #endif
1642 
1643 	amdgpu_ualink_xcp_sysfs_init(adev);
1644 	adev->ualink.sysfs_init = true;
1645 
1646 	return 0;
1647 
1648 err_config:
1649 	kobject_del(&adev->ualink.config->kobj);
1650 err_setup:
1651 	kobject_del(&adev->ualink.setup->kobj);
1652 err_info:
1653 	kobject_del(&info->kobj);
1654 err:
1655 	dev_warn(adev->dev, "Failed to create UALink sysfs: %d\n", r);
1656 	return 0;
1657 }
1658 
1659 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev)
1660 {
1661 	if (!adev->ualink.sysfs_init)
1662 		return;
1663 
1664 	amdgpu_ualink_xcp_sysfs_fini(adev);
1665 	kobject_del(&adev->ualink.stations->kobj);
1666 	kobject_del(&adev->ualink.config->kobj);
1667 	kobject_del(&adev->ualink.setup->kobj);
1668 	kobject_del(&adev->ualink.info->kobj);
1669 	adev->ualink.sysfs_init = false;
1670 }
1671 
1672 static int amdgpu_ualink_xcp_sysfs_add(struct amdgpu_xcp *xcp)
1673 {
1674 	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
1675 	int r;
1676 
1677 	r = kobject_init_and_add(&xcp->ualink.kobj, &ualink_xcp_info_ktype,
1678 				 &xcp->ddev->dev->kobj, "ualink");
1679 	if (r)
1680 		goto err;
1681 
1682 	r = sysfs_create_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1683 	if (r)
1684 		goto err;
1685 
1686 	/* pin info so it outlives this node regardless of teardown order */
1687 	kobject_get(&adev->ualink.info->kobj);
1688 	xcp->ualink.sysfs = true;
1689 	return 0;
1690 err:
1691 	kobject_put(&xcp->ualink.kobj);
1692 	return r;
1693 }
1694 
1695 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp)
1696 {
1697 	if (!xcp->ualink.sysfs)
1698 		return;
1699 
1700 	sysfs_update_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1701 }
1702 
1703 static void amdgpu_ualink_xcp_sysfs_remove(struct amdgpu_xcp *xcp)
1704 {
1705 	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
1706 
1707 	if (!xcp->ualink.sysfs)
1708 		return;
1709 
1710 	/* group is only populated for valid partitions */
1711 	if (xcp->valid)
1712 		sysfs_remove_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
1713 	kobject_put(&adev->ualink.info->kobj);
1714 	kobject_put(&xcp->ualink.kobj);
1715 	xcp->ualink.sysfs = false;
1716 }
1717 
1718 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev)
1719 {
1720 	struct amdgpu_xcp *xcp;
1721 	int i;
1722 
1723 	if (!adev->xcp_mgr)
1724 		return;
1725 
1726 	for (i = 0; i < MAX_XCP; i++) {
1727 		xcp = &adev->xcp_mgr->xcp[i];
1728 		if (!xcp->ddev || amdgpu_xcp_is_primary(xcp))
1729 			continue;
1730 		amdgpu_ualink_xcp_sysfs_add(xcp);
1731 	}
1732 }
1733 
1734 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev)
1735 {
1736 	struct amdgpu_xcp *xcp;
1737 	int i;
1738 
1739 	if (!adev->xcp_mgr)
1740 		return;
1741 
1742 	for (i = 0; i < MAX_XCP; i++) {
1743 		xcp = &adev->xcp_mgr->xcp[i];
1744 		amdgpu_ualink_xcp_sysfs_remove(xcp);
1745 	}
1746 }
1747 
1748 static int amdgpu_ualink_npa_alloc_va(struct amdgpu_device *adev,
1749 			       struct drm_mm_node *mm_node,
1750 			       u64 va, u64 range_start,
1751 			       u64 range_end, int size)
1752 {
1753 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1754 	u64 alignment = 0;
1755 	int ret;
1756 
1757 	mutex_lock(&npa_mm->mm_lock);
1758 	if (va) {
1759 		mm_node->start = va;
1760 		mm_node->size = size;
1761 		ret = drm_mm_reserve_node(&npa_mm->mm, mm_node);
1762 	} else {
1763 		if (!range_start && !range_end) {
1764 			range_start = npa_mm->va_start;
1765 			range_end = npa_mm->va_start + npa_mm->va_size;
1766 		}
1767 
1768 		/* If size is greater than 2MB (or 512 pages),
1769 		 * align it to 2MB (or 512 pages) granularity.
1770 		 */
1771 		if (size >= 0x200)
1772 			alignment = 0x200;
1773 
1774 		ret = drm_mm_insert_node_in_range(&npa_mm->mm, mm_node, size,
1775 						  alignment, 0, range_start,
1776 						  range_end, 0);
1777 	}
1778 	mutex_unlock(&npa_mm->mm_lock);
1779 
1780 	if (ret)
1781 		dev_err(adev->dev, "Failed to allocate NPA address\n");
1782 
1783 	return ret;
1784 }
1785 
1786 static void amdgpu_ualink_npa_free_va(struct amdgpu_device *adev,
1787 			       struct drm_mm_node *mm_node)
1788 {
1789 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1790 
1791 	mutex_lock(&npa_mm->mm_lock);
1792 	drm_mm_remove_node(mm_node);
1793 	mutex_unlock(&npa_mm->mm_lock);
1794 }
1795 
1796 static void amdgpu_ualink_npa_mm_init(struct amdgpu_device *adev)
1797 {
1798 	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
1799 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
1800 	u64 va_size;
1801 
1802 	mutex_init(&npa_mm->mm_lock);
1803 
1804 	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
1805 		/* 51 bit address space in Source-Identification mode.
1806 		 * 39 bits in page granularity.
1807 		 */
1808 		npa_mm->va_start = 0;
1809 		va_size = GENMASK_ULL(38, 0);
1810 	} else {
1811 		/* 41 bit address space in Source-Alias mode.
1812 		 * 29 bits in page granularity.
1813 		 */
1814 		/* First 8MB is reserved for Metadata NPAs */
1815 		npa_mm->va_start = 0x800;
1816 		va_size = GENMASK_ULL(28, 0);
1817 	}
1818 
1819 	npa_mm->va_size = va_size - npa_mm->va_start;
1820 	drm_mm_init(&npa_mm->mm, npa_mm->va_start, npa_mm->va_size);
1821 }
1822 
1823 static void amdgpu_ualink_npa_mm_fini(struct amdgpu_device *adev)
1824 {
1825 	mutex_destroy(&adev->ualink.npa_mm.mm_lock);
1826 	drm_mm_takedown(&adev->ualink.npa_mm.mm);
1827 }
1828 
1829 /* The caller of this function is expected to hold the XA lock when calling
1830  * this function.
1831  */
1832 static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev,
1833 				   struct amdgpu_ualink_handle *handle)
1834 {
1835 	bool unique = false;
1836 
1837 	do {
1838 		handle->handle_lo = get_random_u64();
1839 		/* Replace bottom 10 bits in handle_lo with accId */
1840 		handle->handle_lo &= ~AMDGPU_UALINK_HANDLE_ACCID_MASK;
1841 		handle->handle_lo |= adev->ualink.info->ppod.accel_id;
1842 
1843 		/* Don't generate/store a Handle with value 0. */
1844 		if (!handle->handle_lo)
1845 			continue;
1846 
1847 		/* Find if the handle already exists in the exporter xarray.
1848 		 * If it already exists, then regenerate the handle since we
1849 		 * want the handle to be unique.
1850 		 */
1851 		unique = !xa_load(&adev->ualink.exp_xa, handle->handle_lo);
1852 	} while (!unique);
1853 
1854 	handle->handle_hi = get_random_u64();
1855 	dev_dbg(adev->dev, "GENERATE-HANDLE: generated handle: %llx:%llx\n",
1856 		handle->handle_hi, handle->handle_lo);
1857 }
1858 
1859 static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref)
1860 {
1861 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
1862 	struct amdgpu_device *adev;
1863 
1864 	exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node,
1865 				   refcount);
1866 	adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev);
1867 	queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work);
1868 }
1869 
1870 static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref)
1871 {
1872 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
1873 	struct amdgpu_device *adev;
1874 
1875 	imp_xa_node = container_of(ref, struct amdgpu_ualink_imp_xa_node,
1876 				   refcount);
1877 	adev = imp_xa_node->adev;
1878 
1879 	dev_dbg(adev->dev, "IMP-CLEANUP: Remove entry from XA for handle:%llx:%llx\n",
1880 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1881 
1882 	/* Remove node from the Xarray */
1883 	if (!xa_erase(&adev->ualink.imp_xa, imp_xa_node->handle.handle_lo))
1884 		dev_err(adev->dev, "IMP-CLEANUP: Failed to find entry in XA for handle:%llx:%llx\n",
1885 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1886 
1887 	if (imp_xa_node->npa_addr && imp_xa_node->size && imp_xa_node->dmabuf) {
1888 		dev_dbg(adev->dev,
1889 			"IMP-CLEANUP: dmabuf free for NPA:%llx handle:%llx:%llx, fc:%lu\n",
1890 			imp_xa_node->npa_addr, imp_xa_node->handle.handle_hi,
1891 			imp_xa_node->handle.handle_lo,
1892 			file_count(imp_xa_node->dmabuf->file));
1893 		dma_buf_put(imp_xa_node->dmabuf);
1894 		drm_gem_handle_delete(adev->ualink.client.file,
1895 				      imp_xa_node->gem_handle);
1896 
1897 	}
1898 
1899 	dev_dbg(adev->dev, "IMP-CLEANUP: Freeing XA entry for handle:%llx:%llx\n",
1900 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
1901 
1902 	kfree(imp_xa_node);
1903 }
1904 
1905 static int amdgpu_ualink_exp_xa_entry_get(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
1906 {
1907 	return kref_get_unless_zero(&exp_xa_node->refcount);
1908 }
1909 
1910 static void amdgpu_ualink_exp_xa_entry_put(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
1911 {
1912 	kref_put(&exp_xa_node->refcount, amdgpu_ualink_cleanup_exp_xa_node);
1913 }
1914 
1915 static int amdgpu_ualink_imp_xa_entry_get(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
1916 {
1917 	return kref_get_unless_zero(&imp_xa_node->refcount);
1918 }
1919 
1920 static void amdgpu_ualink_imp_xa_entry_put(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
1921 {
1922 	kref_put(&imp_xa_node->refcount, amdgpu_ualink_cleanup_imp_xa_node);
1923 }
1924 
1925 static int amdgpu_ualink_send_npa_release_msg(struct amdgpu_device *adev,
1926 				    u32 remote_acc_id,
1927 				    struct amdgpu_ualink_handle handle)
1928 {
1929 	u32 dw0, dw1, dw2, dw3;
1930 
1931 	dw0 = lower_32_bits(handle.handle_lo);
1932 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1933 	dw0 |= AMDGPU_UALINK_NPA_RELEASE_MSG;
1934 
1935 	dw1 = upper_32_bits(handle.handle_lo);
1936 	dw2 = lower_32_bits(handle.handle_hi);
1937 	dw3 = upper_32_bits(handle.handle_hi);
1938 
1939 	dev_dbg(adev->dev, "SEND NPA-RELEASE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1940 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1941 
1942 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1943 					      dw2, dw3);
1944 }
1945 
1946 static int amdgpu_ualink_send_npa_revoke_msg(struct amdgpu_device *adev,
1947 					u32 remote_acc_id,
1948 					struct amdgpu_ualink_handle handle)
1949 {
1950 	u32 dw0, dw1, dw2, dw3;
1951 
1952 	dw0 = lower_32_bits(handle.handle_lo);
1953 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1954 	dw0 |= AMDGPU_UALINK_NPA_REVOKE_MSG;
1955 
1956 	dw1 = upper_32_bits(handle.handle_lo);
1957 	dw2 = lower_32_bits(handle.handle_hi);
1958 	dw3 = upper_32_bits(handle.handle_hi);
1959 
1960 	dev_dbg(adev->dev, "SEND NPA-REVOKE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1961 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1962 
1963 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1964 					      dw2, dw3);
1965 }
1966 
1967 static int amdgpu_ualink_send_npa_fail_msg(struct amdgpu_device *adev,
1968 				    u32 remote_acc_id,
1969 				    struct amdgpu_ualink_handle handle,
1970 				    u32 fail_reason)
1971 {
1972 	u32 dw0, dw1, dw2, dw3;
1973 
1974 	dw0 = lower_32_bits(handle.handle_lo);
1975 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1976 	dw0 |= AMDGPU_UALINK_NPA_FAIL_MSG;
1977 
1978 	dw1 = upper_32_bits(handle.handle_lo);
1979 	dw2 = fail_reason & 0xFF;
1980 	dw3 = 0;
1981 
1982 	dev_dbg(adev->dev, "SEND NPA-FAIL: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
1983 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
1984 
1985 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
1986 					      dw2, dw3);
1987 }
1988 
1989 static int amdgpu_ualink_send_npa_rsp_msg(struct amdgpu_device *adev,
1990 					  u32 remote_acc_id,
1991 					  struct amdgpu_ualink_handle handle,
1992 					  u32 npa_addr, u32 size)
1993 {
1994 	u32 dw0, dw1, dw2, dw3;
1995 
1996 	dw0 = lower_32_bits(handle.handle_lo);
1997 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
1998 	dw0 |= AMDGPU_UALINK_NPA_RSP_MSG;
1999 
2000 	dw1 = upper_32_bits(handle.handle_lo);
2001 	dw2 = size;
2002 	dw3 = npa_addr;
2003 
2004 	dev_dbg(adev->dev, "SEND NPA-RSP: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
2005 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
2006 
2007 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
2008 					      dw2, dw3);
2009 }
2010 
2011 static int amdgpu_ualink_send_npa_req_msg(struct amdgpu_device *adev,
2012 					  u32 remote_acc_id,
2013 					  struct amdgpu_ualink_handle handle)
2014 {
2015 	u32 dw0, dw1, dw2, dw3;
2016 
2017 	dw0 =  lower_32_bits(handle.handle_lo);
2018 	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
2019 	dw0 |= AMDGPU_UALINK_NPA_REQ_MSG;
2020 
2021 	dw1 = upper_32_bits(handle.handle_lo);
2022 	dw2 = lower_32_bits(handle.handle_hi);
2023 	dw3 = upper_32_bits(handle.handle_hi);
2024 
2025 	dev_dbg(adev->dev, "SEND NPA-REQ: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
2026 		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
2027 
2028 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
2029 					      dw2, dw3);
2030 }
2031 
2032 static u64 amdgpu_ualink_get_export_pte_flags(struct amdgpu_device *adev,
2033 				       struct amdgpu_bo *bo,
2034 				       u64 mapping_flags)
2035 {
2036 	u64 pte_flags = adev->gmc.init_pte_flags;
2037 
2038 	pte_flags |= (AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE |
2039 		      AMDGPU_PTE_WRITEABLE);
2040 	mapping_flags |= AMDGPU_VM_MTYPE_DEFAULT;
2041 
2042 	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, bo, mapping_flags,
2043 			      &pte_flags);
2044 
2045 	return pte_flags;
2046 }
2047 
2048 static int amdgpu_ualink_unmap_npa_addr(struct amdgpu_device *adev,
2049 					struct amdgpu_bo *bo,
2050 					u64 npa_addr, u64 size)
2051 {
2052 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2053 	uint64_t pte_value = adev->gmc.noretry_flags;
2054 	struct amdgpu_bo *bos[] = { bo };
2055 	struct dma_fence *fence = NULL;
2056 	struct drm_exec exec;
2057 	int r;
2058 
2059 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
2060 	if (unlikely(r)) {
2061 		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_npa_addr\n");
2062 		return r;
2063 	}
2064 
2065 	r = amdgpu_vm_update_range(adev, vm, false, false, true,
2066 				false, NULL, npa_addr, npa_addr + size - 1,
2067 				pte_value, 0, 0, NULL, NULL, &vm->last_update);
2068 	if (r) {
2069 		dev_err(adev->dev,
2070 			"Failed to unmap NPA addr (%llx) from NPA VM\n", npa_addr);
2071 		goto out;
2072 	}
2073 
2074 	r = amdgpu_vm_update_pdes(adev, vm, false);
2075 	if (r) {
2076 		dev_err(adev->dev,
2077 			"Failed %d to update page directories during unmapping NPA: 0x%llx\n",
2078 			r, npa_addr);
2079 		goto out;
2080 	}
2081 
2082 	fence = dma_fence_get(vm->last_update);
2083 	if (fence) {
2084 		r = dma_fence_wait(fence, false);
2085 		dma_fence_put(fence);
2086 		fence = NULL;
2087 		if (r) {
2088 			dev_dbg(adev->dev,
2089 				"UNMAP-NPA: dma fence wait failed, error: %d\n", r);
2090 			goto out;
2091 		}
2092 	}
2093 
2094 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2095 out:
2096 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2097 
2098 	return r;
2099 }
2100 
2101 static int amdgpu_ualink_map_npa_addr(struct amdgpu_device *adev, u64 npa_addr,
2102 				u64 size, struct amdgpu_bo *bo, u64 offset,
2103 				u64 pte_flags)
2104 {
2105 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2106 	struct amdgpu_bo *bos[] = { bo };
2107 	struct dma_fence *fence = NULL;
2108 	struct drm_exec exec;
2109 	int r;
2110 
2111 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
2112 	if (unlikely(r)) {
2113 		dev_err(adev->dev, "Failed to reserve VM and BO in map_npa_addr\n");
2114 		return r;
2115 	}
2116 
2117 	r = amdgpu_vm_update_range(adev, vm, false, false, true,
2118 				false, NULL, npa_addr, npa_addr + size - 1,
2119 				pte_flags, offset, adev->vm_manager.vram_base_offset,
2120 				bo->tbo.resource, NULL, &vm->last_update);
2121 	if (r) {
2122 		dev_warn(adev->dev,
2123 			"Failed to map NPA addr (%llx) into NPA VM\n", npa_addr);
2124 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2125 		goto out;
2126 	}
2127 
2128 	r = amdgpu_vm_update_pdes(adev, vm, false);
2129 	if (r) {
2130 		dev_err(adev->dev,
2131 			"failed %d to update page directories for NPA: 0x%llx\n",
2132 			r, npa_addr);
2133 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2134 		goto unmap_npa;
2135 	}
2136 
2137 	fence = dma_fence_get(vm->last_update);
2138 	if (fence) {
2139 		r = dma_fence_wait(fence, false);
2140 		dma_fence_put(fence);
2141 		fence = NULL;
2142 		if (r) {
2143 			pr_debug("failed %d to dma fence wait\n", r);
2144 			amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2145 			goto unmap_npa;
2146 		}
2147 	}
2148 
2149 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2150 
2151 	/* TLB flush may be needed after updated page directories */
2152 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2153 
2154 	return 0;
2155 
2156 unmap_npa:
2157 	amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
2158 out:
2159 	return r;
2160 }
2161 
2162 static int amdgpu_ualink_send_hello_ack_msg(struct amdgpu_device *adev,
2163 					    u32 remote_acc_id)
2164 {
2165 	dev_dbg(adev->dev, "SEND HELLO-ACK: HELLO-ACK message to remote AccId:%u\n",
2166 		remote_acc_id);
2167 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id,
2168 		AMDGPU_UALINK_HELLO_ACK_MSG,
2169 		0, 0, 0);
2170 }
2171 
2172 static int amdgpu_ualink_send_hello_msg(struct amdgpu_device *adev,
2173 					u32 remote_acc_id)
2174 {
2175 	u32 dw0;
2176 
2177 	dw0 = AMDGPU_UALINK_HELLO_MSG;
2178 	dw0 |= (remote_acc_id << AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT);
2179 	dw0 |= (adev->ualink.info->ppod.accel_id <<
2180 		AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT);
2181 
2182 	dev_dbg(adev->dev, "SEND HELLO: HELLO message to remote AccId:%u\n",
2183 		remote_acc_id);
2184 	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, 0,
2185 					      0, 0);
2186 }
2187 
2188 static u32 amdgpu_ualink_check_conn_ready(struct amdgpu_device *adev,
2189 					  u32 remote_acc_id, u32 gen_count)
2190 {
2191 	struct amdgpu_ualink_connection *conn_state;
2192 	u32 current_gen_count = 0;
2193 
2194 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2195 
2196 	/* Check if the connection is established. */
2197 	mutex_lock(&conn_state->lock);
2198 	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
2199 	    (!gen_count || conn_state->generation_count == gen_count))
2200 		current_gen_count = conn_state->generation_count;
2201 	mutex_unlock(&conn_state->lock);
2202 
2203 	return current_gen_count;
2204 }
2205 
2206 static void amdgpu_ualink_process_hello_ack_msg(struct amdgpu_device *adev,
2207 					       u32 sender_acc_id)
2208 {
2209 	struct amdgpu_ualink_connection *conn_state;
2210 
2211 	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
2212 		dev_err(adev->dev,
2213 			"HELLO-ACK: sender AccId out of range:%u\n",
2214 			sender_acc_id);
2215 		return;
2216 	}
2217 
2218 	conn_state = &adev->ualink.conn_state[sender_acc_id];
2219 
2220 	/* If we are in IN_PROGRESS state, then transition the connection
2221 	 * state to established and signal that the HELLO ACK is received.
2222 	 * Otherwise, ignore the HELLO ACK.
2223 	 */
2224 	mutex_lock(&conn_state->lock);
2225 
2226 	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) {
2227 		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
2228 		conn_state->generation_count++;
2229 		complete(&conn_state->hello_done);
2230 	} else {
2231 		dev_dbg(adev->dev,
2232 			"HELLO-ACK: already connected, ignoring from AccId:%u\n",
2233 			sender_acc_id);
2234 	}
2235 	mutex_unlock(&conn_state->lock);
2236 }
2237 
2238 static void amdgpu_ualink_process_hello_msg(struct amdgpu_device *adev,
2239 					   u32 receiver_acc_id,
2240 					   u32 sender_acc_id,
2241 					   u32 src_acc_id)
2242 {
2243 	struct amdgpu_ualink_connection *conn_state;
2244 	u32 generation_count;
2245 	int r;
2246 
2247 	if (receiver_acc_id != adev->ualink.info->ppod.accel_id) {
2248 		dev_err(adev->dev,
2249 			"HELLO: receiver AccId mismatch got:%u self:%u\n",
2250 			receiver_acc_id, adev->ualink.info->ppod.accel_id);
2251 		return;
2252 	}
2253 
2254 	/* src_acc_id is the AccId received in IH cookie. Confirm it
2255 	 * matches with the sender AccId.
2256 	 */
2257 	if (sender_acc_id != src_acc_id) {
2258 		dev_err(adev->dev,
2259 			"HELLO: sender AccId mismatch sender:%u IH cookie:%u\n",
2260 			sender_acc_id, src_acc_id);
2261 		return;
2262 	}
2263 
2264 	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
2265 		dev_err(adev->dev,
2266 			"HELLO: sender AccId out of range:%u\n",
2267 			sender_acc_id);
2268 		return;
2269 	}
2270 
2271 	conn_state = &adev->ualink.conn_state[sender_acc_id];
2272 
2273 	/* Check if connection is already established. If yes, then receiving HELLO msg
2274 	 * triggers a reset handling scenario.
2275 	 * If the connection is not ready, and we receive a HELLO msg, then
2276 	 * transition the state to PENDING.
2277 	 * Otherwise, leave it IN_PROGRESS.
2278 	 */
2279 	mutex_lock(&conn_state->lock);
2280 	if (conn_state->state != AMDGPU_UALINK_CONN_ESTABLISHED) {
2281 		if (conn_state->state == AMDGPU_UALINK_CONN_NOT_READY)
2282 			conn_state->state = AMDGPU_UALINK_CONN_PENDING;
2283 		/* otherwise, leave it IN_PROGRESS to signal the completion below */
2284 		mutex_unlock(&conn_state->lock);
2285 	} else {
2286 		/* Leave state ESTABLISHED so handle_connection_reset()
2287 		 * performs the transition and imp/exp XA cleanup.
2288 		 */
2289 		generation_count = conn_state->generation_count;
2290 		mutex_unlock(&conn_state->lock);
2291 
2292 		amdgpu_ualink_handle_connection_reset(adev, sender_acc_id,
2293 						AMDGPU_UALINK_CONN_PENDING,
2294 						generation_count);
2295 	}
2296 
2297 	/* Peer may have rebooted. */
2298 	r = amdgpu_ualink_ring_resync_ptrs(adev, sender_acc_id);
2299 	if (r < 0) {
2300 		dev_err(adev->dev,
2301 			"HELLO: resync sender AccId %u ptrs failed %d\n",
2302 			sender_acc_id, r);
2303 		goto out_set_state;
2304 	}
2305 	if (r)
2306 		amdgpu_ualink_reset_peer_rings(adev, sender_acc_id);
2307 
2308 	r = amdgpu_ualink_send_hello_ack_msg(adev, sender_acc_id);
2309 	if (r)
2310 		dev_err(adev->dev, "HELLO-ACK: send failed to remote AccId:%u\n",
2311 			sender_acc_id);
2312 
2313 out_set_state:
2314 	mutex_lock(&conn_state->lock);
2315 	if (r) {
2316 		conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
2317 	} else {
2318 		/* If we are in IN_PROGRESS state and we receivied the HELLO message,
2319 		 * upon receiving the HELLO message, transition the state to ESTABLISHED
2320 		 * and signal the completion.
2321 		 */
2322 		if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS)
2323 			complete(&conn_state->hello_done);
2324 		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
2325 		conn_state->generation_count++;
2326 	}
2327 	mutex_unlock(&conn_state->lock);
2328 }
2329 
2330 /* True if nothing is exported to, or imported from, this remote accelerator. */
2331 static bool amdgpu_ualink_peer_idle(struct amdgpu_device *adev, u32 remote_accel_id)
2332 {
2333 	/* Unlocked: a racing import at worst costs one redundant ring resync. */
2334 	return list_empty(&adev->ualink.imp_handles_list[remote_accel_id]) &&
2335 	       list_empty(&adev->ualink.exp_handles_list[remote_accel_id]);
2336 }
2337 
2338 static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev,
2339 					  u32 remote_acc_id)
2340 {
2341 	struct amdgpu_ualink_connection *conn_state;
2342 	int r;
2343 
2344 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2345 
2346 	/* Connection state management goes through different states.
2347 	 * The states are:
2348 	 * - NOT_READY: The connection is not ready.
2349 	 * - IN_PROGRESS: GPU sent HELLO message and is waiting for the HELLO_ACK.
2350 	 * - PENDING: GPU received HELLO message and is in the process of sending
2351 	 *            the HELLO_ACK.
2352 	 * - ESTABLISHED: The connection is established.
2353 	 */
2354 
2355 	/* Grab the lock and check if connection establishment was
2356 	 * already done by another thread.
2357 	 */
2358 	mutex_lock(&conn_state->lock);
2359 	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS ||
2360 	    conn_state->state == AMDGPU_UALINK_CONN_PENDING) {
2361 		r = -EAGAIN;
2362 		goto out;
2363 	}
2364 
2365 	if (conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED &&
2366 	    !amdgpu_ualink_peer_idle(adev, remote_acc_id)) {
2367 		r = 0;
2368 		goto out;
2369 	}
2370 
2371 	/* Peer is idle or not connected, it may have rebooted. Skip the
2372 	 * handshake if the rings are still in sync, it would reset the remote.
2373 	 */
2374 	r = amdgpu_ualink_ring_resync_ptrs(adev, remote_acc_id);
2375 	if (r < 0)
2376 		goto out;
2377 	if (!r && conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED)
2378 		goto out;
2379 
2380 	amdgpu_ualink_reset_peer_rings(adev, remote_acc_id);
2381 
2382 	conn_state->state = AMDGPU_UALINK_CONN_IN_PROGRESS;
2383 
2384 	mutex_unlock(&conn_state->lock);
2385 
2386 	/* Send HELLO message */
2387 	r = amdgpu_ualink_send_hello_msg(adev, remote_acc_id);
2388 	if (r) {
2389 		dev_warn(adev->dev,
2390 			"HELLO: Send failed to remote AccId:%u\n",
2391 			remote_acc_id);
2392 		goto reset_state;
2393 	}
2394 
2395 	/* Wait for the HELLO_ACK to come back */
2396 	/* complete(conn_state->hello_done) should be called from the IRQ
2397 	 * handler when the HELLO_ACK is received.
2398 	 */
2399 	r = wait_for_completion_interruptible_timeout(&conn_state->hello_done,
2400 				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
2401 	if (r == -ERESTARTSYS) {
2402 		dev_err_ratelimited(adev->dev,
2403 			"HELLO-ACK: interrupted by signal\n");
2404 		goto reset_state;
2405 	} else if (r == 0) {
2406 		dev_warn(adev->dev,
2407 			"HELLO-ACK: Timeout from remote AccId:%u\n",
2408 			remote_acc_id);
2409 		r = -ETIMEDOUT;
2410 		goto reset_state;
2411 	}
2412 
2413 	return 0;
2414 
2415 reset_state:
2416 	mutex_lock(&conn_state->lock);
2417 	conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
2418 out:
2419 	mutex_unlock(&conn_state->lock);
2420 
2421 	return r;
2422 }
2423 
2424 /* Invalidate an importer node's mappings and drop its last ref (frees the
2425  * node, dma-buf and GEM handle). Caller must have unlinked it from the
2426  * per-remote list and must not hold the imp_xa lock.
2427  */
2428 static void amdgpu_ualink_release_imp_xa_node(struct amdgpu_device *adev,
2429 				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
2430 {
2431 	struct amdgpu_bo *bo;
2432 
2433 	dev_dbg(adev->dev,
2434 		"IMP-CLEANUP: handle:%llx:%llx npa:%llx size:%llx\n",
2435 		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
2436 		imp_xa_node->npa_addr, imp_xa_node->size);
2437 
2438 	if (imp_xa_node->dmabuf) {
2439 		bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
2440 		amdgpu_ualink_invalidate_import_mappings(bo);
2441 	}
2442 
2443 	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
2444 }
2445 
2446 static void amdgpu_ualink_cleanup_imp_xa_entries(struct amdgpu_device *adev,
2447 						 u32 remote_acc_id)
2448 {
2449 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
2450 	struct list_head *imp_handles_list;
2451 
2452 	dev_dbg(adev->dev,
2453 		"IMP-RESET: Cleaning up all XA entries for remote:%u\n",
2454 		remote_acc_id);
2455 
2456 	imp_handles_list = &adev->ualink.imp_handles_list[remote_acc_id];
2457 
2458 	xa_lock(&adev->ualink.imp_xa);
2459 	while (!list_empty(imp_handles_list)) {
2460 		imp_xa_node = list_first_entry(imp_handles_list,
2461 					struct amdgpu_ualink_imp_xa_node, list);
2462 		list_del_init(&imp_xa_node->list);
2463 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
2464 		xa_unlock(&adev->ualink.imp_xa);
2465 
2466 		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
2467 
2468 		xa_lock(&adev->ualink.imp_xa);
2469 	}
2470 	xa_unlock(&adev->ualink.imp_xa);
2471 }
2472 
2473 static void amdgpu_ualink_cleanup_exp_xa_entries(struct amdgpu_device *adev,
2474 						 u32 remote_acc_id)
2475 {
2476 	struct amdgpu_ualink_importer_entry *importer_entry;
2477 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2478 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
2479 	struct list_head *exp_handles_list;
2480 	struct drm_mm_node *mm_node;
2481 	u64 npa_addr, size;
2482 
2483 	/* Get the list head for the list containing all the handles
2484 	 * exported to this remote GPU.
2485 	 */
2486 	exp_handles_list = &adev->ualink.exp_handles_list[remote_acc_id];
2487 
2488 	dev_dbg(adev->dev,
2489 		"EXP-RESET: Cleaning up all XA entries for remote:%u\n",
2490 		remote_acc_id);
2491 
2492 	xa_lock(&adev->ualink.exp_xa);
2493 	while (!list_empty(exp_handles_list)) {
2494 		importer_entry = list_first_entry(exp_handles_list,
2495 					struct amdgpu_ualink_importer_entry, list);
2496 		list_del_init(&importer_entry->list);
2497 
2498 		exp_xa_node = importer_entry->parent;
2499 		if (!amdgpu_ualink_exp_xa_entry_get(exp_xa_node))
2500 			continue;
2501 
2502 		xa_unlock(&adev->ualink.exp_xa);
2503 		/* Clear the bit corresponding to this remote GPU in
2504 		 * the importer bitmap.
2505 		 */
2506 		if (!test_and_clear_bit(remote_acc_id,
2507 					exp_xa_node->importers_bitmap)) {
2508 			amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
2509 			xa_lock(&adev->ualink.exp_xa);
2510 			continue;
2511 		}
2512 
2513 		mutex_lock(&exp_xa_node->node_lock);
2514 		size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2515 		/* Unpin the BO */
2516 		if (likely(!amdgpu_bo_reserve(exp_xa_node->bo, true))) {
2517 			amdgpu_bo_unpin(exp_xa_node->bo);
2518 			amdgpu_bo_unreserve(exp_xa_node->bo);
2519 		} else {
2520 			dev_warn(adev->dev,
2521 				"EXP-RESET: BO reserve to unpin failed handle:%llx:%llx\n",
2522 				exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo);
2523 		}
2524 
2525 		dev_dbg(adev->dev,
2526 			"EXP-RESET: handle:%llx:%llx pin_count:%d, importers:%d\n",
2527 			exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
2528 			exp_xa_node->bo->tbo.pin_count,
2529 			bitmap_weight(exp_xa_node->importers_bitmap,
2530 			AMDGPU_UALINK_ACCEL_MAX));
2531 
2532 		WARN_ON(exp_xa_node->bo->tbo.pin_count <
2533 			bitmap_weight(exp_xa_node->importers_bitmap,
2534 			AMDGPU_UALINK_ACCEL_MAX));
2535 
2536 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
2537 			npa_addr = importer_entry->npa_addr;
2538 			mm_node = importer_entry->mm_node;
2539 			mutex_unlock(&exp_xa_node->node_lock);
2540 
2541 			dev_dbg(adev->dev,
2542 				"EXP-RESET: Unmap NPA:%llx size: %llx remote:%u handle:%llx:%llx\n",
2543 				npa_addr, size, remote_acc_id,
2544 				exp_xa_node->handle.handle_hi,
2545 				exp_xa_node->handle.handle_lo);
2546 			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
2547 						     npa_addr, size);
2548 			amdgpu_ualink_npa_free_va(adev, mm_node);
2549 			kfree(mm_node);
2550 
2551 			/* Reset the NPA addr and mm_node */
2552 			mutex_lock(&exp_xa_node->node_lock);
2553 			importer_entry->npa_addr = 0;
2554 			importer_entry->mm_node = NULL;
2555 		} else if (bitmap_empty(exp_xa_node->importers_bitmap,
2556 				AMDGPU_UALINK_ACCEL_MAX)) {
2557 			/* In Source-Aliasing mode, if there are no importers
2558 			 * for this handle, then we can unmap and free the
2559 			 * NPA address.
2560 			 */
2561 			mm_node = exp_xa_node->importer_entries[0].mm_node;
2562 			npa_addr = exp_xa_node->importer_entries[0].npa_addr;
2563 			mutex_unlock(&exp_xa_node->node_lock);
2564 
2565 			dev_dbg(adev->dev,
2566 				"EXP-RESET: Unmap NPA:%llx size: %llx handle:%llx:%llx\n",
2567 				npa_addr, size, exp_xa_node->handle.handle_hi,
2568 				exp_xa_node->handle.handle_lo);
2569 			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
2570 						     npa_addr, size);
2571 			amdgpu_ualink_npa_free_va(adev, mm_node);
2572 			kfree(mm_node);
2573 			mutex_lock(&exp_xa_node->node_lock);
2574 			exp_xa_node->importer_entries[0].npa_addr = 0;
2575 			exp_xa_node->importer_entries[0].mm_node = NULL;
2576 		}
2577 
2578 		mutex_unlock(&exp_xa_node->node_lock);
2579 		xa_lock(&adev->ualink.exp_xa);
2580 		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
2581 	}
2582 	xa_unlock(&adev->ualink.exp_xa);
2583 }
2584 
2585 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
2586 						u32 remote_acc_id, u32 state,
2587 						u32 generation_count)
2588 {
2589 	struct amdgpu_ualink_connection *conn_state;
2590 
2591 	conn_state = &adev->ualink.conn_state[remote_acc_id];
2592 
2593 	mutex_lock(&conn_state->lock);
2594 	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
2595 	     (conn_state->generation_count == generation_count)) {
2596 		conn_state->state = state;
2597 		mutex_unlock(&conn_state->lock);
2598 
2599 		amdgpu_ualink_cleanup_imp_xa_entries(adev, remote_acc_id);
2600 		amdgpu_ualink_cleanup_exp_xa_entries(adev, remote_acc_id);
2601 	} else {
2602 		mutex_unlock(&conn_state->lock);
2603 	}
2604 }
2605 
2606 /* Set PTE.X = 1 for all importer entries to retry RPCs. */
2607 static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev,
2608 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2609 {
2610 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2611 	struct amdgpu_ualink_importer_entry *imp_entry;
2612 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2613 	u64 pte_flags, npa_addr, size;
2614 	struct dma_fence *fence = NULL;
2615 	struct amdgpu_bo *bo;
2616 	struct drm_exec exec;
2617 	u32 remote_acc_id;
2618 	int r;
2619 
2620 	bo = exp_xa_node->bo;
2621 	size = amdgpu_bo_ngpu_pages(bo);
2622 
2623 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false);
2624 	if (unlikely(r)) {
2625 		dev_err(adev->dev, "Failed to reserve VM and BO in force_retry_rpcs\n");
2626 		return;
2627 	}
2628 
2629 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2630 			 AMDGPU_UALINK_ACCEL_MAX) {
2631 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2632 			imp_entry = &exp_xa_node->importer_entries[0];
2633 		else
2634 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2635 		npa_addr = imp_entry->npa_addr;
2636 
2637 		/* If the connection state changed while we are freeing
2638 		 * the BO, then ignore this importer. We will unmap this
2639 		 * address eventually in amdgpu_ualink_unmap_all_npa_addr()
2640 		 * function. Also, we will clear the corresponding bit in
2641 		 * the importer_bitmap in the same function.
2642 		 */
2643 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2644 					imp_entry->generation_count))
2645 			continue;
2646 
2647 		/* Set PTE.X = 1 */
2648 		pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo,
2649 					AMDGPU_VM_PAGE_EXECUTABLE);
2650 		dev_dbg(adev->dev,
2651 			"RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n",
2652 			npa_addr, remote_acc_id, pte_flags);
2653 
2654 		r = amdgpu_vm_update_range(adev, vm, false, false, true,
2655 					false, NULL, npa_addr, npa_addr + size - 1,
2656 					pte_flags, 0, adev->vm_manager.vram_base_offset,
2657 					bo->tbo.resource, NULL, &vm->last_update);
2658 
2659 		if (r)
2660 			dev_warn(adev->dev,
2661 				"RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n",
2662 				npa_addr, remote_acc_id, r);
2663 
2664 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2665 			break;
2666 	}
2667 
2668 	r = amdgpu_vm_update_pdes(adev, vm, false);
2669 	if (r) {
2670 		dev_err(adev->dev,
2671 			"Failed %d to update page directories during force retry rpcs\n",
2672 			r);
2673 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2674 		return;
2675 	}
2676 
2677 	fence = dma_fence_get(vm->last_update);
2678 	if (fence) {
2679 		r = dma_fence_wait(fence, false);
2680 		dma_fence_put(fence);
2681 		fence = NULL;
2682 		if (r)
2683 			dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r);
2684 	}
2685 
2686 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2687 
2688 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2689 }
2690 
2691 static void amdgpu_ualink_send_tlb_shootdown(struct amdgpu_device *adev,
2692 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2693 {
2694 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2695 	u32 accel_id = adev->ualink.info->ppod.accel_id;
2696 	struct amdgpu_ualink_importer_entry *imp_entry;
2697 	u64 npa_addr, size;
2698 	u32 remote_acc_id;
2699 	int r;
2700 
2701 	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2702 
2703 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2704 			 AMDGPU_UALINK_ACCEL_MAX) {
2705 		/*
2706 		 * Replace the GPU-id bits with local accel_id
2707 		 */
2708 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2709 			imp_entry = &exp_xa_node->importer_entries[0];
2710 		else
2711 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2712 
2713 		npa_addr = STRIP_NPA(imp_entry->npa_addr);
2714 		npa_addr = GENERATE_NPA(npa_addr, accel_id);
2715 
2716 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2717 					imp_entry->generation_count)) {
2718 			clear_bit(remote_acc_id,
2719 				  exp_xa_node->importers_bitmap);
2720 			continue;
2721 		}
2722 
2723 		dev_dbg(adev->dev,
2724 			"EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n",
2725 			remote_acc_id);
2726 		r = amdgpu_ualink_remote_shootdown(adev, remote_acc_id,
2727 					npa_addr, size,
2728 					AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN);
2729 		if (r) {
2730 			dev_err(adev->dev,
2731 				"EXP-CLEANUP: TLB shootdown send failed to remote:%u\n",
2732 				remote_acc_id);
2733 
2734 			/* Send failed: mark the connection NOT_READY so the
2735 			 * rest of this cleanup (and other cleanups) skip the
2736 			 * unreachable peer via amdgpu_ualink_check_conn_ready().
2737 			 */
2738 			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
2739 						AMDGPU_UALINK_CONN_NOT_READY,
2740 						imp_entry->generation_count);
2741 		}
2742 	}
2743 }
2744 
2745 /* Unmap all NPA addresses associated with a BO (UALink handle). This function is used
2746  * only in Source Identification mode.
2747  */
2748 static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev,
2749 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
2750 {
2751 	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
2752 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2753 	struct amdgpu_ualink_importer_entry *imp_entry;
2754 	u64 pte_value = adev->gmc.noretry_flags;
2755 	struct dma_fence *fence = NULL;
2756 	struct drm_exec exec;
2757 	u64 npa_addr, size;
2758 	u32 remote_acc_id;
2759 	int r;
2760 
2761 	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
2762 
2763 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1,
2764 						 &exec, false);
2765 	if (unlikely(r)) {
2766 		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_all_npa_addr\n");
2767 		return;
2768 	}
2769 
2770 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2771 			 AMDGPU_UALINK_ACCEL_MAX) {
2772 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2773 			imp_entry = &exp_xa_node->importer_entries[0];
2774 		else
2775 			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2776 		npa_addr = imp_entry->npa_addr;
2777 
2778 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2779 					imp_entry->generation_count)) {
2780 			clear_bit(remote_acc_id,
2781 				  exp_xa_node->importers_bitmap);
2782 			continue;
2783 		}
2784 
2785 		dev_dbg(adev->dev,
2786 			"UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n",
2787 			npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
2788 			remote_acc_id, pte_value);
2789 
2790 		r = amdgpu_vm_update_range(adev, vm, false,
2791 					   false, true, false, NULL, npa_addr,
2792 					   npa_addr + size - 1, pte_value, 0,
2793 					   0, NULL, NULL, &vm->last_update);
2794 
2795 		if (r)
2796 			dev_err(adev->dev,
2797 				"UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n",
2798 				npa_addr, exp_xa_node->handle.handle_hi,
2799 				exp_xa_node->handle.handle_lo, remote_acc_id);
2800 
2801 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2802 			break;
2803 	}
2804 
2805 	r = amdgpu_vm_update_pdes(adev, vm, false);
2806 	if (r) {
2807 		dev_err(adev->dev,
2808 			"Failed %d to update page directories during all NPA addresses unmapping\n",
2809 			r);
2810 		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2811 		return;
2812 	}
2813 
2814 	fence = dma_fence_get(vm->last_update);
2815 	if (fence) {
2816 		r = dma_fence_wait(fence, false);
2817 		dma_fence_put(fence);
2818 		fence = NULL;
2819 		if (r)
2820 			dev_err(adev->dev,
2821 				"UNMAP-NPA: dma fence wait failed\n");
2822 	}
2823 
2824 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
2825 
2826 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
2827 }
2828 
2829 static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev,
2830 				struct amdgpu_ualink_exp_xa_node *exp_xa_node,
2831 				unsigned long *importers_bitmap)
2832 {
2833 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
2834 	struct drm_mm_node *mm_node;
2835 	u32 remote_acc_id;
2836 
2837 	for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) {
2838 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2839 			mm_node = exp_xa_node->importer_entries[0].mm_node;
2840 		else
2841 			mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node;
2842 
2843 		if (!mm_node) {
2844 			if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2845 				break;
2846 			continue;
2847 		}
2848 
2849 		dev_dbg(adev->dev,
2850 			"FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n",
2851 			mm_node->start, exp_xa_node->handle.handle_hi,
2852 			exp_xa_node->handle.handle_lo, remote_acc_id);
2853 
2854 		amdgpu_ualink_npa_free_va(adev, mm_node);
2855 		kfree(mm_node);
2856 
2857 		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
2858 			break;
2859 	}
2860 }
2861 
2862 static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work)
2863 {
2864 	DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
2865 	struct amdgpu_ualink_importer_entry *imp_entry;
2866 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
2867 	struct amdgpu_ualink_handle handle;
2868 	struct amdgpu_device *adev;
2869 	struct amdgpu_bo *bo;
2870 	u32 remote_acc_id;
2871 	int r;
2872 
2873 	exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node,
2874 				   cleanup_work);
2875 	bo = exp_xa_node->bo;
2876 	adev = amdgpu_ttm_adev(bo->tbo.bdev);
2877 	handle = exp_xa_node->handle;
2878 
2879 	/* Revoking access to an exported memory follows the steps:
2880 	 * 1. Set PTE.X = 1 to retry for RPCs.
2881 	 * 2. Send Remote TLB Shootdowns to all importers.
2882 	 * 3. Unmap the NPA address from NPA VM.
2883 	 * 4. Drop the ref count for the BO.
2884 	 * 5. Send NPA_REVOKE to all importers.
2885 	 * 6. Wait for NPA_RELEASE from all importers.
2886 	 * 7. Free the NPA address once all responses are received.
2887 	 */
2888 	/* If there are no importers for this BO/handle */
2889 	if (bitmap_empty(exp_xa_node->importers_bitmap,
2890 			 AMDGPU_UALINK_ACCEL_MAX)) {
2891 		/* Release the dma_buf created at export time (used for the
2892 		 * local-import shortcut). The has-importers path below drops
2893 		 * it too; this branch must not skip it or the dma_buf and the
2894 		 * BO it pins are leaked.
2895 		 */
2896 		dma_buf_put(exp_xa_node->dmabuf);
2897 		/* Drop the BO reference so it can be freed. */
2898 		amdgpu_bo_unref(&bo);
2899 		exp_xa_node->bo = NULL;
2900 		goto free_node;
2901 	}
2902 
2903 	dev_dbg(adev->dev,
2904 		"EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n",
2905 		handle.handle_hi, handle.handle_lo,
2906 		AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap);
2907 
2908 	bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap,
2909 		    AMDGPU_UALINK_ACCEL_MAX);
2910 
2911 	/* Set PTE.X = 1 for NPA addresses from all importers*/
2912 	amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node);
2913 
2914 	/* Send TLB-shootdown to all importer GPUs */
2915 	amdgpu_ualink_send_tlb_shootdown(adev, exp_xa_node);
2916 
2917 	/* Unmap all NPA addresses for this BO from NPA VM */
2918 	amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
2919 
2920 	dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n",
2921 		handle.handle_hi, handle.handle_lo, bo->tbo.pin_count,
2922 		bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX));
2923 	WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap,
2924 						AMDGPU_UALINK_ACCEL_MAX));
2925 
2926 	/* Unpin the BO */
2927 	if (likely(!amdgpu_bo_reserve(bo, true))) {
2928 		bo->ualink_handle_lo = 0ULL;
2929 		for_each_set_bit(remote_acc_id, orig_importers_bitmap,
2930 				 AMDGPU_UALINK_ACCEL_MAX)
2931 			amdgpu_bo_unpin(bo);
2932 		amdgpu_bo_unreserve(bo);
2933 	} else {
2934 		dev_warn(adev->dev,
2935 			"EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n",
2936 			handle.handle_hi, handle.handle_lo);
2937 	}
2938 
2939 	/* Free the DMABuf */
2940 	dma_buf_put(exp_xa_node->dmabuf);
2941 	/* Drop the reference to the BO so it can be freed. */
2942 	amdgpu_bo_unref(&bo);
2943 	exp_xa_node->bo = NULL;
2944 
2945 	/* Seed npa_release_bitmap with all importers before any NPA-REVOKE, so
2946 	 * the release IRQ only sees it empty once every responder has replied.
2947 	 * Dead peers are dropped from it in the send loop below.
2948 	 */
2949 	bitmap_copy(exp_xa_node->npa_release_bitmap,
2950 		    exp_xa_node->importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
2951 
2952 	/* Send NPA-REVOKE to all importers which have imported this memory.
2953 	 * On send failure clear the bit (no response will arrive) and mark the
2954 	 * connection NOT_READY so other cleanups skip this unreachable peer.
2955 	 */
2956 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
2957 				 AMDGPU_UALINK_ACCEL_MAX) {
2958 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
2959 
2960 		/* Re-check the connection right before the send: concurrent
2961 		 * cleanup workers all seed the bitmap while the peer is still
2962 		 * ESTABLISHED, so once its egress dies and the first worker
2963 		 * marks it NOT_READY, this skips the rest instead of flooding
2964 		 * the log with failed LSDMA sends to the same dead peer.
2965 		 */
2966 		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
2967 					imp_entry->generation_count)) {
2968 			clear_bit(remote_acc_id,
2969 				  exp_xa_node->npa_release_bitmap);
2970 			continue;
2971 		}
2972 
2973 		dev_dbg(adev->dev,
2974 			"EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n",
2975 			remote_acc_id);
2976 		r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle);
2977 		if (r) {
2978 			dev_err(adev->dev,
2979 				"EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n",
2980 				remote_acc_id);
2981 			clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
2982 
2983 			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
2984 						AMDGPU_UALINK_CONN_NOT_READY,
2985 						imp_entry->generation_count);
2986 		}
2987 	}
2988 
2989 	/* Wait for the NPA_RELEASE to come back from all importers */
2990 	if (!bitmap_empty(exp_xa_node->npa_release_bitmap,
2991 			AMDGPU_UALINK_ACCEL_MAX)) {
2992 		dev_dbg(adev->dev,
2993 			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n",
2994 			handle.handle_hi, handle.handle_lo,
2995 			AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap);
2996 
2997 		/* Wait for the NPA_RELEASE to come back from all importers */
2998 		r = wait_for_completion_timeout(&exp_xa_node->npa_done,
2999 					msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
3000 
3001 		if (r == 0)
3002 			dev_warn(adev->dev,
3003 				"EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n",
3004 				handle.handle_hi, handle.handle_lo);
3005 	}
3006 
3007 	/* Free the NPA addresses given to all the importers */
3008 	amdgpu_ualink_free_all_npa_va(adev, exp_xa_node,
3009 				      orig_importers_bitmap);
3010 
3011 	/* Warn about all importers that didn't respond back with
3012 	 * NPA-RELEASE message and trigger connection timeout handling.
3013 	 */
3014 	for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap,
3015 			 AMDGPU_UALINK_ACCEL_MAX) {
3016 		dev_warn(adev->dev,
3017 			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n",
3018 			handle.handle_hi, handle.handle_lo, remote_acc_id);
3019 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
3020 		amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
3021 						AMDGPU_UALINK_CONN_NOT_READY,
3022 						imp_entry->generation_count);
3023 	}
3024 
3025 free_node:
3026 	xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo);
3027 	mutex_destroy(&exp_xa_node->node_lock);
3028 	kfree(exp_xa_node);
3029 }
3030 
3031 /* This function is a copy of amdgpu_dma_buf_move_notify() function.
3032  * amdgpu_dma_buf_move_notify is only called for import attachments.
3033  * But NPA DMABufs don't use attachments because they are imported
3034  * on the same device. So we need to invalidate the GPUVM mappings
3035  * manually.
3036  */
3037 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo)
3038 {
3039 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
3040 	struct ttm_operation_ctx ctx = { false, false };
3041 	struct ttm_placement placement = {};
3042 	struct amdgpu_vm_bo_base *bo_base;
3043 	struct drm_exec exec;
3044 	int r;
3045 
3046 	/*
3047 	 * Lock the BO together with every client VM page directory it is
3048 	 * mapped into in a single drm_exec transaction.
3049 	 */
3050 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
3051 	drm_exec_until_all_locked(&exec) {
3052 		r = drm_exec_lock_obj(&exec, &bo->tbo.base);
3053 		drm_exec_retry_on_contention(&exec);
3054 		if (unlikely(r))
3055 			goto fini;
3056 
3057 		for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
3058 			r = amdgpu_vm_lock_pd(bo_base->vm, &exec, 0);
3059 			drm_exec_retry_on_contention(&exec);
3060 			if (unlikely(r))
3061 				goto fini;
3062 		}
3063 	}
3064 
3065 	amdgpu_vm_bo_move(bo, NULL, false);
3066 
3067 	r = ttm_bo_validate(&bo->tbo, &placement, &ctx);
3068 	if (r) {
3069 		dev_err(adev->dev, "Failed to invalidate NPA DMA-buf import (%d)\n",
3070 			r);
3071 		goto fini;
3072 	}
3073 
3074 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
3075 		struct amdgpu_vm *vm = bo_base->vm;
3076 
3077 		/*
3078 		 * Fences for the two SDMA page table updates were already
3079 		 * reserved by amdgpu_vm_lock_pd() above (it reserves 2 +
3080 		 * num_fences on the same VM root PD dma_resv).
3081 		 */
3082 		r = amdgpu_vm_clear_freed(adev, vm, NULL);
3083 		if (!r)
3084 			r = amdgpu_vm_handle_moved(adev, vm, &exec.ticket);
3085 
3086 		if (r && r != -EBUSY)
3087 			dev_err(adev->dev, "Failed to invalidate VM page tables (%d))\n",
3088 				r);
3089 	}
3090 
3091 fini:
3092 	drm_exec_fini(&exec);
3093 }
3094 
3095 static int amdgpu_ualink_map_npa_to_dmabuf(struct amdgpu_device *adev,
3096 				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
3097 {
3098 	struct ttm_operation_ctx ctx = { false, false };
3099 	u32 initial_domain = AMDGPU_GEM_DOMAIN_NPA;
3100 	struct drm_gem_object *gobj = NULL;
3101 	u64 alloc_flags, npa_addr, size;
3102 	struct dma_buf *dmabuf;
3103 	struct amdgpu_bo *bo;
3104 	u32 handle;
3105 	int r;
3106 
3107 	npa_addr = imp_xa_node->npa_addr;
3108 	size = imp_xa_node->size;
3109 
3110 	dev_dbg(adev->dev, "Create NPA BO addr 0x%llx size in pages 0x%llx\n",
3111 		npa_addr, size);
3112 
3113 	/*
3114 	 * Create the BO directly in the NPA domain.
3115 	 */
3116 	alloc_flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
3117 	r = amdgpu_gem_object_create(adev, size * AMDGPU_GPU_PAGE_SIZE, 1,
3118 				     initial_domain, alloc_flags,
3119 				     ttm_bo_type_device, NULL, &gobj, 0);
3120 	if (r) {
3121 		dev_err(adev->dev, "Failed to create NPA BO. ret %d\n", r);
3122 		return r;
3123 	}
3124 
3125 	bo = gem_to_amdgpu_bo(gobj);
3126 
3127 	r = amdgpu_bo_reserve(bo, false);
3128 	if (unlikely(r != 0)) {
3129 		dev_err(adev->dev, "Failed to reserve NPA BO, r: %d\n", r);
3130 		goto err_reserve_failed;
3131 	}
3132 
3133 	/*
3134 	 * Drop the arbitrarily-placed NPA node and re-create it at the exact
3135 	 * remote window, mirroring amdgpu_bo_create_kernel_at().
3136 	 */
3137 	ttm_resource_free(&bo->tbo, &bo->tbo.resource);
3138 
3139 	bo->placements[0].fpfn = npa_addr;
3140 	bo->placements[0].lpfn = npa_addr + size;
3141 
3142 	r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.resource, &ctx);
3143 	amdgpu_bo_unreserve(bo);
3144 	if (r) {
3145 		dev_err(adev->dev,
3146 			"Failed to place NPA BO at 0x%llx, r: %d\n", npa_addr, r);
3147 		goto err_validate_failed;
3148 	}
3149 
3150 	r = drm_gem_handle_create(adev->ualink.client.file, gobj, &handle);
3151 	if (r) {
3152 		dev_err(adev->dev,
3153 			"Failed to get handle for NPA GEM object, r: %d\n", r);
3154 		goto err_validate_failed;
3155 	}
3156 	drm_gem_object_put(gobj);
3157 
3158 	dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, adev->ualink.client.file,
3159 						handle, DRM_CLOEXEC | DRM_RDWR);
3160 	if (IS_ERR(dmabuf)) {
3161 		r = PTR_ERR(dmabuf);
3162 		dev_err(adev->dev,
3163 			"Failed to generate DMABuf for NPA GEM object\n");
3164 		goto err_dmabuf_failed;
3165 	}
3166 
3167 	imp_xa_node->dmabuf = dmabuf;
3168 	imp_xa_node->gem_handle = handle;
3169 
3170 	return 0;
3171 
3172 err_dmabuf_failed:
3173 	drm_gem_handle_delete(adev->ualink.client.file, handle);
3174 	return r;
3175 err_validate_failed:
3176 err_reserve_failed:
3177 	drm_gem_object_put(gobj);
3178 
3179 	return r;
3180 }
3181 
3182 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo)
3183 {
3184 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
3185 	struct amdgpu_ualink_importer_entry *imp_entry;
3186 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3187 	u32 remote_acc_id;
3188 	int r;
3189 
3190 	if (!bo->ualink_handle_lo)
3191 		return;
3192 
3193 	/* Remove the entry from the Xarray. */
3194 	xa_lock(&adev->ualink.exp_xa);
3195 	exp_xa_node = __xa_erase(&adev->ualink.exp_xa,
3196 				 bo->ualink_handle_lo);
3197 	if (!exp_xa_node) {
3198 		xa_unlock(&adev->ualink.exp_xa);
3199 		dev_warn(adev->dev,
3200 			 "Exp XA: handle_lo:%llx not found\n",
3201 			 bo->ualink_handle_lo);
3202 		return;
3203 	}
3204 
3205 	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
3206 			 AMDGPU_UALINK_ACCEL_MAX) {
3207 		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
3208 		list_del_init(&imp_entry->list);
3209 	}
3210 	xa_unlock(&adev->ualink.exp_xa);
3211 
3212 	/* Add it to the Handle_Invalid xarray */
3213 	r = xa_err(xa_store(&adev->ualink.handle_invalid_xa,
3214 			    bo->ualink_handle_lo,
3215 			    exp_xa_node, GFP_KERNEL));
3216 	if (r)
3217 		dev_err(adev->dev,
3218 			"Handle_Invalid XA store failed handle:%llx:%llx error:%d\n",
3219 			exp_xa_node->handle.handle_hi,
3220 			exp_xa_node->handle.handle_lo, r);
3221 
3222 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3223 }
3224 
3225 static void amdgpu_ualink_process_npa_release_msg(struct amdgpu_device *adev,
3226 						 u32 remote_acc_id,
3227 						 struct amdgpu_ualink_handle handle)
3228 {
3229 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3230 
3231 	xa_lock(&adev->ualink.handle_invalid_xa);
3232 	exp_xa_node = xa_load(&adev->ualink.handle_invalid_xa, handle.handle_lo);
3233 	if (!exp_xa_node) {
3234 		dev_warn(adev->dev,
3235 			 "NPA-RELEASE: Handle (%llx:%llx) not found\n",
3236 			 handle.handle_hi, handle.handle_lo);
3237 		goto out;
3238 	}
3239 
3240 	/* Confirm that the complete handle matches. */
3241 	if (handle.handle_hi != exp_xa_node->handle.handle_hi) {
3242 		dev_warn(adev->dev,
3243 			 "NPA-RELEASE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
3244 			 exp_xa_node->handle.handle_hi,
3245 			 handle.handle_hi, handle.handle_lo, remote_acc_id);
3246 		goto out;
3247 	}
3248 
3249 	/* Test and clear the bit corresponding to the remote GPU id to signal
3250 	 * the arrival of NPA_RELEASE message from it.
3251 	 * If the corresponding bit wasn't set, then raise a warning and ignore
3252 	 * the NPA-Release message from the remote GPU.
3253 	 */
3254 	if (!test_and_clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap)) {
3255 		dev_warn(adev->dev,
3256 			 "NPA-RELEASE: unexpected from remote:%u handle:%llx:%llx\n",
3257 			 remote_acc_id, handle.handle_hi, handle.handle_lo);
3258 		goto out;
3259 	}
3260 
3261 	/* Signal completion if NPA_Release received from all importers */
3262 	if (bitmap_empty(exp_xa_node->npa_release_bitmap,
3263 			 AMDGPU_UALINK_ACCEL_MAX))
3264 		complete(&exp_xa_node->npa_done);
3265 
3266 out:
3267 	xa_unlock(&adev->ualink.handle_invalid_xa);
3268 }
3269 
3270 static void amdgpu_ualink_process_npa_revoke_msg(struct amdgpu_device *adev,
3271 						u32 remote_acc_id,
3272 						struct amdgpu_ualink_handle handle)
3273 {
3274 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3275 	struct amdgpu_bo *bo;
3276 	int r = 0;
3277 
3278 	/* Remove the entry from the Xarray. */
3279 	xa_lock(&adev->ualink.imp_xa);
3280 	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
3281 	if (!imp_xa_node) {
3282 		xa_unlock(&adev->ualink.imp_xa);
3283 		dev_warn(adev->dev,
3284 			 "NPA-REVOKE: Handle (%llx:%llx) not found\n",
3285 			 handle.handle_hi, handle.handle_lo);
3286 		return;
3287 	}
3288 
3289 	/* Confirm that the complete handle matches. */
3290 	if (handle.handle_hi != imp_xa_node->handle.handle_hi) {
3291 		xa_unlock(&adev->ualink.imp_xa);
3292 		dev_warn(adev->dev,
3293 			 "NPA-REVOKE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
3294 			 imp_xa_node->handle.handle_hi,
3295 			 handle.handle_hi, handle.handle_lo, remote_acc_id);
3296 		return;
3297 	}
3298 
3299 	switch (READ_ONCE(imp_xa_node->node_state)) {
3300 	case AMDGPU_UALINK_NODE_READY:
3301 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
3302 		list_del_init(&imp_xa_node->list);
3303 		xa_unlock(&adev->ualink.imp_xa);
3304 
3305 		/* Invalidate the GPUVM mappings */
3306 		bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
3307 		amdgpu_ualink_invalidate_import_mappings(bo);
3308 
3309 		/* Drop the refcount for the node */
3310 		amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
3311 		break;
3312 	case AMDGPU_UALINK_NODE_PENDING:
3313 		/* The import is still building the dma-buf and nothing has
3314 		 * been handed to user-space yet. The importing thread sees
3315 		 * the teardown state and unwinds.
3316 		 */
3317 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
3318 		xa_unlock(&adev->ualink.imp_xa);
3319 		break;
3320 	default:
3321 		/* NPA-REVOKE always follows NPA-RSP, so a NOT_READY node means
3322 		 * a stale revoke, and a node in teardown is already being
3323 		 * released by whoever moved it there.
3324 		 */
3325 		xa_unlock(&adev->ualink.imp_xa);
3326 		return;
3327 	}
3328 
3329 	r = amdgpu_ualink_send_npa_release_msg(adev, remote_acc_id, handle);
3330 	if (r)
3331 		dev_err(adev->dev,
3332 			"NPA-Release send failed remote:%u handle:%llx:%llx error:%d\n",
3333 			remote_acc_id, handle.handle_hi, handle.handle_lo, r);
3334 }
3335 
3336 static void amdgpu_ualink_process_npa_fail_msg(struct amdgpu_device *adev,
3337 				       u32 remote_acc_id, u64 partial_handle,
3338 				       u32 fail_reason)
3339 {
3340 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3341 	int r = 0;
3342 
3343 	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
3344 		dev_warn(adev->dev,
3345 			"NPA-FAIL: no connection with remote AccId:%u\n",
3346 			remote_acc_id);
3347 		goto conn_setup;
3348 	}
3349 
3350 	xa_lock(&adev->ualink.imp_xa);
3351 	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
3352 	if (!imp_xa_node) {
3353 		xa_unlock(&adev->ualink.imp_xa);
3354 		dev_warn(adev->dev,
3355 			"NPA-FAIL: imp XA handle not found:%llx\n",
3356 			partial_handle);
3357 		return;
3358 	}
3359 
3360 	imp_xa_node->fail_reason = fail_reason;
3361 	/* Signal completion done to signal response received for NPA-REQ
3362 	 * message.
3363 	 * If the node is in NOT_READY state, then set the node state to
3364 	 * PENDING and signal the completion. If the node is not in NOT_READY
3365 	 * state, then it is an unsolicited NPA-FAIL message and we
3366 	 * log a debug message.
3367 	 */
3368 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
3369 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
3370 		complete(&imp_xa_node->npa_done);
3371 	} else {
3372 		dev_dbg(adev->dev,
3373 			"NPA-FAIL: unsolicited for handle:%llx:%llx from AccId:%u\n",
3374 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
3375 			remote_acc_id);
3376 	}
3377 	xa_unlock(&adev->ualink.imp_xa);
3378 
3379 	return;
3380 
3381 conn_setup:
3382 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3383 	if (r)
3384 		dev_warn(adev->dev,
3385 			"NPA-FAIL: connection setup failed with remote AccId:%u\n",
3386 			remote_acc_id);
3387 }
3388 
3389 static void amdgpu_ualink_process_npa_rsp_msg(struct amdgpu_device *adev,
3390 				      u32 remote_acc_id, u64 partial_handle,
3391 				      u64 npa_addr, u64 size)
3392 {
3393 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3394 	int r = 0;
3395 
3396 	/* Check if the connection is established. If it is not, then start
3397 	 * connection setup.
3398 	 */
3399 	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
3400 		dev_warn(adev->dev,
3401 			"NPA-RSP: no connection with remote AccId:%u\n",
3402 			remote_acc_id);
3403 		goto conn_setup;
3404 	}
3405 
3406 	xa_lock(&adev->ualink.imp_xa);
3407 	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
3408 	if (!imp_xa_node) {
3409 		xa_unlock(&adev->ualink.imp_xa);
3410 		dev_warn(adev->dev,
3411 			"NPA-RSP: imp XA handle not found:%llx\n", partial_handle);
3412 		return;
3413 	}
3414 
3415 	/* NPA addr received in NPA-RSP is page aligned and without the remote
3416 	 * GPU-id in Bits 41-50. Assemble back the NPA address before storing
3417 	 * it.
3418 	 */
3419 	imp_xa_node->npa_addr = GENERATE_NPA(npa_addr, remote_acc_id);
3420 	/* Size is in number of GPU pages granularity. */
3421 	imp_xa_node->size = size;
3422 
3423 	/* Signal completion done to signal NPA_RSP received.
3424 	 * If the node is in NOT_READY state, then set the node state to
3425 	 * PENDING and signal the completion. If the node is not in NOT_READY
3426 	 * state, then it is an unsolicited NPA-RSP message and we
3427 	 * log a debug message.
3428 	 */
3429 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
3430 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
3431 		complete(&imp_xa_node->npa_done);
3432 	} else {
3433 		dev_dbg(adev->dev,
3434 			"NPA-RSP: unsolicited for handle:%llx:%llx from AccId:%u\n",
3435 			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
3436 			remote_acc_id);
3437 	}
3438 	xa_unlock(&adev->ualink.imp_xa);
3439 
3440 	return;
3441 
3442 conn_setup:
3443 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3444 	if (r)
3445 		dev_warn(adev->dev,
3446 			"NPA-RSP: connection setup failed with remote AccId:%u\n",
3447 			remote_acc_id);
3448 }
3449 
3450 static void amdgpu_ualink_process_npa_req_msg(struct amdgpu_device *adev,
3451 				      u32 remote_acc_id,
3452 				      struct amdgpu_ualink_handle handle)
3453 {
3454 	struct amdgpu_ualink_importer_entry *importer_entry, *npa_addr_entry;
3455 	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
3456 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3457 	u64 range_start, range_end, pte_flags;
3458 	struct drm_mm_node *mm_node = NULL;
3459 	int r = 0, fail_reason = 0;
3460 	bool send_npa_fail = true;
3461 	u64 npa_addr = 0, size;
3462 	struct amdgpu_bo *bo;
3463 	u32 gen_count;
3464 
3465 	/* Check if the connection is established. If it is not, then start
3466 	 * connection setup.
3467 	 */
3468 	gen_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
3469 	if (!gen_count) {
3470 		dev_warn(adev->dev,
3471 			"NPA-REQ: no connection with remote AccId:%u\n",
3472 			remote_acc_id);
3473 		goto conn_setup;
3474 	}
3475 
3476 	/* Check entry exists in Exporter XA. If yes, increase the refcount
3477 	 * for the node.
3478 	 */
3479 	xa_lock(&adev->ualink.exp_xa);
3480 	exp_xa_node = xa_load(&adev->ualink.exp_xa, handle.handle_lo);
3481 	if (!exp_xa_node  || (handle.handle_hi != exp_xa_node->handle.handle_hi) ||
3482 	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
3483 		xa_unlock(&adev->ualink.exp_xa);
3484 		dev_warn(adev->dev,
3485 			"NPA-REQ: exp XA handle not found handle:%llx:%llx\n",
3486 			handle.handle_hi, handle.handle_lo);
3487 		fail_reason = AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE;
3488 		goto handle_invalid_fail;
3489 	}
3490 	xa_unlock(&adev->ualink.exp_xa);
3491 
3492 	bo = exp_xa_node->bo;
3493 	size = amdgpu_bo_ngpu_pages(bo);
3494 
3495 	/* Pin the BO */
3496 	r = amdgpu_bo_reserve(bo, true);
3497 	if (unlikely(r)) {
3498 		dev_warn(adev->dev,
3499 			"NPA-REQ: BO reserve failed handle:%llx:%llx\n",
3500 			handle.handle_hi, handle.handle_lo);
3501 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3502 		goto bo_reserve_fail;
3503 	}
3504 	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
3505 	amdgpu_bo_unreserve(bo);
3506 	if (r) {
3507 		dev_warn(adev->dev,
3508 			"NPA-REQ: BO pin failed handle:%llx:%llx\n",
3509 			handle.handle_hi, handle.handle_lo);
3510 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3511 		goto bo_pin_fail;
3512 	}
3513 
3514 	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
3515 		mutex_lock(&exp_xa_node->node_lock);
3516 		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
3517 		npa_addr_entry = importer_entry;
3518 		mutex_unlock(&exp_xa_node->node_lock);
3519 		/* Check if NPA address is already allocated for this importer.
3520 		 * If yes, then send the NPA-FAIL message back to the remote GPU.
3521 		 */
3522 		if (importer_entry->npa_addr) {
3523 			fail_reason = AMDGPU_UALINK_NPA_FAIL_DUPLICATE;
3524 			goto npa_duplicate_fail;
3525 		}
3526 
3527 		range_start = ((u64)remote_acc_id << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT) |
3528 				AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED;
3529 		range_end = range_start | AMDGPU_UALINK_NPA_ADDR_RANGE_MASK;
3530 	} else {
3531 		/* We store NPA-address in importer_entries[0] in
3532 		 * Source-Aliasing mode.
3533 		 */
3534 		mutex_lock(&exp_xa_node->node_lock);
3535 		npa_addr_entry = &exp_xa_node->importer_entries[0];
3536 		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
3537 		/* Check if NPA address is already allocated for this importer.
3538 		 * If yes, then set the corresponding bit in the importers_bitmap,
3539 		 * set the generation count and send the NPA-RSP back to the remote GPU.
3540 		 */
3541 		if (npa_addr_entry->npa_addr) {
3542 			npa_addr = npa_addr_entry->npa_addr;
3543 			set_bit(remote_acc_id, exp_xa_node->importers_bitmap);
3544 			importer_entry->generation_count = gen_count;
3545 			mutex_unlock(&exp_xa_node->node_lock);
3546 			dev_dbg(adev->dev,
3547 				"NPA-REQ: NPA:%llx size:%llx handle:%llx:%llx\n",
3548 				npa_addr, size, handle.handle_hi, handle.handle_lo);
3549 
3550 			goto send_npa_rsp;
3551 		}
3552 		mutex_unlock(&exp_xa_node->node_lock);
3553 
3554 		range_start = 0;
3555 		range_end = 0;
3556 	}
3557 
3558 	mm_node = kzalloc_obj(*mm_node);
3559 	if (!mm_node) {
3560 		dev_warn(adev->dev,
3561 			"NPA-REQ: mm_node alloc failed handle:%llx:%llx\n",
3562 			handle.handle_hi, handle.handle_lo);
3563 		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
3564 		goto mem_alloc_fail;
3565 	}
3566 
3567 	/* Allocate NPA address */
3568 	r = amdgpu_ualink_npa_alloc_va(adev, mm_node, 0, range_start,
3569 					range_end, size);
3570 	if (r) {
3571 		dev_warn(adev->dev,
3572 			"NPA-REQ: NPA addr alloc failed handle:%llx:%llx\n",
3573 			handle.handle_hi, handle.handle_lo);
3574 		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
3575 		goto npa_alloc_fail;
3576 	}
3577 	npa_addr = mm_node->start;
3578 
3579 	pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 0);
3580 	dev_dbg(adev->dev,
3581 		"NPA-REQ: Allocated NPA:%llx size:%llx PTE:%llx handle:%llx:%llx\n",
3582 		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
3583 
3584 	/* Map the NPA address into NPA VM*/
3585 	r = amdgpu_ualink_map_npa_addr(adev, npa_addr, size, bo, 0, pte_flags);
3586 	if (r) {
3587 		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
3588 		dev_warn(adev->dev,
3589 			"NPA-REQ: NPA addr (%llx) map failed handle:%llx:%llx\n",
3590 			npa_addr, handle.handle_hi, handle.handle_lo);
3591 		goto map_npa_fail;
3592 	}
3593 
3594 	dev_dbg(adev->dev,
3595 		"NPA-REQ: Mapped NPA:%llx size:%llx pte:%llx handle:%llx:%llx\n",
3596 		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
3597 send_npa_rsp:
3598 	/* Send NPA-RSP back to the remote GPU */
3599 	r = amdgpu_ualink_send_npa_rsp_msg(adev, remote_acc_id, handle,
3600 					   STRIP_NPA(npa_addr), size);
3601 	if (r) {
3602 		dev_warn(adev->dev,
3603 			"NPA-REQ: send NPA-RSP failed remote:%u handle:%llx:%llx\n",
3604 			remote_acc_id, handle.handle_hi, handle.handle_lo);
3605 		send_npa_fail = false;
3606 		goto send_npa_rsp_fail;
3607 	}
3608 
3609 	dev_dbg(adev->dev,
3610 		"NPA-REQ: Sent NPA-RSP with NPA:%llx size:%llx handle:%llx:%llx\n",
3611 		npa_addr, size, handle.handle_hi, handle.handle_lo);
3612 
3613 	/* If this is the first time we are setting the bit for this importer,
3614 	 * then store the NPA address, mm_node and generation count.
3615 	 */
3616 	mutex_lock(&exp_xa_node->node_lock);
3617 	if (!test_and_set_bit(remote_acc_id, exp_xa_node->importers_bitmap)) {
3618 		npa_addr_entry->npa_addr = npa_addr;
3619 		npa_addr_entry->mm_node = mm_node;
3620 		importer_entry->generation_count = gen_count;
3621 	}
3622 	mutex_unlock(&exp_xa_node->node_lock);
3623 
3624 	dev_dbg(adev->dev,
3625 		"NPA-REQ: BO pin_count:%d, importers:%d, handle:%llx:%llx\n",
3626 		bo->tbo.pin_count, bitmap_weight(exp_xa_node->importers_bitmap,
3627 		AMDGPU_UALINK_ACCEL_MAX), handle.handle_hi, handle.handle_lo);
3628 	WARN_ON(bo->tbo.pin_count < bitmap_weight(exp_xa_node->importers_bitmap,
3629 						   AMDGPU_UALINK_ACCEL_MAX));
3630 
3631 	/* Add this node to the exported handles list for the remote GPU,
3632 	 * but only if the node is still in exp_xa. If revoke already erased
3633 	 * it, skip the list_add to avoid a dangling list entry. The cleanup
3634 	 * worker is guaranteed to run after we drop our ref, so it will see
3635 	 * this importer in the bitmap and send NPA-REVOKE.
3636 	 */
3637 	xa_lock(&adev->ualink.exp_xa);
3638 	if (xa_load(&adev->ualink.exp_xa, exp_xa_node->handle.handle_lo) == exp_xa_node)
3639 		list_add(&importer_entry->list, &adev->ualink.exp_handles_list[remote_acc_id]);
3640 	xa_unlock(&adev->ualink.exp_xa);
3641 
3642 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3643 
3644 	return;
3645 
3646 send_npa_rsp_fail:
3647 	mutex_lock(&exp_xa_node->node_lock);
3648 	clear_bit(remote_acc_id, exp_xa_node->importers_bitmap);
3649 	mutex_unlock(&exp_xa_node->node_lock);
3650 	if (mm_node)
3651 		amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
3652 
3653 map_npa_fail:
3654 	if (mm_node)
3655 		amdgpu_ualink_npa_free_va(adev, mm_node);
3656 
3657 npa_alloc_fail:
3658 	kfree(mm_node);
3659 mem_alloc_fail:
3660 npa_duplicate_fail:
3661 	r = amdgpu_bo_reserve(bo, true);
3662 	if (likely(!r)) {
3663 		amdgpu_bo_unpin(bo);
3664 		amdgpu_bo_unreserve(bo);
3665 	} else {
3666 		dev_warn(adev->dev,
3667 			"NPA-REQ: BO reserve to unpin failed for handle:%llx:%llx\n",
3668 			handle.handle_hi, handle.handle_lo);
3669 	}
3670 
3671 bo_pin_fail:
3672 bo_reserve_fail:
3673 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3674 
3675 handle_invalid_fail:
3676 	if (send_npa_fail) {
3677 		r = amdgpu_ualink_send_npa_fail_msg(adev, remote_acc_id,
3678 						    handle, fail_reason);
3679 		if (r)
3680 			dev_warn(adev->dev,
3681 				"NPA-REQ: send NPA-FAIL failed remote:%u handle:%llx:%llx\n",
3682 				remote_acc_id, handle.handle_hi, handle.handle_lo);
3683 	}
3684 	return;
3685 conn_setup:
3686 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3687 	if (r)
3688 		dev_warn(adev->dev,
3689 			"NPA-REQ: connection setup failed with remote AccId:%u\n",
3690 			remote_acc_id);
3691 }
3692 
3693 static int amdgpu_ualink_translate_npa_fail_reason(struct amdgpu_device *adev,
3694 						   u32 fail_reason)
3695 {
3696 	switch (fail_reason) {
3697 	case AMDGPU_UALINK_NPA_FAIL_NOSPACE:
3698 		return -ENOSPC;
3699 	case AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE:
3700 	case AMDGPU_UALINK_NPA_FAIL_DUPLICATE:
3701 	case AMDGPU_UALINK_NPA_FAIL_ERROR:
3702 		return -EINVAL;
3703 	default:
3704 		dev_err(adev->dev,
3705 			"IMPORT: invalid NPA-FAIL reason:%u\n",
3706 			fail_reason);
3707 		return -EINVAL;
3708 	}
3709 }
3710 
3711 static int amdgpu_ualink_do_import_handle(struct amdgpu_device *adev,
3712 				struct amdgpu_ualink_imp_xa_node *imp_xa_node,
3713 				u32 remote_acc_id)
3714 {
3715 	struct amdgpu_ualink_handle handle = imp_xa_node->handle;
3716 	u32 generation_count;
3717 	int r;
3718 
3719 	/* First check if the connection is setup with the
3720 	 * remote GPU. If yes, then initiate the NPA protocol to
3721 	 * get the NPA address.
3722 	 * If not, then initiate the HELLO protocol to first setup
3723 	 * the connection and once the connection is setup, then
3724 	 * initiate the NPA protocol.
3725 	 */
3726 	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
3727 	if (r) {
3728 		if (r != -EAGAIN)
3729 			dev_warn(adev->dev,
3730 			"IMPORT: connection setup failed with remote AccId:%u\n",
3731 			remote_acc_id);
3732 		return r;
3733 	}
3734 
3735 	/* Send NPA_REQ message */
3736 	r = amdgpu_ualink_send_npa_req_msg(adev, remote_acc_id, handle);
3737 	if (r) {
3738 		dev_warn(adev->dev,
3739 			"IMPORT: NPA-REQ send failed to remote AccId:%u\n",
3740 			remote_acc_id);
3741 		goto reset_conn;
3742 	}
3743 
3744 	/* Wait for the NPA_RSP to come back */
3745 	r = wait_for_completion_interruptible_timeout(&imp_xa_node->npa_done,
3746 				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
3747 	if (r == -ERESTARTSYS) {
3748 		dev_err_ratelimited(adev->dev,
3749 			"IMPORT: NPA-RSP wait interrupted by signal\n");
3750 		return r;
3751 	} else if (r == 0) {
3752 		dev_warn(adev->dev,
3753 			"IMPORT: NPA-RSP timeout from remote AccId:%u\n",
3754 			remote_acc_id);
3755 		r = -ETIMEDOUT;
3756 		goto reset_conn;
3757 	}
3758 
3759 	/* If the NPA addr/size isn't filled with valid values, then either
3760 	 * we got a NPA_FAIL or something bad happened. In either case, we
3761 	 * return the error back to user-space.
3762 	 */
3763 	if (imp_xa_node->fail_reason) {
3764 		dev_warn(adev->dev,
3765 			"IMPORT: NPA-REQ failed with fail_reason:%d handle:%llx:%llx\n",
3766 			imp_xa_node->fail_reason, handle.handle_hi, handle.handle_lo);
3767 		return amdgpu_ualink_translate_npa_fail_reason(adev,
3768 						imp_xa_node->fail_reason);
3769 	}
3770 
3771 	if (!imp_xa_node->npa_addr || !imp_xa_node->size) {
3772 		dev_warn(adev->dev,
3773 			"IMPORT: invalid npa:%llx or size:%llx\n",
3774 			imp_xa_node->npa_addr, imp_xa_node->size);
3775 		imp_xa_node->npa_addr = 0;
3776 		imp_xa_node->size = 0;
3777 		return -EINVAL;
3778 	}
3779 
3780 	r = amdgpu_ualink_map_npa_to_dmabuf(adev, imp_xa_node);
3781 	if (r) {
3782 		dev_warn(adev->dev,
3783 			"IMPORT: dmabuf creation failed npa:%llx size:%llx\n",
3784 			imp_xa_node->npa_addr, imp_xa_node->size);
3785 		imp_xa_node->npa_addr = 0;
3786 		imp_xa_node->size = 0;
3787 		return r;
3788 	}
3789 
3790 	/* Add this node to the imported handles list for the remote GPU,
3791 	 * unless the exporter revoked the handle while the import was in
3792 	 * flight. The dmabuf is released with the last node reference.
3793 	 */
3794 	xa_lock(&adev->ualink.imp_xa);
3795 	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_TEARDOWN) {
3796 		xa_unlock(&adev->ualink.imp_xa);
3797 		dev_dbg(adev->dev,
3798 			"IMPORT: handle:%llx:%llx revoked during import\n",
3799 			handle.handle_hi, handle.handle_lo);
3800 		return -EINVAL;
3801 	}
3802 	list_add(&imp_xa_node->list, &adev->ualink.imp_handles_list[remote_acc_id]);
3803 	WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_READY);
3804 	xa_unlock(&adev->ualink.imp_xa);
3805 
3806 	return 0;
3807 
3808 reset_conn:
3809 	dev_dbg(adev->dev,
3810 		"IMPORT: Resetting connection for remote:%u\n", remote_acc_id);
3811 	generation_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
3812 	amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
3813 					      AMDGPU_UALINK_CONN_NOT_READY,
3814 					      generation_count);
3815 	return r;
3816 }
3817 
3818 static int amdgpu_ualink_local_import(struct amdgpu_device *adev,
3819 				      u32 remote_acc_id,
3820 				      struct amdgpu_ualink_handle *handle,
3821 				      int *fd_out)
3822 {
3823 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
3824 	struct amdgpu_device *peer_adev;
3825 	int fd;
3826 
3827 	mutex_lock(&mgpu_info.mutex);
3828 	peer_adev = find_peer_adev(remote_acc_id);
3829 	mutex_unlock(&mgpu_info.mutex);
3830 	if (!peer_adev) {
3831 		dev_err(adev->dev,
3832 			"IMPORT LOCAL: peer adev not found for AccId:%u\n",
3833 			remote_acc_id);
3834 		return -ENODEV;
3835 	}
3836 
3837 	xa_lock(&peer_adev->ualink.exp_xa);
3838 	exp_xa_node = xa_load(&peer_adev->ualink.exp_xa, handle->handle_lo);
3839 	if (!exp_xa_node ||
3840 	    exp_xa_node->handle.handle_hi != handle->handle_hi ||
3841 	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
3842 		xa_unlock(&peer_adev->ualink.exp_xa);
3843 		dev_err(adev->dev,
3844 			"IMPORT LOCAL: handle:%llx:%llx not found in peer exp_xa\n",
3845 			handle->handle_hi, handle->handle_lo);
3846 		return -EINVAL;
3847 	}
3848 	xa_unlock(&peer_adev->ualink.exp_xa);
3849 
3850 	get_dma_buf(exp_xa_node->dmabuf);
3851 	/* Get a new fd for the DMABuf */
3852 	fd = dma_buf_fd(exp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
3853 	if (fd < 0) {
3854 		dev_err(adev->dev,
3855 			"IMPORT LOCAL: dma-buf fd failed handle:%llx:%llx\n",
3856 			handle->handle_hi, handle->handle_lo);
3857 		dma_buf_put(exp_xa_node->dmabuf);
3858 		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3859 		return fd;
3860 	}
3861 
3862 	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
3863 	*fd_out = fd;
3864 	return 0;
3865 }
3866 
3867 int amdgpu_ualink_import_handle(struct drm_device *dev,
3868 				const struct amdgpu_ualink_handle *ualink_handle,
3869 				int *fd_out)
3870 {
3871 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
3872 	struct amdgpu_device *adev = drm_to_adev(dev);
3873 	struct amdgpu_ualink_handle handle = *ualink_handle;
3874 	u32 remote_acc_id, node_state;
3875 	int r = 0, fd;
3876 
3877 	remote_acc_id = (handle.handle_lo &
3878 			AMDGPU_UALINK_HANDLE_ACCID_MASK);
3879 
3880 	if (remote_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
3881 		dev_err(adev->dev,
3882 			"IMPORT: invalid remote AccId:%u\n", remote_acc_id);
3883 		return -EINVAL;
3884 	}
3885 
3886 	if (amdgpu_ualink_is_local_accel(adev, remote_acc_id))
3887 		return amdgpu_ualink_local_import(adev, remote_acc_id,
3888 						  &handle, fd_out);
3889 
3890 	xa_lock(&adev->ualink.imp_xa);
3891 	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
3892 
3893 	if (imp_xa_node) {
3894 		/* If node state is Not_ready/Pending, then some other
3895 		 * thread is already trying the NPA protocol for the same
3896 		 * ualink handle. Back off and let the thread finish.
3897 		 * If the node state is Teardown, then it means this
3898 		 * node is about to be removed. So let user-space know
3899 		 * that this handle is invalid.
3900 		 */
3901 		node_state = READ_ONCE(imp_xa_node->node_state);
3902 		if (node_state == AMDGPU_UALINK_NODE_NOT_READY ||
3903 		    node_state == AMDGPU_UALINK_NODE_PENDING) {
3904 			xa_unlock(&adev->ualink.imp_xa);
3905 			r = -EAGAIN;
3906 			goto out;
3907 		} else if (node_state == AMDGPU_UALINK_NODE_TEARDOWN) {
3908 			xa_unlock(&adev->ualink.imp_xa);
3909 			r = -EINVAL;
3910 			goto out;
3911 		}
3912 
3913 		/* Increase the refcount while we are processing the request */
3914 		r = amdgpu_ualink_imp_xa_entry_get(imp_xa_node) ? 0 : -EINVAL;
3915 		xa_unlock(&adev->ualink.imp_xa);
3916 
3917 		/* If the refcount has become 0 but the entry is not yet
3918 		 * removed from the Xarray, then return error to user-space.
3919 		 */
3920 		if (r)
3921 			goto out;
3922 	} else {
3923 		xa_unlock(&adev->ualink.imp_xa);
3924 		/* if the partial handle doesn't exist in the Importer xarray then
3925 		 * initiate the NPA protocol and generate the DMABuf corresponding
3926 		 * to the NPA address.
3927 		 * First store the entry in the Xarray.
3928 		 */
3929 		imp_xa_node = kzalloc_obj(*imp_xa_node);
3930 		if (!imp_xa_node) {
3931 			r = -ENOMEM;
3932 			goto out;
3933 		}
3934 		imp_xa_node->adev = adev;
3935 		imp_xa_node->node_state = AMDGPU_UALINK_NODE_NOT_READY;
3936 		imp_xa_node->handle = handle;
3937 		init_completion(&imp_xa_node->npa_done);
3938 		INIT_LIST_HEAD(&imp_xa_node->list);
3939 		kref_init(&imp_xa_node->refcount);
3940 
3941 		/* Take an extra reference to store in the Xarray. The error
3942 		 * handling paths will drop both these references, while a
3943 		 * successful path will drop only one reference to the Xarray
3944 		 * entry.
3945 		 */
3946 		amdgpu_ualink_imp_xa_entry_get(imp_xa_node);
3947 
3948 		/* Check if another thread created a node for the same handle while
3949 		 * we were trying to create and initialize the node.
3950 		 */
3951 		r = xa_insert(&adev->ualink.imp_xa, handle.handle_lo,
3952 			      imp_xa_node, GFP_KERNEL);
3953 		if (r) {
3954 			kfree(imp_xa_node);
3955 			/* -EBUSY means another thread raced us and inserted a
3956 			 * node for the same handle. Ask user-space to retry.
3957 			 */
3958 			if (r == -EBUSY)
3959 				r = -EAGAIN;
3960 			else
3961 				dev_err(adev->dev,
3962 					"IMPORT: XA insert failed for handle:%llx:%llx err:%d\n",
3963 					handle.handle_hi, handle.handle_lo, r);
3964 			goto out;
3965 		}
3966 
3967 		r = amdgpu_ualink_do_import_handle(adev, imp_xa_node, remote_acc_id);
3968 
3969 		/* If error is returned, then cleanup the xarray entry before returning
3970 		 * the error back to user-space
3971 		 */
3972 		if (r) {
3973 			amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
3974 			if (r != -EAGAIN)
3975 				dev_err(adev->dev,
3976 					"IMPORT: XA import failed for handle:%llx:%llx\n",
3977 					handle.handle_hi, handle.handle_lo);
3978 			goto cleanup;
3979 		}
3980 	}
3981 
3982 	/* dma_buf_fd consumes a reference and assigns it to the fd.
3983 	 * Therefore take an extra reference to be consumed. It will be
3984 	 * released when user mode closes the fd.
3985 	 */
3986 	get_dma_buf(imp_xa_node->dmabuf);
3987 
3988 	fd = dma_buf_fd(imp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
3989 	if (fd >= 0) {
3990 		*fd_out = fd;
3991 	} else {
3992 		dma_buf_put(imp_xa_node->dmabuf);
3993 		r = fd;
3994 		dev_err(adev->dev,
3995 			"IMPORT: dma-buf fd creation failed handle:%llx:%llx\n",
3996 			handle.handle_hi, handle.handle_lo);
3997 	}
3998 
3999 cleanup:
4000 	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
4001 out:
4002 	return r;
4003 }
4004 
4005 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp,
4006 				u32 gem_handle,
4007 				struct amdgpu_ualink_handle *handle_out)
4008 {
4009 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
4010 	struct amdgpu_ualink_handle handle;
4011 	struct drm_gem_object *gobj;
4012 	struct amdgpu_device *adev;
4013 	struct amdgpu_bo *robj;
4014 	int r = 0, i;
4015 
4016 	gobj = drm_gem_object_lookup(filp, gem_handle);
4017 	if (!gobj)
4018 		return -ENOENT;
4019 
4020 	robj = gem_to_amdgpu_bo(gobj);
4021 	adev = amdgpu_ttm_adev(robj->tbo.bdev);
4022 
4023 	if (!(robj->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
4024 		dev_err(adev->dev, "Only VRAM BOs can be exported\n");
4025 		r = -EOPNOTSUPP;
4026 		goto out;
4027 	}
4028 
4029 	if (!robj->ualink_handle_lo) {
4030 		/* If no ualink handle generated for BO, then generate one and
4031 		 * add it to the exporter Xarray.
4032 		 */
4033 		exp_xa_node = kzalloc_obj(*exp_xa_node);
4034 		if (!exp_xa_node) {
4035 			dev_err(adev->dev, "Failed to allocate exp_xa_node\n");
4036 			r = -ENOMEM;
4037 			goto out;
4038 		}
4039 
4040 		amdgpu_bo_ref(robj);
4041 		exp_xa_node->bo = robj;
4042 		init_completion(&exp_xa_node->npa_done);
4043 		bitmap_zero(exp_xa_node->importers_bitmap,
4044 			    AMDGPU_UALINK_ACCEL_MAX);
4045 		bitmap_zero(exp_xa_node->npa_release_bitmap,
4046 			AMDGPU_UALINK_ACCEL_MAX);
4047 		kref_init(&exp_xa_node->refcount);
4048 		mutex_init(&exp_xa_node->node_lock);
4049 		INIT_WORK(&exp_xa_node->cleanup_work,
4050 			  amdgpu_ualink_exp_cleanup_worker);
4051 		for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4052 			INIT_LIST_HEAD(&exp_xa_node->importer_entries[i].list);
4053 			exp_xa_node->importer_entries[i].parent = exp_xa_node;
4054 		}
4055 		/* DMABuf handle for local import of fabric handles */
4056 		exp_xa_node->dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, filp,
4057 						gem_handle, DRM_CLOEXEC | DRM_RDWR);
4058 		if (IS_ERR(exp_xa_node->dmabuf)) {
4059 			r = PTR_ERR(exp_xa_node->dmabuf);
4060 			dev_err(adev->dev, "Failed to generate DMABuf for the BO\n");
4061 			goto err_unref_bo;
4062 		}
4063 
4064 		xa_lock(&adev->ualink.exp_xa);
4065 		amdgpu_generate_ualink_handle(adev, &handle);
4066 		exp_xa_node->handle = handle;
4067 		r = __xa_insert(&adev->ualink.exp_xa, handle.handle_lo,
4068 				exp_xa_node, GFP_KERNEL);
4069 		xa_unlock(&adev->ualink.exp_xa);
4070 		if (r) {
4071 			dev_err(adev->dev, "Failed to insert exp_xa_node into XA: %d\n", r);
4072 			goto err_put_dmabuf;
4073 		}
4074 
4075 		r = amdgpu_bo_reserve(robj, false);
4076 		if (r)
4077 			goto err_erase_xa;
4078 
4079 		if (robj->ualink_handle_lo) {
4080 			amdgpu_bo_unreserve(robj);
4081 			r = -EAGAIN;
4082 			goto err_erase_xa;
4083 		}
4084 
4085 		robj->ualink_handle_lo = handle.handle_lo;
4086 		amdgpu_bo_unreserve(robj);
4087 
4088 		/* Return the generated handle back to the caller */
4089 		*handle_out = handle;
4090 	} else {
4091 		handle_out->handle_lo = robj->ualink_handle_lo;
4092 
4093 		/* Do a sanity check to ensure the handle exists in the XA */
4094 		xa_lock(&adev->ualink.exp_xa);
4095 		exp_xa_node = xa_load(&adev->ualink.exp_xa,
4096 				      robj->ualink_handle_lo);
4097 		xa_unlock(&adev->ualink.exp_xa);
4098 		WARN(!exp_xa_node, "Exp XA: Handle_Lo: %llx not found",
4099 		     robj->ualink_handle_lo);
4100 		if (exp_xa_node)
4101 			handle_out->handle_hi = exp_xa_node->handle.handle_hi;
4102 	}
4103 
4104 	drm_gem_object_put(gobj);
4105 	return 0;
4106 
4107 err_erase_xa:
4108 	xa_erase(&adev->ualink.exp_xa, handle.handle_lo);
4109 err_put_dmabuf:
4110 	dma_buf_put(exp_xa_node->dmabuf);
4111 err_unref_bo:
4112 	amdgpu_bo_unref(&exp_xa_node->bo);
4113 	kfree(exp_xa_node);
4114 out:
4115 	drm_gem_object_put(gobj);
4116 	return r;
4117 }
4118 
4119 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data,
4120 				   struct drm_file *filp)
4121 {
4122 	union drm_amdgpu_ualink_handle *args = data;
4123 	struct amdgpu_device *adev = drm_to_adev(dev);
4124 	struct amdgpu_ualink_handle handle = {};
4125 	u32 gem_handle;
4126 	int r, fd = -1;
4127 
4128 	/* The ioctl is registered for every amdgpu device, but the UALink
4129 	 * software state only exists on ASICs that instantiate the UALink IP
4130 	 * block. Bail out if it is absent, and short-circuit before the
4131 	 * accel_state dereference below.
4132 	 */
4133 	if (!adev->ualink.info ||
4134 	    adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
4135 		dev_dbg(adev->dev,
4136 			"ualink device not available or not in active state\n");
4137 		return -EOPNOTSUPP;
4138 	}
4139 
4140 	/* The input and output members of the ioctl argument alias each other.
4141 	 * Latch every input field before invoking the handlers, and only write
4142 	 * the output fields afterwards.
4143 	 */
4144 	switch (args->in.op) {
4145 	case DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT:
4146 		gem_handle = args->in.gem_handle;
4147 		r = amdgpu_ualink_export_handle(dev, filp, gem_handle, &handle);
4148 		if (!r) {
4149 			args->out.export_ualink_handle[0] = handle.handle_lo;
4150 			args->out.export_ualink_handle[1] = handle.handle_hi;
4151 		}
4152 		break;
4153 	case DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT:
4154 		handle.handle_lo = args->in.import_ualink_handle[0];
4155 		handle.handle_hi = args->in.import_ualink_handle[1];
4156 		r = amdgpu_ualink_import_handle(dev, &handle, &fd);
4157 		if (!r)
4158 			args->out.import_dmabuf_handle = fd;
4159 		break;
4160 	default:
4161 		r = -EINVAL;
4162 		break;
4163 	}
4164 
4165 	return r;
4166 }
4167 
4168 int amdgpu_ualink_manager_start(struct amdgpu_device *adev)
4169 {
4170 	int i, r;
4171 
4172 	r = amdgpu_vm_init(adev, &adev->ualink.npa_vm, 0);
4173 	if (r)
4174 		goto out;
4175 
4176 	/* For pinning page tables and using CPU for page table updates. */
4177 	r = amdgpu_vm_make_npa(adev, &adev->ualink.npa_vm);
4178 	if (r)
4179 		goto uninit_vm;
4180 
4181 	adev->ualink.npa_wq = alloc_workqueue("NPA WQ", WQ_UNBOUND, 0);
4182 	if (unlikely(!adev->ualink.npa_wq)) {
4183 		dev_err(adev->dev, "Failed to allocate NPA WQ\n");
4184 		r = -ENOMEM;
4185 		goto uninit_vm;
4186 	}
4187 
4188 	/* Map this VM to NPA VMID */
4189 	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid,
4190 			amdgpu_gmc_pd_addr(adev->ualink.npa_vm.root.bo));
4191 
4192 	xa_init(&adev->ualink.exp_xa);
4193 	xa_init(&adev->ualink.imp_xa);
4194 	xa_init(&adev->ualink.handle_invalid_xa);
4195 
4196 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4197 		init_completion(&adev->ualink.conn_state[i].hello_done);
4198 		mutex_init(&adev->ualink.conn_state[i].lock);
4199 		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
4200 		INIT_LIST_HEAD(&adev->ualink.exp_handles_list[i]);
4201 		INIT_LIST_HEAD(&adev->ualink.imp_handles_list[i]);
4202 	}
4203 
4204 	amdgpu_ualink_npa_mm_init(adev);
4205 
4206 	return 0;
4207 
4208 uninit_vm:
4209 	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
4210 out:
4211 	return r;
4212 }
4213 
4214 /* Free every importer entry left in imp_xa at manager stop. */
4215 static void amdgpu_ualink_teardown_imp_xa_entries(struct amdgpu_device *adev)
4216 {
4217 	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
4218 	unsigned long index;
4219 
4220 	xa_for_each(&adev->ualink.imp_xa, index, imp_xa_node) {
4221 		list_del_init(&imp_xa_node->list);
4222 		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
4223 		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
4224 	}
4225 }
4226 
4227 /* Free a single exporter node at manager stop. This is the message-free
4228  * portion of amdgpu_ualink_exp_cleanup_worker().
4229  */
4230 static void amdgpu_ualink_teardown_exp_xa_node(struct amdgpu_device *adev,
4231 				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
4232 {
4233 	DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
4234 	struct amdgpu_bo *bo = exp_xa_node->bo;
4235 	u32 remote_acc_id;
4236 
4237 	bitmap_copy(importers_bitmap, exp_xa_node->importers_bitmap,
4238 		    AMDGPU_UALINK_ACCEL_MAX);
4239 
4240 	dev_dbg(adev->dev,
4241 		"EXP-STOP: handle:%llx:%llx importers bitmap: %*pbl\n",
4242 		exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
4243 		AMDGPU_UALINK_ACCEL_MAX, importers_bitmap);
4244 
4245 	if (!bitmap_empty(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)) {
4246 		amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
4247 
4248 		/* Unlink importers and unpin the BO once per importer. */
4249 		if (likely(!amdgpu_bo_reserve(bo, true))) {
4250 			bo->ualink_handle_lo = 0ULL;
4251 			for_each_set_bit(remote_acc_id, importers_bitmap,
4252 					 AMDGPU_UALINK_ACCEL_MAX) {
4253 				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
4254 				amdgpu_bo_unpin(bo);
4255 			}
4256 			amdgpu_bo_unreserve(bo);
4257 		} else {
4258 			dev_warn(adev->dev,
4259 				"EXP-STOP: BO reserve to unpin failed handle:%llx:%llx\n",
4260 				exp_xa_node->handle.handle_hi,
4261 				exp_xa_node->handle.handle_lo);
4262 			for_each_set_bit(remote_acc_id, importers_bitmap,
4263 					 AMDGPU_UALINK_ACCEL_MAX)
4264 				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
4265 		}
4266 
4267 		amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, importers_bitmap);
4268 	}
4269 
4270 	/* Release the export dma-buf and drop the BO ref. */
4271 	dma_buf_put(exp_xa_node->dmabuf);
4272 	amdgpu_bo_unref(&bo);
4273 	exp_xa_node->bo = NULL;
4274 
4275 	mutex_destroy(&exp_xa_node->node_lock);
4276 	kfree(exp_xa_node);
4277 }
4278 
4279 /* Free every exporter entry left in exp_xa at manager stop. These were
4280  * exported but never revoked, so no cleanup worker was queued for them.
4281  */
4282 static void amdgpu_ualink_teardown_exp_xa_entries(struct amdgpu_device *adev)
4283 {
4284 	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
4285 	unsigned long index;
4286 
4287 	xa_for_each(&adev->ualink.exp_xa, index, exp_xa_node) {
4288 		xa_erase(&adev->ualink.exp_xa, index);
4289 		amdgpu_ualink_teardown_exp_xa_node(adev, exp_xa_node);
4290 	}
4291 }
4292 
4293 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev)
4294 {
4295 	int i;
4296 
4297 	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 0);
4298 
4299 	/* Mark connections down so the drained workers and the teardown below
4300 	 * take the message-free path in amdgpu_ualink_check_conn_ready().
4301 	 */
4302 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
4303 		mutex_lock(&adev->ualink.conn_state[i].lock);
4304 		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
4305 		mutex_unlock(&adev->ualink.conn_state[i].lock);
4306 	}
4307 
4308 	/* Drain in-flight exporter cleanup work so revoked nodes in
4309 	 * handle_invalid_xa free themselves and empty that xarray.
4310 	 */
4311 	drain_workqueue(adev->ualink.npa_wq);
4312 
4313 	/* Free live entries before the NPA allocator is torn down, so NPA
4314 	 * addresses can still be unmapped and freed.
4315 	 */
4316 	amdgpu_ualink_teardown_imp_xa_entries(adev);
4317 	amdgpu_ualink_teardown_exp_xa_entries(adev);
4318 
4319 	amdgpu_ualink_npa_mm_fini(adev);
4320 
4321 	/* All three xarrays are empty by now. */
4322 	xa_destroy(&adev->ualink.exp_xa);
4323 	xa_destroy(&adev->ualink.imp_xa);
4324 	xa_destroy(&adev->ualink.handle_invalid_xa);
4325 
4326 	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++)
4327 		mutex_destroy(&adev->ualink.conn_state[i].lock);
4328 
4329 	destroy_workqueue(adev->ualink.npa_wq);
4330 	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
4331 }
4332 
4333 /*
4334  * UALink remote interrupt and shootdown
4335  */
4336 
4337 /* UALINK ring buffer size, same for both remote shootdown and interrupt ring */
4338 #define AMDGPU_UALINK_RB_SIZE	4096
4339 
4340 #define AMDGPU_UALINK_METADATA_HEADER	0x4E485446
4341 
4342 /* 2MB NPA start address for 2MB page mapping */
4343 #define AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET SZ_2M
4344 
4345 struct amdgpu_ualink_metadata {
4346 	u32 header;
4347 
4348 	/*
4349 	 * "ring entries" as unit. So the ring-size-in-bytes could be calculated
4350 	 * as entry-size * 2^RBsize. This would make the minimum size a single
4351 	 * entry and the maximum size 32786 entries.
4352 	 *
4353 	 * u32 rb_size:4;
4354 	 * u32 reserved0:4;
4355 	 * u32 vmid:4;
4356 	 * u32 reserved1:20;
4357 	 */
4358 	u32 rb_size;
4359 
4360 	/* ring buffer base address for remote interrupt */
4361 	u64 ri_rb;
4362 
4363 	/* tlb invalidate ring buffer base address for remote shootdown */
4364 	u64 tlb_inv_rb;
4365 
4366 	/* remote interrupt Tail pointer, write pointer address */
4367 	u64 tailptr_ri;
4368 
4369 	/* TLB invalidate ring buffer's Tail pointer, write pointer address */
4370 	u64 tailptr_tlb_inv;
4371 };
4372 
4373 struct amdgpu_ualink_wb {
4374 	/* last finished command seq number */
4375 	u32 data;
4376 
4377 	/* command complete error code */
4378 	u32 status;
4379 
4380 	/* ring rptr updated by FW */
4381 	u64 rptr;
4382 };
4383 
4384 /*
4385  * For remote interrupt and shootdown
4386  */
4387 struct amdgpu_ualink_ring {
4388 	u32 rb_size;
4389 
4390 	/* writeback data */
4391 	u32 seq;
4392 
4393 	/* local copy */
4394 	u64 wptr, rptr;
4395 
4396 	/* NPA gart mapping for SDMA */
4397 	u64 rb_npa_gart;
4398 	u64 wptr_npa_gart;
4399 	u64 doorbell_npa_gart;
4400 
4401 	/* gart mapping node */
4402 	struct drm_mm_node mm_node_rb;
4403 	struct drm_mm_node mm_node_wptr;
4404 	struct drm_mm_node mm_node_doorbell;
4405 
4406 	/* seq, complete status write back cpu address */
4407 	struct amdgpu_ualink_wb *wb_cpu;
4408 
4409 	/* true if fw write back address updated successfully */
4410 	bool ready;
4411 };
4412 
4413 struct amdgpu_ualink_peer {
4414 	/* remote interrupt and shootdown uses same SDMA entity */
4415 	struct mutex lock;
4416 
4417 	/* SDMA engine to send remote command via NPA */
4418 	struct drm_sched_entity entity;
4419 
4420 	/* to select different DXS ports, cycles through different values */
4421 	u32 dxs_port;
4422 
4423 	struct amdgpu_ualink_ring interrupt;
4424 	struct amdgpu_ualink_ring shootdown;
4425 };
4426 
4427 struct amdgpu_ualink_remote {
4428 	/* ualink metadata passed to MPNHT FW */
4429 	struct amdgpu_bo		*metadata_bo;
4430 	u64				metadata_gpu_addr;
4431 	void				*metadata_cpu_addr;
4432 
4433 	/* ualink ring, tlb ring buffer, wptr */
4434 	struct amdgpu_bo		*ring_bo;
4435 	u64				rb_gpu_addr;
4436 	void				*rb_cpu_addr;
4437 
4438 	/* ualink rptr, wb data, statuss for address alias mode */
4439 	struct amdgpu_bo		*rptr_bo;
4440 	u64				rptr_gpu_addr;
4441 	void				*rptr_cpu_addr;
4442 	u64				rptr_npa;
4443 
4444 	/* address alias mode alloc npa address for shared wb */
4445 	struct drm_mm_node		rptr_mm_node;
4446 
4447 	/*
4448 	 * Owned snapshot of the vPod's active accelerator bitmap, taken at
4449 	 * sw_init time. Must NOT alias info->vpod.active_accel_bits, which
4450 	 * psp_ual_query_info() refreshes on every commit: aliasing it would let
4451 	 * teardown unmap a different set than setup mapped, leaking drm_mm nodes
4452 	 * in the shared GTT manager.
4453 	 */
4454 	DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
4455 	u32				num_accel;
4456 
4457 	/* remote GPUs ring buffer, read, write pointer local copy and gart mapping */
4458 	struct amdgpu_ualink_peer	peer[AMDGPU_UALINK_ACCEL_MAX];
4459 
4460 	/* use lsdma write to NPA address for remote interrupt */
4461 	bool			       use_lsdma;
4462 };
4463 
4464 static inline struct amdgpu_ualink_remote *to_remote(struct amdgpu_device *adev)
4465 {
4466 	return adev->ualink.remote;
4467 }
4468 
4469 /*
4470  * Returns true if the vPod membership most recently reported by firmware
4471  * (info->vpod.active_accel_bits, read back via psp_ual_query_info() on the
4472  * committing GPU) differs from the owned snapshot captured when this
4473  * accelerator was last activated (remote->active_accel_bits). Used to
4474  * decide whether an accelerator must be (re)built to match the new member
4475  * set. A NULL remote means the accelerator has never been activated (no
4476  * snapshot yet), which also counts as "changed" so it gets brought up.
4477  */
4478 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev)
4479 {
4480 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4481 
4482 	if (!remote)
4483 		return true;
4484 
4485 	return !bitmap_equal(remote->active_accel_bits,
4486 			     adev->ualink.info->vpod.active_accel_bits,
4487 			     AMDGPU_UALINK_ACCEL_MAX);
4488 }
4489 
4490 static inline u32 ualink_accel_id(struct amdgpu_device *adev)
4491 {
4492 	return adev->ualink.info->ppod.accel_id;
4493 }
4494 
4495 static inline enum amdgpu_ualink_addr_mode ualink_addr_mode(struct amdgpu_device *adev)
4496 {
4497 	return adev->ualink.info->vpod.addr_mode;
4498 }
4499 
4500 /*
4501  * address mode and NPA address for ring wptr, rptr, wb status, wb data
4502  *
4503  * source identification address mode
4504  *
4505  * we need to export different tail pointers for different remote GPU accel_id,
4506  * hence we need to allocate a whole page for each remote GPU separately.
4507  *
4508  * rptr, wb data, status share the whole page with wptr.
4509  *
4510  *     1 page reserved NPA address for 1 active remote GPU, max 255 pages
4511  *     starting address: wptr NPA address + accel_id * 4K
4512  *     offset:
4513  *         remote interrupt ring wptr 0
4514  *         remote shootdown ring wptr 8
4515  *         remote interrupt wb status 16
4516  *         remote interrupt wb data   20
4517  *         remote interrupt ring rptr 24
4518  *         remote shootdown wb status 32
4519  *         remote shootdown wb data   36
4520  *         remote shootdown ring rptr 40
4521  *
4522  * source aliasing address mode
4523  *
4524  * wptr for different remote GPU can share same reserved 1 page NPA address page.
4525  *
4526  * to save reserved NPA address space, rptr, wb data, status for different
4527  * remote GPU share same allocated 2 pages NPA address.
4528  *
4529  *     1 page reserved NPA address for wptr of all active remote GPUs
4530  *     offset:
4531  *         remote interrupt ring wptr accel_id * 16
4532  *         remote shootdown ring wptr accel_id * 16 + 8
4533  *
4534  *     2 page allocated NPA address for rptr, writeback of all active remote GPUs
4535  *     starting address: at accel_id * 32 for each remote GPU
4536  *     offset:
4537  *         remote interrupt wb status 0
4538  *         remote interrupt wb data   4
4539  *         remote interrupt ring rptr 8
4540  *         remote shootdown wb status 16
4541  *         remote shootdown wb data   20
4542  *         remote shootdown ring rptr 24
4543  */
4544 static inline u32 ualink_wptr_size(struct amdgpu_device *adev)
4545 {
4546 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4547 		return AMDGPU_GPU_PAGE_SIZE;
4548 	else
4549 		return 0;	/* wptr uses the ring buffer for itself */
4550 }
4551 
4552 static inline u32 ualink_wb_size(struct amdgpu_device *adev)
4553 {
4554 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4555 		return 0;	/* share same page with wptr */
4556 	else
4557 		return 2 * sizeof(struct amdgpu_ualink_wb);
4558 }
4559 
4560 static inline u32 ualink_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
4561 {
4562 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4563 		return 0;	/* one separate page per GPU */
4564 	else
4565 		return accel_id * 2 * sizeof(u64);	/* two pointers per GPU */
4566 }
4567 
4568 static inline u32 ualink_tlb_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
4569 {
4570 	return ualink_wptr_offset(adev, accel_id) + sizeof(u64);
4571 }
4572 
4573 static inline u32 ualink_wb_offset(struct amdgpu_device *adev, u32 accel_id)
4574 {
4575 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
4576 		return 2 * sizeof(u64);	/* after two wptr pointers */
4577 	else
4578 		return accel_id * ualink_wb_size(adev);
4579 }
4580 
4581 static inline u32 ualink_tlb_wb_offset(struct amdgpu_device *adev, u32 accel_id)
4582 {
4583 	return ualink_wb_offset(adev, accel_id) + sizeof(struct amdgpu_ualink_wb);
4584 }
4585 
4586 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, u32 flush_type)
4587 {
4588 	u32 bit;
4589 
4590 	bit = AMDGPU_MMHUB0_START;
4591 
4592 	for_each_set_bit_from(bit, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
4593 		amdgpu_gmc_flush_gpu_tlb(adev, adev->vm_manager.npa_vmid,
4594 					bit, flush_type);
4595 }
4596 
4597 /**
4598  * amdgpu_ualink_npa_vm_map_range - Map a range in the NPA VM
4599  * @adev: amdgpu device pointer
4600  * @bo: buffer object backing the mapping
4601  * @pte_flags: page table entry flags
4602  * @offset: offset into the buffer object in bytes
4603  * @size_in_pages: size of the range to map in pages
4604  * @npa_in_pages: NPA target address in pages
4605  *
4606  * Maps a buffer object range into the NPA VM page table at the specified
4607  * NPA address.
4608  *
4609  * Return: 0 on success, negative error code on failure
4610  */
4611 static int amdgpu_ualink_npa_vm_map_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
4612 				   u64 pte_flags, u64 offset, u64 size_in_pages,
4613 				   u64 npa_in_pages)
4614 {
4615 	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
4616 	int r;
4617 
4618 	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
4619 		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
4620 		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4621 
4622 	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
4623 				   false, NULL, npa_in_pages,
4624 				   npa_in_pages + size_in_pages - 1,
4625 				   pte_flags, offset, adev->vm_manager.vram_base_offset,
4626 				   bo->tbo.resource, NULL, &npa_vm->last_update);
4627 	if (r)
4628 		dev_dbg(adev->dev, "failed %d to map npa 0x%llx to NPA VM\n", r,
4629 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4630 	return r;
4631 }
4632 
4633 /**
4634  * amdgpu_npa_vm_unmap_range - Unmap a range from the NPA VM
4635  * @adev: amdgpu device pointer
4636  * @bo: buffer object backing the mapping
4637  * @pte_flags: page table entry flags
4638  * @offset: offset into the buffer object in bytes
4639  * @size_in_pages: size of the range to unmap in pages
4640  * @npa_in_pages: NPA target address in pages
4641  *
4642  * Removes a previously established mapping from the NPA VM page table.
4643  *
4644  * Return: 0 on success, negative error code on failure
4645  */
4646 static int amdgpu_ualink_npa_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
4647 				     u64 pte_flags, u64 offset, u64 size_in_pages,
4648 				     u64 npa_in_pages)
4649 {
4650 	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
4651 	int r;
4652 
4653 	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
4654 		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
4655 		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4656 
4657 	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
4658 				   false, NULL, npa_in_pages,
4659 				   npa_in_pages + size_in_pages - 1,
4660 				   pte_flags, offset, 0, bo->tbo.resource, NULL,
4661 				   &npa_vm->last_update);
4662 	if (r)
4663 		dev_dbg(adev->dev, "failed %d to unmap npa 0x%llx from NPA VM\n", r,
4664 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
4665 	return r;
4666 }
4667 
4668 /*
4669  * Reserved NPA space for remote shootdown and interrupt ring buffer,
4670  * write pointers, read pointers and writeback buffers
4671  *
4672  * NPA ring type: 0 = shootdown, 1 = interrupt, 2 = tailptr (wptr)
4673  */
4674 enum ring_buffer_type {
4675 	RB_TYPE_TLB_INV			= 0,
4676 	RB_TYPE_REMOTE_INTERRUPT	= 1,
4677 	RB_TYPE_TAILPTR			= 2
4678 };
4679 
4680 /**
4681  * amdgpu_ualink_npa_addr - Get reserved NPA address for ring buffer
4682  * @adev: amdgpu device pointer
4683  * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
4684  * @src_accel_id: Source accelerator ID
4685  * @dst_accel_id: Destination accelerator ID
4686  *
4687  * Return: NPA address for the specified ring buffer type and GPUs
4688  */
4689 static u64 amdgpu_ualink_npa_addr(struct amdgpu_device *adev, u32 type,
4690 				  u32 src_accel_id, u32 dst_accel_id)
4691 {
4692 	u32 addr_mode = ualink_addr_mode(adev);
4693 	u64 npa;
4694 
4695 	WARN_ON_ONCE(src_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
4696 		     dst_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
4697 		     type > RB_TYPE_TAILPTR ||
4698 		     (addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT &&
4699 		     addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS));
4700 
4701 	switch (addr_mode) {
4702 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT:
4703 		npa = (u64)src_accel_id << 41 | type << 12;
4704 		break;
4705 
4706 	case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS:
4707 	default:
4708 		if (type == RB_TYPE_TAILPTR)
4709 			npa = (u64)dst_accel_id << 13;
4710 		else
4711 			npa = (u64)src_accel_id << 13 | type << 12;
4712 
4713 		/*
4714 		 * In order to avoid address conflicts between source-identification
4715 		 * and source-aliasing mode, add 2MB to the buffer addresses in
4716 		 * source-aliasing mode. This way in a misconfigured cluster, a GPU
4717 		 * configured in the wrong address mode will access unmapped NPA addresses
4718 		 * rather than the wrong buffer mappings.
4719 		 */
4720 		npa += AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET;
4721 		break;
4722 	}
4723 
4724 	dev_dbg(adev->dev, "addr mode %d from accel %d to accel %d type %d NPA 0x%llx\n",
4725 		addr_mode, src_accel_id, dst_accel_id, type, npa);
4726 
4727 	return npa;
4728 }
4729 
4730 /**
4731  * amdgpu_ualink_gart_npa_addr - Get GART-mapped NPA address for ring buffer
4732  * @adev: amdgpu device pointer
4733  * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
4734  * @src_accel_id: Source accelerator ID
4735  * @dst_accel_id: Destination accelerator ID
4736  *
4737  * Computes the NPA address suitable for GART mapping by adding dst_accid_id to
4738  * NPA address to access remote by SDMA.
4739 
4740  * clears the source GPU ID bits from the NPA address which is set for
4741  * address-identification mode.
4742  *
4743  * Return: NPA address adjusted for GART mapping
4744  */
4745 
4746 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT		41
4747 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK		GENMASK_ULL(50, 41)
4748 
4749 static u64 amdgpu_ualink_gart_npa_addr(struct amdgpu_device *adev, u32 type,
4750 					u32 src_accel_id, u32 dst_accel_id)
4751 {
4752 	u64 npa;
4753 
4754 	npa = amdgpu_ualink_npa_addr(adev, type, src_accel_id, dst_accel_id);
4755 
4756 	/* wiping out source accelerator id */
4757 	npa &= ~AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK;
4758 
4759 	return npa | ((u64)dst_accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT);
4760 }
4761 
4762 /**
4763  * amdgpu_ualink_reserve_npa_vm_and_bos - Reserve the NPA VM page directory and BOs
4764  * @adev: amdgpu device pointer
4765  * @bos: array of buffer objects to reserve
4766  * @n_bos: number of entries in @bos
4767  * @exec: drm_exec context to initialize and use for locking
4768  * @interruptible: true to use the interruptible dma_resv_lock
4769  *
4770  * Initializes @exec and uses it to lock all BOs in @bos together with the
4771  * NPA VM page directory, retrying on contention. On failure, @exec is
4772  * finalized and the error code is returned.
4773  *
4774  * Returns: 0 on success, negative error code on failure.
4775  */
4776 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
4777 						struct amdgpu_bo *bos[], u32 n_bos,
4778 						struct drm_exec *exec,
4779 						bool interruptible)
4780 {
4781 	u32 flags = DRM_EXEC_IGNORE_DUPLICATES;
4782 	int i, r = 0;
4783 
4784 	if (interruptible)
4785 		flags |= DRM_EXEC_INTERRUPTIBLE_WAIT;
4786 
4787 	dev_dbg(adev->dev, "reserve NPA vm and %d bos\n", n_bos);
4788 
4789 	drm_exec_init(exec, flags, 0);
4790 
4791 	drm_exec_until_all_locked(exec) {
4792 		for (i = 0; i < n_bos; i++) {
4793 			r = drm_exec_lock_obj(exec, &bos[i]->tbo.base);
4794 			drm_exec_retry_on_contention(exec);
4795 			if (unlikely(r))
4796 				goto out;
4797 		}
4798 
4799 		r = amdgpu_vm_lock_pd(&adev->ualink.npa_vm, exec, 0);
4800 		drm_exec_retry_on_contention(exec);
4801 		if (unlikely(r))
4802 			goto out;
4803 	}
4804 
4805 out:
4806 	if (r)
4807 		drm_exec_fini(exec);
4808 	return r;
4809 
4810 }
4811 
4812 /**
4813  * amdgpu_ualink_unreserve_npa_vm_and_bos - Release the NPA VM page directory and BOs
4814  * @adev: amdgpu device pointer
4815  * @exec: drm_exec context previously initialized by
4816  *        amdgpu_ualink_reserve_npa_vm_and_bos()
4817  *
4818  * Finalizes @exec, releasing all locks on the NPA VM page directory and
4819  * the associated BOs acquired during reservation.
4820  */
4821 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
4822 						   struct drm_exec *exec)
4823 {
4824 	dev_dbg(adev->dev, "unreserve NPA vm and bos\n");
4825 	drm_exec_fini(exec);
4826 }
4827 
4828 /**
4829  * amdgpu_ualink_metadata_npa_unmapping - Tear down NPA address mappings
4830  * @adev: amdgpu device pointer
4831  *
4832  * Unmaps all NPA address mappings from the NPA VM for ring buffers,
4833  * write pointers, and read pointers. Waits for outstanding DMA fences
4834  * and flushes the TLB.
4835  */
4836 static void amdgpu_ualink_metadata_npa_unmapping(struct amdgpu_device *adev)
4837 {
4838 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4839 	u64 timeout = msecs_to_jiffies(2000);
4840 	u32 dst_accel_id = ualink_accel_id(adev);
4841 	struct amdgpu_bo *bos[2];
4842 	u32 n_bos;
4843 	struct dma_fence *fence;
4844 	struct drm_exec exec;
4845 	u32 rptr_size, rptr_size_in_pages;
4846 	u32 rb_size, rb_size_in_pages;
4847 	u64 pte_flags = adev->gmc.noretry_flags;
4848 	u64 npa;
4849 	int r;
4850 
4851 	if (!remote->ring_bo)
4852 		return;
4853 
4854 	rb_size = AMDGPU_UALINK_RB_SIZE;
4855 	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
4856 
4857 	bos[0] = remote->ring_bo;
4858 	n_bos = 1;
4859 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4860 		bos[1] = remote->rptr_bo;
4861 		n_bos = 2;
4862 	}
4863 
4864 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
4865 	if (unlikely(r))
4866 		return;
4867 
4868 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4869 		/* rptr npa mapping, up to allocated 2 pages npa address */
4870 		rptr_size = ualink_wb_size(adev);
4871 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
4872 		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
4873 
4874 		npa = remote->rptr_npa;
4875 
4876 		amdgpu_ualink_npa_vm_unmap_range(adev, remote->rptr_bo,
4877 						 pte_flags, 0, rptr_size_in_pages,
4878 						 npa >> AMDGPU_GPU_PAGE_SHIFT);
4879 		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
4880 
4881 		/*
4882 		 * unmap wptr, remote interrupt, shootdown ring.
4883 		 * wptr page is part of the single 2MB mapping for remote GPUs
4884 		 * interrupt and shootdown ring buffer
4885 		 */
4886 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
4887 					     0, dst_accel_id);
4888 		amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, pte_flags,
4889 						 0,
4890 						 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
4891 						 npa >> AMDGPU_GPU_PAGE_SHIFT);
4892 	} else {
4893 		u32 wptr_offset = 2 * rb_size * remote->num_accel;
4894 		u32 idx = 0;
4895 		u32 accel_id;
4896 
4897 		/* source identification mode */
4898 		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
4899 			if (accel_id == dst_accel_id)
4900 				continue;
4901 
4902 			/* remote interrupt ring npa mapping */
4903 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
4904 						     accel_id, dst_accel_id);
4905 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4906 							 pte_flags, idx * 2 * rb_size,
4907 							 rb_size_in_pages,
4908 							 npa >> AMDGPU_GPU_PAGE_SHIFT);
4909 
4910 			/* remote shootdown ring npa mapping */
4911 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
4912 						     dst_accel_id);
4913 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4914 							 pte_flags, (idx * 2 + 1) * rb_size,
4915 							 rb_size_in_pages,
4916 							 npa >> AMDGPU_GPU_PAGE_SHIFT);
4917 
4918 			/* wptr, rptr npa mapping */
4919 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
4920 						      dst_accel_id);
4921 			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
4922 						  pte_flags, wptr_offset + idx * PAGE_SIZE,
4923 						  1, npa >> AMDGPU_GPU_PAGE_SHIFT);
4924 			idx++;
4925 		}
4926 	}
4927 
4928 	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
4929 	if (r) {
4930 		dev_dbg(adev->dev, "failed %d to update directories\n", r);
4931 		goto out_unreserve;
4932 	}
4933 
4934 	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
4935 	if (fence) {
4936 		r = dma_fence_wait_timeout(fence, true, timeout);
4937 		dma_fence_put(fence);
4938 		if (r <= 0)
4939 			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
4940 	}
4941 
4942 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
4943 out_unreserve:
4944 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
4945 }
4946 
4947 /**
4948  * amdgpu_ualink_metadata_npa_mapping - Setup NPA address mapping in VM
4949  * @adev: amdgpu device pointer
4950  *
4951  * Maps NPA addresses to GPA for metadata ring buffers, write pointers,
4952  * on NPA VMID 15.
4953  *
4954  * Return: 0 on success, negative error code on failure
4955  */
4956 static int amdgpu_ualink_metadata_npa_mapping(struct amdgpu_device *adev)
4957 {
4958 	struct amdgpu_ualink_remote *remote = to_remote(adev);
4959 	u32 dst_accel_id = ualink_accel_id(adev);
4960 	u64 timeout = msecs_to_jiffies(2000);
4961 	struct amdgpu_bo *bos[2];
4962 	u32 n_bos;
4963 	struct dma_fence *fence;
4964 	struct drm_exec exec;
4965 	u32 rb_size, rb_size_in_pages;
4966 	u64 npa, npa_in_pages, pte_flags;
4967 	int r;
4968 
4969 	rb_size = AMDGPU_UALINK_RB_SIZE;
4970 	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
4971 
4972 	bos[0] = remote->ring_bo;
4973 	n_bos = 1;
4974 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4975 		bos[1] = remote->rptr_bo;
4976 		n_bos = 2;
4977 	}
4978 
4979 	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
4980 	if (unlikely(r))
4981 		return r;
4982 
4983 	pte_flags = amdgpu_ttm_tt_pte_flags(adev, remote->ring_bo->tbo.ttm,
4984 					    remote->ring_bo->tbo.resource);
4985 	dev_dbg(adev->dev, "init pte_flags 0x%llx\n", pte_flags);
4986 
4987 	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, remote->ring_bo,
4988 			      AMDGPU_VM_MTYPE_DEFAULT, &pte_flags);
4989 	dev_dbg(adev->dev, "after get coherent pte_flags 0x%llx\n", pte_flags);
4990 
4991 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
4992 		u32 rptr_size, rptr_size_in_pages;
4993 
4994 		/* rptr npa mapping, up to allocated 2 pages npa address */
4995 		rptr_size = ualink_wb_size(adev);
4996 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
4997 		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
4998 
4999 		r = amdgpu_ualink_npa_alloc_va(adev, &remote->rptr_mm_node,
5000 					       0, 0, 0,
5001 					       rptr_size_in_pages);
5002 		if (r)
5003 			goto out;
5004 
5005 		npa_in_pages = remote->rptr_mm_node.start;
5006 
5007 		dev_dbg(adev->dev, "source aliasing rptr alloc 0x%llx and map to npa vm\n",
5008 			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
5009 
5010 		r = amdgpu_ualink_npa_vm_map_range(adev, remote->rptr_bo, pte_flags, 0,
5011 						   rptr_size_in_pages,
5012 						   npa_in_pages);
5013 		if (r)
5014 			goto error_npa_mapping;
5015 
5016 		remote->rptr_npa = npa_in_pages << AMDGPU_GPU_PAGE_SHIFT;
5017 
5018 		/* wptr, remote interrupt, shootdown ring, single big 2MB mapping */
5019 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
5020 					     0, dst_accel_id);
5021 		r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5022 						   0,
5023 						   2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
5024 						   npa >> AMDGPU_GPU_PAGE_SHIFT);
5025 		if (r)
5026 			goto error_npa_mapping;
5027 
5028 	} else {
5029 		u32 wptr_offset = 2 * rb_size * remote->num_accel;
5030 		u32 accel_id, idx = 0;
5031 
5032 		/* Source identification mode */
5033 		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5034 			if (accel_id == dst_accel_id)
5035 				continue;
5036 
5037 			/* remote interrupt ring npa mapping */
5038 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
5039 						      accel_id, dst_accel_id);
5040 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5041 							   idx * 2 * rb_size,
5042 							   rb_size_in_pages,
5043 							   npa >> AMDGPU_GPU_PAGE_SHIFT);
5044 			if (r)
5045 				goto error_npa_mapping;
5046 
5047 			/* remote shootdown ring npa mapping */
5048 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
5049 						     dst_accel_id);
5050 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
5051 							   (idx * 2 + 1) * rb_size,
5052 							   rb_size_in_pages,
5053 							   npa >> AMDGPU_GPU_PAGE_SHIFT);
5054 			if (r)
5055 				goto error_npa_mapping;
5056 
5057 			/* wptr, rptr npa mapping */
5058 			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
5059 						     dst_accel_id);
5060 			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo,
5061 							   pte_flags,
5062 							   wptr_offset + idx * PAGE_SIZE,
5063 							   1, npa >> AMDGPU_GPU_PAGE_SHIFT);
5064 			if (r)
5065 				goto error_npa_mapping;
5066 
5067 			idx++;
5068 		}
5069 	}
5070 
5071 	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
5072 	if (r) {
5073 		dev_dbg(adev->dev, "failed %d to update directories\n", r);
5074 		goto error_npa_mapping;
5075 	}
5076 
5077 	/* TODO: only wait the last fence, then flush TLB */
5078 	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
5079 	if (fence) {
5080 		r = dma_fence_wait_timeout(fence, true, timeout);
5081 		dma_fence_put(fence);
5082 		if (r <= 0)
5083 			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
5084 	}
5085 
5086 	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
5087 
5088 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
5089 	return 0;
5090 
5091 error_npa_mapping:
5092 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
5093 		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
5094 	if (r)
5095 		dev_dbg(adev->dev, "failed %d to map NPA vm\n", r);
5096 
5097 out:
5098 	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
5099 
5100 	return r;
5101 }
5102 
5103 /**
5104  * amdgpu_ualink_sdma_entities_init - Initialize SDMA job scheduler entities
5105  * @adev: amdgpu device pointer
5106  *
5107  * Sets up SDMA job scheduler entities for remote interrupt and shootdown.
5108  * Each entity uses one SDMA scheduler/ring.
5109  *
5110  * Return: 0 on success, negative error code on failure
5111  */
5112 static int amdgpu_ualink_sdma_entities_init(struct amdgpu_device *adev)
5113 {
5114 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5115 	struct drm_gpu_scheduler *sched;
5116 	u32 accel_id;
5117 	int i = 0, r;
5118 
5119 	dev_dbg(adev->dev, "enter\n");
5120 
5121 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5122 		sched = &adev->sdma.instance[accel_id % adev->sdma.num_instances].ring.sched;
5123 
5124 		r = drm_sched_entity_init(&remote->peer[accel_id].entity, DRM_SCHED_PRIORITY_HIGH,
5125 					  &sched, 1, NULL);
5126 		if (r)
5127 			goto out_free;
5128 		i++;
5129 	}
5130 
5131 	remote->use_lsdma = true;
5132 
5133 	dev_dbg(adev->dev, "exit\n");
5134 	return 0;
5135 
5136 out_free:
5137 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5138 		if (i--)
5139 			drm_sched_entity_destroy(&remote->peer[accel_id].entity);
5140 		else
5141 			break;
5142 	}
5143 
5144 	return r;
5145 }
5146 
5147 static void amdgpu_ualink_sdma_entities_fini(struct amdgpu_device *adev)
5148 {
5149 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5150 	u32 accel_id;
5151 
5152 	dev_dbg(adev->dev, "enter\n");
5153 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX)
5154 		drm_sched_entity_destroy(&remote->peer[accel_id].entity);
5155 	dev_dbg(adev->dev, "exit\n");
5156 }
5157 
5158 /**
5159  * amdgpu_ualink_gart_map - Allocate GART entry and map NPA address
5160  * @adev: amdgpu device pointer
5161  * @npages: Number of pages to map
5162  * @npa: NPA address to map
5163  * @mm_node: DRM memory manager node for GART allocation
5164  * @pte_flags: the GART mapping flags
5165  *
5166  * Allocates GART entries and sets up NPA address mapping without TTM BO.
5167  *
5168  * Return: 0 on success, negative error code on failure
5169  */
5170 static int amdgpu_ualink_gart_map(struct amdgpu_device *adev, u64 npages,
5171 				   u64 npa, struct drm_mm_node *mm_node,
5172 				   u64 pte_flags)
5173 {
5174 	struct ttm_resource_manager *man =
5175 				ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
5176 	struct amdgpu_gtt_mgr *mgr =
5177 				container_of(man, struct amdgpu_gtt_mgr, manager);
5178 	dma_addr_t *dma_addr;
5179 	int i, r;
5180 
5181 	dev_dbg(adev->dev, "npa 0x%llx npages 0x%llx\n", npa, npages);
5182 
5183 	if (npages > 1) {
5184 		dma_addr = kmalloc_objs(*dma_addr, npages);
5185 		if (!dma_addr)
5186 			return -ENOMEM;
5187 
5188 		for (i = 0; i < npages; i++)
5189 			dma_addr[i] = npa + i * AMDGPU_GPU_PAGE_SIZE;
5190 	} else {
5191 		dma_addr = (dma_addr_t *)&npa;
5192 	}
5193 
5194 	r = amdgpu_gtt_mgr_alloc_entries(mgr, mm_node, npages, DRM_MM_INSERT_BEST);
5195 	if (r)
5196 		goto out;
5197 	amdgpu_gart_bind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, dma_addr,
5198 			 pte_flags);
5199 
5200 out:
5201 	dev_dbg(adev->dev, "npa 0x%llx mapped to 0x%llx npages 0x%llx r=%d\n",
5202 		npa, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, r);
5203 
5204 	if (npages > 1)
5205 		kfree(dma_addr);
5206 	return r;
5207 }
5208 
5209 /**
5210  * amdgpu_ualink_gart_unmap - Unmap and free GART entry
5211  * @adev: amdgpu device pointer
5212  * @npages: Number of pages to unmap
5213  * @mm_node: DRM memory manager node for GART allocation
5214  *
5215  * Unbinds GART mapping and frees allocated entries.
5216  */
5217 static void amdgpu_ualink_gart_unmap(struct amdgpu_device *adev, u64 npages,
5218 				      struct drm_mm_node *mm_node)
5219 {
5220 	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
5221 	struct amdgpu_gtt_mgr *mgr =
5222 				container_of(man, struct amdgpu_gtt_mgr, manager);
5223 
5224 	dev_dbg(adev->dev, "0x%llx npages 0x%llx\n", mm_node->start << AMDGPU_GPU_PAGE_SHIFT,
5225 		npages);
5226 
5227 	if (!drm_mm_node_allocated(mm_node))
5228 		return;
5229 	amdgpu_gart_unbind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages);
5230 	amdgpu_gtt_mgr_free_entries(mgr, mm_node);
5231 }
5232 
5233 /**
5234  * amdgpu_ualink_metadata_fini - Clean up ualink metadata structures
5235  * @adev: amdgpu device pointer
5236  *
5237  * Unmaps NPA addresses and frees all allocated buffers for metadata,
5238  * ring buffers, and pointers.
5239  */
5240 static void amdgpu_ualink_metadata_fini(struct amdgpu_device *adev)
5241 {
5242 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5243 
5244 	dev_dbg(adev->dev, "accel_id %u\n", ualink_accel_id(adev));
5245 	amdgpu_bo_free_kernel(&remote->rptr_bo, &remote->rptr_gpu_addr,
5246 				      &remote->rptr_cpu_addr);
5247 	amdgpu_bo_free_kernel(&remote->ring_bo, &remote->rb_gpu_addr,
5248 			      &remote->rb_cpu_addr);
5249 	amdgpu_bo_free_kernel(&remote->metadata_bo, &remote->metadata_gpu_addr,
5250 			      &remote->metadata_cpu_addr);
5251 }
5252 
5253 /**
5254  * amdgpu_ualink_metadata_init - Initialize ualink metadata structures
5255  * @adev: amdgpu device pointer
5256  *
5257  * Allocates and initializes metadata structures, ring buffers, and write/read
5258  * pointers for multi-GPU ualink. Communicates with firmware to load metadata.
5259  *
5260  * Return: 0 on success, negative error code on failure
5261  */
5262 static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev)
5263 {
5264 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5265 	struct amdgpu_ualink_metadata *metadata;
5266 	u32 size, rb_size, wptr_size, rptr_size, metadata_size;
5267 	u64 rb_gpu_addr, wptr_gpu_addr;
5268 	u32 status, accel_id;
5269 	int r;
5270 
5271 	bitmap_copy(remote->active_accel_bits,
5272 		    adev->ualink.info->vpod.active_accel_bits,
5273 		    AMDGPU_UALINK_ACCEL_MAX);
5274 	dev_dbg(adev->dev, "%d active accelerators config in vpod\n",
5275 		bitmap_weight(remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX));
5276 
5277 	/*
5278 	 * allocate metadata entries and ring buffer for all remote GPUs,
5279 	 * to get 2MB page ring buffer NPA mapping for remote access.
5280 	 */
5281 	remote->num_accel = AMDGPU_UALINK_ACCEL_MAX;
5282 
5283 	status = adev->ualink.msg_ctl->check_status(adev);
5284 	if (status != AMDGPU_NHT_FW_ST_PREINIT &&
5285 	    status != AMDGPU_NHT_FW_ST_HALT) {
5286 		dev_dbg(adev->dev, "fw status 0x%x not preinit or halt\n", status);
5287 		return -ENODEV;
5288 	}
5289 
5290 	dev_dbg(adev->dev, "accel_id %u addr_mode %d fw status 0x%x\n",
5291 		ualink_accel_id(adev), ualink_addr_mode(adev), status);
5292 
5293 	/* Alloc metadata structure for all GPUs */
5294 	metadata_size = sizeof(struct amdgpu_ualink_metadata) * AMDGPU_UALINK_ACCEL_MAX;
5295 	metadata_size = AMDGPU_GPU_PAGE_ALIGN(metadata_size);
5296 
5297 	dev_dbg(adev->dev, "metadata size 0x%x\n", metadata_size);
5298 
5299 	/* pinned system memory */
5300 	r = amdgpu_bo_create_kernel(adev, metadata_size, PAGE_SIZE,
5301 				    AMDGPU_GEM_DOMAIN_GTT,
5302 				    &remote->metadata_bo, &remote->metadata_gpu_addr,
5303 				    &remote->metadata_cpu_addr);
5304 	if (r)
5305 		goto out;
5306 
5307 	memset(remote->metadata_cpu_addr, 0, metadata_size);
5308 	metadata = remote->metadata_cpu_addr;
5309 
5310 	dev_dbg(adev->dev, "metadata gpu address 0x%llx\n", remote->metadata_gpu_addr);
5311 
5312 	/* Allocate ring buffers, wptr for remote interrupt and shootdown */
5313 	rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
5314 	wptr_size = ualink_wptr_size(adev);
5315 	size = (rb_size + wptr_size) * remote->num_accel;
5316 	size = AMDGPU_GPU_PAGE_ALIGN(size);
5317 
5318 	dev_dbg(adev->dev, "rb_size 0x%x wptr_size 0x%x total alloc size 0x%x\n",
5319 		rb_size, wptr_size, size);
5320 
5321 	/* pinned VRAM */
5322 	r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
5323 				    AMDGPU_GEM_DOMAIN_VRAM,
5324 				    &remote->ring_bo,
5325 				    &remote->rb_gpu_addr,
5326 				    &remote->rb_cpu_addr);
5327 	if (r)
5328 		goto out;
5329 
5330 	memset(remote->rb_cpu_addr, 0, size);
5331 
5332 	dev_dbg(adev->dev, "rb gpu addr 0x%llx cpu addr 0x%p vram_start 0x%llx vram_base 0x%llx\n",
5333 		remote->rb_gpu_addr, remote->rb_cpu_addr, adev->gmc.vram_start,
5334 		adev->vm_manager.vram_base_offset);
5335 
5336 	/*
5337 	 * address aliasing mode, shared wptr pagee left is not enough for wb data,
5338 	 * wb status and rptr, alloc another BO, and then alloc npa address and map
5339 	 * to npa vm, for remote to access.
5340 	 *
5341 	 * No gart mapping required for wb data, status and rptr because this is
5342 	 * updated by firmware.
5343 	 */
5344 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
5345 		rptr_size = AMDGPU_GPU_PAGE_ALIGN(ualink_wb_size(adev) * remote->num_accel);
5346 		dev_dbg(adev->dev, "source aliasing mode rptr_size 0x%x\n", rptr_size);
5347 
5348 		/* pinned VRAM */
5349 		r = amdgpu_bo_create_kernel(adev, rptr_size, PAGE_SIZE,
5350 					    AMDGPU_GEM_DOMAIN_VRAM,
5351 					    &remote->rptr_bo,
5352 					    &remote->rptr_gpu_addr,
5353 					    &remote->rptr_cpu_addr);
5354 		if (r)
5355 			goto out;
5356 
5357 		memset(remote->rptr_cpu_addr, 0, rptr_size);
5358 
5359 		dev_dbg(adev->dev, "source aliasing rptr gpu addr 0x%llx cpu addr 0x%p\n",
5360 			remote->rptr_gpu_addr, remote->rptr_cpu_addr);
5361 	}
5362 
5363 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
5364 		wptr_gpu_addr = remote->rb_gpu_addr + rb_size * ualink_accel_id(adev);
5365 
5366 	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
5367 		dev_dbg(adev->dev, "init for accel_id %u\n", accel_id);
5368 
5369 		if (accel_id == ualink_accel_id(adev))
5370 			continue;
5371 
5372 		rb_gpu_addr = remote->rb_gpu_addr + rb_size * accel_id;
5373 
5374 		if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
5375 			wptr_gpu_addr = remote->rb_gpu_addr + rb_size * remote->num_accel;
5376 			wptr_gpu_addr += wptr_size * accel_id;
5377 		}
5378 
5379 		metadata[accel_id].header = AMDGPU_UALINK_METADATA_HEADER;
5380 		metadata[accel_id].rb_size = fls(AMDGPU_UALINK_RB_SIZE / 64) - 1;
5381 
5382 		/*
5383 		 * remote shootdown type is 0, remote command type is 1
5384 		 * with 2MB ring buffer mapping, remote shootdown ring NPA is before interrupt
5385 		 */
5386 		metadata[accel_id].ri_rb = rb_gpu_addr + AMDGPU_UALINK_RB_SIZE;
5387 		metadata[accel_id].tlb_inv_rb = rb_gpu_addr;
5388 
5389 		metadata[accel_id].tailptr_ri = wptr_gpu_addr +
5390 						ualink_wptr_offset(adev, accel_id);
5391 		metadata[accel_id].tailptr_tlb_inv = wptr_gpu_addr +
5392 						     ualink_tlb_wptr_offset(adev, accel_id);
5393 
5394 		dev_dbg(adev->dev, "init from accel_id %u to accel_id %u, rb_size 0x%x\n",
5395 			accel_id, ualink_accel_id(adev), metadata[accel_id].rb_size);
5396 		dev_dbg(adev->dev, "rb 0x%llx tlb rb 0x%llx\n",
5397 			metadata[accel_id].ri_rb, metadata[accel_id].tlb_inv_rb);
5398 		dev_dbg(adev->dev, "rb wptr at 0x%llx tlb wptr at 0x%llx\n",
5399 			metadata[accel_id].tailptr_ri, metadata[accel_id].tailptr_tlb_inv);
5400 	}
5401 
5402 	r = adev->ualink.msg_ctl->send_metadata(adev,
5403 						    remote->metadata_gpu_addr,
5404 						    AMDGPU_UALINK_ACCEL_MAX << 8);
5405 
5406 out:
5407 	if (r)
5408 		amdgpu_ualink_metadata_fini(adev);
5409 
5410 	dev_dbg(adev->dev, "ret 0x%x\n", r);
5411 	return r;
5412 }
5413 
5414 /*
5415  * nHT Firmware Error code
5416  *
5417  * Success				0x0
5418  * FIFO overflow			0x1
5419  * Invalid RB command			0x2
5420  * Timeout				0x3
5421  * Invalid metadata entry		0x4
5422  * RB overflow				0x5
5423  * Invalid wptr address			0x6
5424  * Interrupt cookie send failure	0x7
5425  * Invalid RB address			0x8
5426  * Invalid writeback address		0x9
5427  */
5428 static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status)
5429 {
5430 	const char *fw_err_code_msg[] = {
5431 		"Unknown FW error status",
5432 		"FIFO overflow",		/* error code 1 */
5433 		"Invalid RB command",
5434 		"Timeout",
5435 		"Invalid metadata entry",
5436 		"RB overflow",
5437 		"Invalid wptr address",
5438 		"Interrupt cookie send failure",
5439 		"Invalid RB address",
5440 		"Invalid writeback address"	/* error code 9 */
5441 		};
5442 
5443 	if (status >= ARRAY_SIZE(fw_err_code_msg))
5444 		status = 0;
5445 
5446 	dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]);
5447 	return;
5448 }
5449 
5450 /**
5451  * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion
5452  * @remote_accel_id: remote accelator id to wait for reply
5453  * @adev: amdgpu device pointer
5454  * @wb_cpu: CPU virtual address of writeback buffer
5455  * @seq: Sequence number to wait for
5456  *
5457  * Polls the writeback buffer waiting for the sequence number to reach or
5458  * exceed the expected value. The writeback buffer contains status and data
5459  * fields updated by firmware to indicate completion and error conditions.
5460  *
5461  * Return: 0 on success
5462  *         -ETIME on timeout
5463  *         -ECOMM on firmware return error status
5464  */
5465 static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev,
5466 					      u32 remote_accel_id,
5467 					      struct amdgpu_ualink_wb *wb_cpu,
5468 					      u32 seq)
5469 {
5470 	/* 2 seconds timeout, long enough for FW to reply */
5471 	ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC);
5472 	u32 status, data;
5473 	u64 rptr;
5474 
5475 	while (true) {
5476 		data = READ_ONCE(wb_cpu->data);
5477 		if (data >= seq) {
5478 			status = READ_ONCE(wb_cpu->status);
5479 			rptr = READ_ONCE(wb_cpu->rptr);
5480 			break;
5481 		}
5482 
5483 		if (ktime_after(ktime_get(), timeout)) {
5484 			dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id);
5485 			return -ETIME;
5486 		}
5487 
5488 		usleep_range(10, 50);
5489 	}
5490 
5491 	if (status) {
5492 		dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n",
5493 			remote_accel_id, status, data, seq, rptr);
5494 
5495 		amdgpu_ualink_remote_error(adev, status);
5496 		return -ECOMM;
5497 	}
5498 
5499 	dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n",
5500 		remote_accel_id, seq, data, rptr);
5501 	return 0;
5502 }
5503 
5504 #define AMDGPU_UALINK_REMOTE_OP_TLB_INV		0x1
5505 #define AMDGPU_UALINK_REMOTE_OP_INT		0x2
5506 #define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB	0x3
5507 
5508 static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data,
5509 					 u32 dw0,  u32 dw1, u32 dw2, u32 dw3)
5510 {
5511 	u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT;
5512 	u32 *cpu_addr = *cpu_addr_p;
5513 	u32 flush_type = dw0;
5514 	u32 size_in_pages = dw3;
5515 	u32 cmd;
5516 
5517 	cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV;
5518 	cmd |= 1 << 28;	/* headptr update */
5519 	cmd |= 1 << 29;	/* wb enable */
5520 
5521 	*cpu_addr++ = cmd;
5522 	*cpu_addr++ = wb_data;
5523 	*cpu_addr++ = 0;
5524 	*cpu_addr++ = 0;
5525 	*cpu_addr++ = 0;
5526 	*cpu_addr++ = 0;
5527 	*cpu_addr++ = 0; /* metadata */
5528 	*cpu_addr++ = 0; /* metadata */
5529 
5530 	pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages);
5531 
5532 	/* Always shootdown everything with full address size s-field coding */
5533 	addr = GENMASK_ULL(51, 11);
5534 
5535 	*cpu_addr++ = lower_32_bits(addr) | flush_type;
5536 	*cpu_addr++ = upper_32_bits(addr);
5537 
5538 	*cpu_addr++ = 0;
5539 	*cpu_addr++ = 0;
5540 	*cpu_addr++ = 0;
5541 	*cpu_addr++ = 0;
5542 	*cpu_addr++ = 0;
5543 	*cpu_addr++ = 0;
5544 	*cpu_addr_p = cpu_addr;
5545 }
5546 
5547 static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data,
5548 					 u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5549 {
5550 	u32 *cpu_addr = *cpu_addr_p;
5551 	u32 cmd;
5552 
5553 	cmd = AMDGPU_UALINK_REMOTE_OP_INT;
5554 	cmd |= 1 << 28;	/* headptr_update */
5555 	cmd |= 1 << 29; /* wb enable */
5556 
5557 	*cpu_addr++ = cmd;
5558 	*cpu_addr++ = wb_data;
5559 	*cpu_addr++ = 0;
5560 	*cpu_addr++ = 0;
5561 	*cpu_addr++ = 0;
5562 	*cpu_addr++ = 0;
5563 	*cpu_addr++ = 0; /* metadata */
5564 	*cpu_addr++ = 0; /* metadata */
5565 	/* updated by f/w, overwrite PASID with GPU ID */
5566 	*cpu_addr++ = 0;
5567 	*cpu_addr++ = 0;
5568 	*cpu_addr++ = 0;
5569 	*cpu_addr++ = 0;
5570 	/* IH ContextID 4 dwords */
5571 	*cpu_addr++ = dw0;
5572 	*cpu_addr++ = dw1;
5573 	*cpu_addr++ = dw2;
5574 	*cpu_addr++ = dw3;
5575 	*cpu_addr_p = cpu_addr;
5576 }
5577 
5578 static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data,
5579 					      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5580 {
5581 	u32 *cpu_addr = *cpu_addr_p;
5582 	u32 cmd;
5583 
5584 	cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB;
5585 	cmd |= 1 << 28;	/* headptr update */
5586 	cmd |= 1 << 29;	/* wb enable */
5587 
5588 	*cpu_addr++ = cmd;
5589 	*cpu_addr++ = wb_data;
5590 	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb_npa */
5591 	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb_npa */
5592 	*cpu_addr++ = 0;
5593 	*cpu_addr++ = 0;
5594 	*cpu_addr++ = 0; /* metadata */
5595 	*cpu_addr++ = 0; /* metadata */
5596 	*cpu_addr++ = 0;
5597 	*cpu_addr++ = 0;
5598 	*cpu_addr++ = 0;
5599 	*cpu_addr++ = 0;
5600 	*cpu_addr++ = 0;
5601 	*cpu_addr++ = 0;
5602 	*cpu_addr++ = 0;
5603 	*cpu_addr++ = 0;
5604 	*cpu_addr_p = cpu_addr;
5605 }
5606 
5607 /*
5608  * Function proto to generate packet using above different emit_ functions
5609  */
5610 typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0,
5611 				   u32 dw1, u32 dw2, u32 dw3);
5612 
5613 /*
5614  * Refresh rptr from the writeback and check the ring has room for one more
5615  * command. The caller must hold the peer lock.
5616  *
5617  * Return 0 on success or a negative error code.
5618  */
5619 static int amdgpu_ualink_ring_check_space(struct amdgpu_device *adev,
5620 					  u32 remote_accel_id,
5621 					  struct amdgpu_ualink_ring *ring,
5622 					  struct amdgpu_ualink_wb *wb_cpu)
5623 {
5624 	ring->rptr = READ_ONCE(wb_cpu->rptr);
5625 
5626 	if (WARN_ON_ONCE(ring->rptr > ring->wptr)) {
5627 		dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n",
5628 			remote_accel_id, ring->wptr, ring->rptr);
5629 		return -EFAULT;
5630 	}
5631 
5632 	if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) {
5633 		dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n",
5634 			remote_accel_id, ring->wptr, ring->rptr);
5635 		return -ENOSPC;
5636 	}
5637 
5638 	return 0;
5639 }
5640 
5641 static int amdgpu_ualink_send_command(struct amdgpu_device *adev,
5642 				      u32 remote_accel_id,
5643 				      struct amdgpu_ualink_ring *ring,
5644 				      struct amdgpu_ualink_wb *wb_cpu,
5645 				      ualink_emit_packet emit_func,
5646 				      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
5647 {
5648 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5649 	struct amdgpu_ualink_peer *peer;
5650 	struct amdgpu_ring *sdma_ring;
5651 	struct dma_fence *fence;
5652 	struct amdgpu_job *job;
5653 	struct amdgpu_ib *ib;
5654 	u32 seq, ndw, ndw_copy_cmd, rem;
5655 	u64 doorbell_npa_gart;
5656 	u32 *src_cpu;
5657 	u64 src;
5658 	int r;
5659 
5660 	/* Refuse sends unless active */
5661 	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
5662 		return -ESHUTDOWN;
5663 
5664 	peer = &remote->peer[remote_accel_id];
5665 
5666 	r = amdgpu_ualink_ring_check_space(adev, remote_accel_id, ring, wb_cpu);
5667 	if (r)
5668 		goto out;
5669 
5670 	/*
5671 	 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell
5672 	 *
5673 	 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to
5674 	 * ensure the ordering of 3 sdma copy commands execution.
5675 	 */
5676 
5677 	/* buffer after sdma commands, starting at 8 dwords boundary */
5678 	ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8);
5679 
5680 	 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */
5681 	ndw = ndw_copy_cmd + 16 + 2 + 1;
5682 	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
5683 				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
5684 				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
5685 	if (r)
5686 		goto out;
5687 
5688 	ib = &job->ibs[0];
5689 	src = ib->gpu_addr + ndw_copy_cmd * 4;
5690 	src_cpu = ib->ptr + ndw_copy_cmd;
5691 
5692 	seq = ++ring->seq;
5693 
5694 	div_u64_rem(ring->wptr, ring->rb_size, &rem);
5695 	dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n",
5696 		src, ring->rb_npa_gart + rem * 64,
5697 		ring->wptr_npa_gart, ring->doorbell_npa_gart, seq);
5698 
5699 	dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr);
5700 
5701 	/* remote command packet */
5702 	emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3);
5703 
5704 	/* wptr value */
5705 	*src_cpu++ = lower_32_bits(ring->wptr + 1);
5706 	*src_cpu++ = upper_32_bits(ring->wptr + 1);
5707 
5708 	/* doorbell value */
5709 	*src_cpu++ = lower_32_bits(ring->wptr + 1);
5710 
5711 	/*
5712 	 * Add an offset to the doorbell address that cycles through different
5713 	 * values of bit 11:8 to use different DXS ports.
5714 	 */
5715 	peer->dxs_port = (peer->dxs_port + 1) & 0xF;
5716 	doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8);
5717 	dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n",
5718 		peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart);
5719 
5720 	dev_dbg(adev->dev, "use %s\n", remote->use_lsdma ? "lsdma" : "sdma");
5721 
5722 	if (remote->use_lsdma) {
5723 		div_u64_rem(ring->wptr, ring->rb_size, &rem);
5724 		r = amdgpu_lsdma_copy_mem(adev, src,
5725 					  ring->rb_npa_gart + rem * 64,
5726 					  64);
5727 		if (!r)
5728 			r = amdgpu_lsdma_copy_mem(adev, src + 64,
5729 						  ring->wptr_npa_gart, 8);
5730 		if (!r)
5731 			r = amdgpu_lsdma_copy_mem(adev, src + 72,
5732 						  doorbell_npa_gart, 4);
5733 
5734 		amdgpu_job_free(job);
5735 
5736 		/* Copy failed: command never reached the remote, so no
5737 		 * writeback will arrive. Return now instead of polling the
5738 		 * completion for the full remote timeout.
5739 		 */
5740 		if (r) {
5741 			dev_dbg(adev->dev,
5742 				"remote %u lsdma copy failed (r %d), skip completion wait\n",
5743 				remote_accel_id, r);
5744 			return r;
5745 		}
5746 
5747 		goto out_wait_complete;
5748 	}
5749 
5750 	sdma_ring = &adev->sdma.instance[0].ring;
5751 
5752 	/*
5753 	 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands
5754 	 * execution doesn't overlap, doorbell update is after data and wptr update
5755 	 * finished.
5756 	 */
5757 	div_u64_rem(ring->wptr, ring->rb_size, &rem);
5758 	amdgpu_emit_copy_buffer(adev, ib, src,
5759 				ring->rb_npa_gart + rem * 64,
5760 				64, 0);
5761 
5762 	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
5763 
5764 	amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0);
5765 
5766 	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
5767 
5768 	amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0);
5769 
5770 	amdgpu_ring_pad_ib(sdma_ring, ib);
5771 	WARN_ON(ib->length_dw > ndw_copy_cmd);
5772 
5773 	fence = amdgpu_job_submit(job);
5774 	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
5775 	dma_fence_put(fence);
5776 	if (r <= 0) {
5777 		dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
5778 			remote_accel_id, r);
5779 		if (r == 0)
5780 			r = -ETIME;
5781 		goto out;
5782 	}
5783 
5784 out_wait_complete:
5785 	/*
5786 	 * poll remote completion writeback data
5787 	 */
5788 	r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq);
5789 
5790 	/*
5791 	 * Advance wptr unless the remote timed out, even if the FW returned an
5792 	 * error status. On timeout the caller resets the connection, which
5793 	 * resyncs wptr from the remote in amdgpu_ualink_setup_connection.
5794 	 */
5795 	if (r != -ETIME)
5796 		ring->wptr++;
5797 
5798 out:
5799 	dev_dbg(adev->dev, "ret r = %d\n", r);
5800 	return r;
5801 }
5802 
5803 static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
5804 				      u32 remote_accel_id,
5805 				      struct amdgpu_ualink_wb **wb_cpu,
5806 				      u64 *wb_npa, u32 type)
5807 {
5808 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5809 	u32 accel_id = ualink_accel_id(adev);
5810 	u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
5811 	uintptr_t wb;
5812 	u64 npa;
5813 	u32 offset = 0;
5814 
5815 	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
5816 		wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel;
5817 		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR,
5818 					     remote_accel_id, accel_id);
5819 	} else {
5820 		wb = (uintptr_t)remote->rptr_cpu_addr;
5821 		npa = remote->rptr_npa;
5822 		npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT;
5823 	}
5824 
5825 	if (type == RB_TYPE_TLB_INV)
5826 		offset = ualink_tlb_wb_offset(adev, remote_accel_id);
5827 	else if (type == RB_TYPE_REMOTE_INTERRUPT)
5828 		offset = ualink_wb_offset(adev, remote_accel_id);
5829 
5830 	*wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset);
5831 	if (wb_npa)
5832 		*wb_npa = npa + offset;
5833 
5834 	dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id,
5835 		remote_accel_id, npa + offset);
5836 }
5837 
5838 struct amdgpu_ualink_wptr_fence_cb {
5839 	struct dma_fence_cb base;
5840 	u64 *wptr_cpu;
5841 	u64 result;
5842 };
5843 
5844 static void amdgpu_ualink_read_wptr_fence_cb(struct dma_fence *fence,
5845 					     struct dma_fence_cb *cb)
5846 {
5847 	struct amdgpu_ualink_wptr_fence_cb *wptr_cb;
5848 
5849 	wptr_cb = container_of(cb, typeof(*wptr_cb), base);
5850 	wptr_cb->result = *wptr_cb->wptr_cpu;
5851 }
5852 
5853 static int amdgpu_ualink_read_remote_wptr(struct amdgpu_device *adev,
5854 					  u32 remote_accel_id,
5855 					  struct amdgpu_ualink_ring *ring,
5856 					  u64 *wptr)
5857 {
5858 	struct amdgpu_ualink_remote *remote = to_remote(adev);
5859 	struct amdgpu_ualink_peer *peer = &remote->peer[remote_accel_id];
5860 	struct amdgpu_ualink_wptr_fence_cb wptr_cb = { };
5861 	struct amdgpu_ring *sdma_ring;
5862 	struct dma_fence *fence;
5863 	struct amdgpu_job *job;
5864 	struct amdgpu_ib *ib;
5865 	u32 ndw, ndw_copy_cmd;
5866 	u64 wptr_gpu, *wptr_cpu;
5867 	int r;
5868 
5869 	*wptr = 0;
5870 
5871 	/* Refuse remote access unless active */
5872 	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
5873 		return -ESHUTDOWN;
5874 
5875 	/* 1 sdma copy command to read wptr */
5876 	ndw_copy_cmd = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
5877 
5878 	/* wptr 2 dwords */
5879 	ndw = ndw_copy_cmd + 2;
5880 	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
5881 				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
5882 				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
5883 	if (r)
5884 		return r;
5885 
5886 	ib = &job->ibs[0];
5887 	wptr_gpu = ib->gpu_addr + ndw_copy_cmd * 4;
5888 	wptr_cpu = (u64 *)(ib->ptr + ndw_copy_cmd);
5889 
5890 	dev_dbg(adev->dev, "read wptr from remote %u npa gart wptr 0x%llx use %s\n",
5891 		remote_accel_id, ring->wptr_npa_gart,
5892 		remote->use_lsdma ? "lsdma" : "sdma");
5893 
5894 	if (remote->use_lsdma) {
5895 		r = amdgpu_lsdma_copy_mem(adev, ring->wptr_npa_gart, wptr_gpu, 8);
5896 		if (!r)
5897 			*wptr = *wptr_cpu;
5898 		amdgpu_job_free(job);
5899 		goto out;
5900 	}
5901 
5902 	wptr_cb.wptr_cpu = wptr_cpu;
5903 
5904 	amdgpu_emit_copy_buffer(adev, ib, ring->wptr_npa_gart, wptr_gpu, 8, 0);
5905 
5906 	sdma_ring = &adev->sdma.instance[0].ring;
5907 	amdgpu_ring_pad_ib(sdma_ring, ib);
5908 	WARN_ON(ib->length_dw > ndw_copy_cmd);
5909 
5910 	fence = amdgpu_job_submit(job);
5911 
5912 	r = dma_fence_add_callback(fence, &wptr_cb.base,
5913 				   amdgpu_ualink_read_wptr_fence_cb);
5914 	if (r == -ENOENT) {
5915 		/* Fence already signaled; the callback won't run, do it inline. */
5916 		amdgpu_ualink_read_wptr_fence_cb(fence, &wptr_cb.base);
5917 		r = 0;
5918 		goto out_put_fence;
5919 	} else if (r) {
5920 		/* -EINVAL: NULL fence or func, not expected here. */
5921 		goto out_put_fence;
5922 	}
5923 
5924 	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
5925 	if (r > 0) {
5926 		r = 0;	/* read wptr successfully */
5927 		goto out_put_fence;
5928 	}
5929 
5930 	dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
5931 		remote_accel_id, r);
5932 
5933 	if (r == 0)
5934 		r = -ETIME;
5935 
5936 	/*
5937 	 * Timed out: the callback is still armed and wptr_cb is on our stack.
5938 	 * Cancel it so it can't fire after we return and write into the freed
5939 	 * stack frame.
5940 	 */
5941 	dma_fence_remove_callback(fence, &wptr_cb.base);
5942 
5943 out_put_fence:
5944 	if (!r)
5945 		*wptr = wptr_cb.result;
5946 	dma_fence_put(fence);
5947 
5948 out:
5949 	dev_dbg(adev->dev, "remote %u wptr 0x%llx return %d\n",
5950 		remote_accel_id, *wptr, r);
5951 	return r;
5952 }
5953 
5954 /**
5955  * amdgpu_ualink_ring_resync_ptrs - sync local ring pointers from remote
5956  * @adev: amdgpu device pointer
5957  * @remote_accel_id: remote accelerator ID
5958  *
5959  * After a reboot on either side the two ends disagree on wptr. Read the remote
5960  * wptr of both peer rings and adopt it as the local wptr and rptr.
5961  *
5962  * Return 0 if both rings already match, 1 if they were resynced, or a negative
5963  * error code if a remote read failed, leaving the rings untouched.
5964  */
5965 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
5966 					  u32 remote_accel_id)
5967 {
5968 	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
5969 	struct amdgpu_ualink_wb *int_wb, *tlb_wb;
5970 	u64 int_wptr, tlb_wptr;
5971 	int r;
5972 
5973 	/* ring->wptr and ring->rptr are owned by the send path. */
5974 	guard(mutex)(&peer->lock);
5975 
5976 	/* Read both before updating either, so a failure changes nothing. */
5977 	r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
5978 					   &peer->interrupt, &int_wptr);
5979 	if (!r)
5980 		r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
5981 						   &peer->shootdown, &tlb_wptr);
5982 	if (r) {
5983 		dev_dbg(adev->dev, "accel_id %u read remote %u wptr failed %d\n",
5984 			ualink_accel_id(adev), remote_accel_id, r);
5985 		return r;
5986 	}
5987 
5988 	if (peer->interrupt.wptr == int_wptr && peer->shootdown.wptr == tlb_wptr) {
5989 		dev_dbg(adev->dev, "accel_id %u wptr 0x%llx 0x%llx is same as remote %u\n",
5990 			ualink_accel_id(adev), int_wptr, tlb_wptr, remote_accel_id);
5991 		return 0;
5992 	}
5993 
5994 	dev_dbg(adev->dev,
5995 		"accel_id %u wptr 0x%llx 0x%llx sync to remote %u wptr 0x%llx 0x%llx\n",
5996 		ualink_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr,
5997 		remote_accel_id, int_wptr, tlb_wptr);
5998 
5999 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL,
6000 				  RB_TYPE_REMOTE_INTERRUPT);
6001 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL,
6002 				  RB_TYPE_TLB_INV);
6003 
6004 	peer->interrupt.wptr = int_wptr;
6005 	peer->interrupt.rptr = int_wptr;
6006 	peer->shootdown.wptr = tlb_wptr;
6007 	peer->shootdown.rptr = tlb_wptr;
6008 
6009 	/* Drop the stale writeback rptr, it disagrees with the resynced wptr. */
6010 	WRITE_ONCE(int_wb->rptr, int_wptr);
6011 	WRITE_ONCE(tlb_wb->rptr, tlb_wptr);
6012 	return 1;
6013 }
6014 
6015 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
6016 					   u32 remote_accel_id)
6017 {
6018 	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
6019 
6020 	dev_dbg(adev->dev, "remote accel_id %u\n", remote_accel_id);
6021 
6022 	guard(mutex)(&peer->lock);
6023 
6024 	peer->interrupt.ready = false;
6025 	peer->shootdown.ready = false;
6026 }
6027 
6028 static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev,
6029 					   u32 remote_accel_id, u32 ring_type)
6030 {
6031 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6032 	struct amdgpu_ualink_peer *peer;
6033 	struct amdgpu_ualink_ring *ring;
6034 	struct amdgpu_ualink_wb *wb_cpu;
6035 	u64 wb_npa;
6036 	int r;
6037 
6038 	peer = &remote->peer[remote_accel_id];
6039 	if (ring_type == RB_TYPE_REMOTE_INTERRUPT)
6040 		ring = &peer->interrupt;
6041 	else if (ring_type == RB_TYPE_TLB_INV)
6042 		ring = &peer->shootdown;
6043 	else
6044 		return -EINVAL;
6045 
6046 	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa,
6047 				  ring_type);
6048 
6049 	r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu,
6050 				       amdgpu_ualink_emit_update_wb_addr,
6051 				       upper_32_bits(wb_npa),
6052 				       lower_32_bits(wb_npa),
6053 				       0, 0);
6054 	if (!r) {
6055 		ring->wb_cpu = wb_cpu;
6056 		ring->ready = true;
6057 	}
6058 	return r;
6059 }
6060 
6061 /**
6062  * amdgpu_ualink_remote_shootdown - Send a remote TLB invalidation request
6063  * @adev: amdgpu device pointer
6064  * @remote_accel_id: accelerator ID of the remote GPU to shootdown
6065  * @addr: page-aligned address to invalidate
6066  * @size_in_pages: number of pages to invalidate, 0 to invalidate entire TLB
6067  * @flush_type: type of TLB flush to perform
6068  *
6069  * Sends a remote TLB shootdown command to the specified peer GPU via the
6070  * UALink ring buffer. The address and size are translated using the S-field
6071  * encoding before being written to the ring.
6072  *
6073  * Return: 0 on success, negative error code on failure
6074  */
6075 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
6076 				   u32 remote_accel_id, u64 addr,
6077 				   u32 size_in_pages, u32 flush_type)
6078 {
6079 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6080 	struct amdgpu_ualink_peer *peer;
6081 	struct amdgpu_ualink_ring *ring;
6082 	int r;
6083 
6084 	dev_dbg(adev->dev, "send remote shootdown to remote_accel_id %u\n",
6085 		remote_accel_id);
6086 
6087 	peer = &remote->peer[remote_accel_id];
6088 	ring = &peer->shootdown;
6089 
6090 	guard(mutex)(&peer->lock);
6091 
6092 	if (!ring->ready) {
6093 		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
6094 		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
6095 						    RB_TYPE_TLB_INV);
6096 		if (r)
6097 			return r;
6098 	}
6099 
6100 	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
6101 					  ring->wb_cpu,
6102 					  amdgpu_ualink_emit_shootdown,
6103 					  flush_type, upper_32_bits(addr),
6104 					  lower_32_bits(addr), size_in_pages);
6105 }
6106 
6107 /**
6108  * amdgpu_ualink_remote_interrupt - Send a remote interrupt to a peer GPU
6109  * @adev: amdgpu device pointer
6110  * @remote_accel_id: accelerator ID of the remote GPU to interrupt
6111  * @dw0: IH context ID dword 0
6112  * @dw1: IH context ID dword 1
6113  * @dw2: IH context ID dword 2
6114  * @dw3: IH context ID dword 3
6115  *
6116  * Sends a remote interrupt command to the specified peer GPU via the
6117  * UALink ring buffer. The four context ID dwords are delivered to the
6118  * remote GPU's interrupt handler.
6119  *
6120  * Return: 0 on success, negative error code on failure
6121  */
6122 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
6123 				   u32 remote_accel_id, u32 dw0, u32 dw1,
6124 				   u32 dw2, u32 dw3)
6125 {
6126 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6127 	struct amdgpu_ualink_peer *peer;
6128 	struct amdgpu_ualink_ring *ring;
6129 	int r;
6130 
6131 	peer = &remote->peer[remote_accel_id];
6132 	ring = &peer->interrupt;
6133 
6134 	guard(mutex)(&peer->lock);
6135 
6136 	if (!ring->ready) {
6137 		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
6138 		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
6139 						    RB_TYPE_REMOTE_INTERRUPT);
6140 		if (r)
6141 			return r;
6142 	}
6143 
6144 	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
6145 					  ring->wb_cpu,
6146 					  amdgpu_ualink_emit_interrupt,
6147 					  dw0, dw1, dw2, dw3);
6148 }
6149 
6150 /**
6151  * amdgpu_ualink_peer_remote_init - Initialize remote peer GPU connections
6152  * @adev: amdgpu device pointer
6153  *
6154  * Sets up ring buffers, GART mappings, and control structures for
6155  * communicating with remote GPUs in the fabric.
6156  *
6157  * Return: 0 on success, negative error code on failure
6158  */
6159 static int amdgpu_ualink_peer_remote_init(struct amdgpu_device *adev)
6160 {
6161 	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
6162 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6163 	struct amdgpu_bo *bo = remote->ring_bo;
6164 	struct amdgpu_ualink_peer *peer;
6165 	struct amdgpu_ualink_ring *interrupt;
6166 	struct amdgpu_ualink_ring *shootdown;
6167 	u32 src_accel_id = ualink_accel_id(adev);
6168 	u32 dst_accel_id;
6169 	u64 npa, flags;
6170 	int r = 0;
6171 
6172 	/* NPA mapping PTE flags VSCT = 0011 */
6173 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, bo->tbo.resource);
6174 	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | AMDGPU_PTE_BUS_ATOMICS;
6175 	flags &= ~AMDGPU_PTE_VALID;
6176 	flags &= ~AMDGPU_PTE_SYSTEM;
6177 	/* PTE.X=0 turn off RPC checks for RBs, wptr and doorbell NPA address */
6178 	flags &= ~AMDGPU_PTE_EXECUTABLE;
6179 
6180 	dev_dbg(adev->dev, "src_accel_id %u gart mapping flags 0x%llx\n",
6181 		src_accel_id, flags);
6182 
6183 	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
6184 		if (dst_accel_id == src_accel_id)
6185 			continue;
6186 
6187 		dev_dbg(adev->dev, "peer_remote to dst_accel_id %u\n", dst_accel_id);
6188 
6189 		peer = &remote->peer[dst_accel_id];
6190 		mutex_init(&peer->lock);
6191 
6192 		interrupt = &peer->interrupt;
6193 		shootdown = &peer->shootdown;
6194 
6195 		/* ring buffer npa gart mapping */
6196 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
6197 						   src_accel_id, dst_accel_id);
6198 		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &interrupt->mm_node_rb, flags);
6199 		if (r)
6200 			break;
6201 		interrupt->rb_npa_gart = adev->gmc.gart_start +
6202 					 (interrupt->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
6203 
6204 		dev_dbg(adev->dev, "rb npa 0x%llx mapped on gart 0x%llx\n",
6205 			npa, interrupt->rb_npa_gart);
6206 
6207 		/* tlb invalidate ring buffer npa gart mapping */
6208 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TLB_INV, src_accel_id,
6209 						  dst_accel_id);
6210 		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &shootdown->mm_node_rb, flags);
6211 		if (r)
6212 			break;
6213 		shootdown->rb_npa_gart = adev->gmc.gart_start +
6214 					 (shootdown->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
6215 
6216 		dev_dbg(adev->dev, "tlb rb npa 0x%llx mapped on gart 0x%llx\n",
6217 			npa, shootdown->rb_npa_gart);
6218 
6219 		/* wptr npa gart mapping */
6220 		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TAILPTR, src_accel_id,
6221 						  dst_accel_id);
6222 		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_wptr, flags);
6223 		if (r)
6224 			break;
6225 
6226 		interrupt->wptr_npa_gart = adev->gmc.gart_start +
6227 					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
6228 					    + ualink_wptr_offset(adev, src_accel_id);
6229 		shootdown->wptr_npa_gart = adev->gmc.gart_start +
6230 					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
6231 					    + ualink_tlb_wptr_offset(adev, src_accel_id);
6232 
6233 		dev_dbg(adev->dev, "rb wptr npa 0x%llx on gart 0x%llx, tlb wptr on gart 0x%llx\n",
6234 			npa, interrupt->wptr_npa_gart, shootdown->wptr_npa_gart);
6235 
6236 		/*
6237 		 * doorbell npa gart mapping, doorbell NPA address coding
6238 		 *
6239 		 * ReqAddr[51:40] = 0xFFF
6240 		 * ReqAddr[39:30] = GPU accel_id
6241 		 * ReqAddr[29:12] = 18'h00000
6242 		 * ReqAddr[11:8]  = Random value
6243 		 * ReqAddr[7:6]   = PMI Index
6244 		 *     0 - PMI0 Remote Interrupt
6245 		 *     1 - PMI1 Remote TLB Shootdown
6246 		 * ReqAddr[5:0]   = 6'h0
6247 		 */
6248 		npa = 0xFFFULL << 40 | (u64)dst_accel_id << 30;
6249 		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_doorbell, flags);
6250 		if (r)
6251 			break;
6252 
6253 		interrupt->doorbell_npa_gart = adev->gmc.gart_start +
6254 			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT);
6255 
6256 		dev_dbg(adev->dev, "rb doorbell npa 0x%llx mapped on gart 0x%llx\n",
6257 			npa, interrupt->doorbell_npa_gart);
6258 
6259 		shootdown->doorbell_npa_gart = adev->gmc.gart_start+
6260 			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT) + (1 << 6);
6261 
6262 		dev_dbg(adev->dev, "tlb doorbell npa 0x%llx mapped on gart 0x%llx\n",
6263 			npa, shootdown->doorbell_npa_gart);
6264 
6265 		interrupt->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
6266 		interrupt->wptr = 0;
6267 		interrupt->rptr = 0;
6268 		interrupt->seq = 0;
6269 
6270 		shootdown->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
6271 		shootdown->wptr = 0;
6272 		shootdown->rptr = 0;
6273 		shootdown->seq = 0;
6274 	}
6275 
6276 	dev_dbg(adev->dev, "peer remote init done r=%d\n", r);
6277 	return r;
6278 }
6279 
6280 /**
6281  * amdgpu_ualink_peer_remote_fini - Clean up remote peer GPU connections
6282  * @adev: amdgpu device pointer
6283  *
6284  * Unmaps GART entries and frees resources for remote GPU communication.
6285  */
6286 static void amdgpu_ualink_peer_remote_fini(struct amdgpu_device *adev)
6287 {
6288 	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
6289 	struct amdgpu_ualink_remote *remote = to_remote(adev);
6290 	u32 src_accel_id = ualink_accel_id(adev);
6291 	u32 dst_accel_id;
6292 	struct amdgpu_ualink_peer *peer;
6293 	struct amdgpu_ualink_ring *ring;
6294 
6295 	dev_dbg(adev->dev, "src_accel_id %u\n", src_accel_id);
6296 
6297 	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
6298 		dev_dbg(adev->dev, "dst_accel_id %u\n", dst_accel_id);
6299 		if (dst_accel_id == src_accel_id)
6300 			continue;
6301 
6302 		peer = &remote->peer[dst_accel_id];
6303 		mutex_destroy(&peer->lock);
6304 
6305 		ring = &peer->interrupt;
6306 		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
6307 		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_wptr);
6308 		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_doorbell);
6309 
6310 		ring = &peer->shootdown;
6311 		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
6312 	}
6313 }
6314 
6315 /**
6316  * amdgpu_ualink_sw_init - Initialize UALink software resources
6317  * @adev: amdgpu device pointer
6318  *
6319  * Performs full UALink software initialization including metadata allocation,
6320  * NPA address mapping, SDMA scheduler entity setup, and remote peer GPU
6321  * connection initialization. On failure, all previously initialized resources
6322  * are cleaned up.
6323  *
6324  * Return: 0 on success, negative error code on failure
6325  */
6326 int amdgpu_ualink_sw_init(struct amdgpu_device *adev)
6327 {
6328 	struct amdgpu_ualink_remote *remote;
6329 	int r;
6330 
6331 	remote = kzalloc_obj(*remote);
6332 	if (!remote)
6333 		return -ENOMEM;
6334 	adev->ualink.remote = remote;
6335 
6336 	r = amdgpu_ualink_metadata_init(adev);
6337 	if (r)
6338 		goto out;
6339 
6340 	r = amdgpu_ualink_metadata_npa_mapping(adev);
6341 	if (r)
6342 		goto out_metadata_fini;
6343 
6344 	r = amdgpu_ualink_sdma_entities_init(adev);
6345 	if (r)
6346 		goto out_npa_unmap;
6347 
6348 	r = amdgpu_ualink_peer_remote_init(adev);
6349 	if (r)
6350 		goto out_sdma_entities_fini;
6351 
6352 	dev_dbg(adev->dev, "ualink sw init succeed\n");
6353 	return 0;
6354 
6355 out_sdma_entities_fini:
6356 	amdgpu_ualink_sdma_entities_fini(adev);
6357 out_npa_unmap:
6358 	amdgpu_ualink_metadata_npa_unmapping(adev);
6359 out_metadata_fini:
6360 	amdgpu_ualink_metadata_fini(adev);
6361 out:
6362 	kfree(adev->ualink.remote);
6363 	adev->ualink.remote = NULL;
6364 	dev_dbg(adev->dev, "ualink sw init failed %d\n", r);
6365 	return r;
6366 }
6367 
6368 /**
6369  * amdgpu_ualink_sw_fini - Tear down UALink software resources
6370  * @adev: amdgpu device pointer
6371  *
6372  * Cleans up all UALink software resources in reverse order of initialization:
6373  * remote peer connections, SDMA entities, NPA mappings, and metadata.
6374  */
6375 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev)
6376 {
6377 	dev_dbg(adev->dev, "halt accel_id %u addr_mode %d\n", ualink_accel_id(adev),
6378 		ualink_addr_mode(adev));
6379 
6380 	adev->ualink.msg_ctl->send_halt(adev);
6381 
6382 	amdgpu_ualink_peer_remote_fini(adev);
6383 	amdgpu_ualink_sdma_entities_fini(adev);
6384 	amdgpu_ualink_metadata_npa_unmapping(adev);
6385 	amdgpu_ualink_metadata_fini(adev);
6386 	kfree(adev->ualink.remote);
6387 	adev->ualink.remote = NULL;
6388 }
6389 
6390 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id)
6391 {
6392 	return amdgpu_ualink_send_hello_msg(adev, remote_accel_id);
6393 }
6394 
6395 /*
6396  * The low bits of handle_lo carry the message-header field on the wire.
6397  * Strip those bits and splice in @acc_id to recover the
6398  * fully-qualified handle_lo value.
6399  */
6400 static inline u64 amdgpu_ualink_reassemble_handle(u64 handle_lo, u32 acc_id)
6401 {
6402 	return (handle_lo & ~AMDGPU_UALINK_MESSAGE_HEADER_MASK) |
6403 	       (acc_id & AMDGPU_UALINK_HANDLE_ACCID_MASK);
6404 }
6405 
6406 static int amdgpu_ualink_process_irq(struct amdgpu_device *adev,
6407 				     struct amdgpu_irq_src *source,
6408 				     struct amdgpu_iv_entry *entry)
6409 {
6410 	u32 sender_acc_id, receiver_acc_id, msg_type, src_acc_id;
6411 	enum amdgpu_ualink_accel_state accel_state;
6412 	struct amdgpu_ualink_handle handle;
6413 	u32 size, npa_addr, fail_reason;
6414 	u32 dw0, dw1, dw2, dw3;
6415 	u32 local_acc_id;
6416 	int handled = 1;
6417 
6418 	if (unlikely(adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)) {
6419 		dev_dbg(adev->dev,
6420 			"UALink manager not initialized, dropping irq\n");
6421 		return handled;
6422 	}
6423 
6424 	/* Only handle interrupts while ACTIVE. Otherwise (e.g. during teardown)
6425 	 * consume stale ring entries but drop them, so they don't touch state
6426 	 * being freed.
6427 	 */
6428 	accel_state = adev->ualink.info->accel_state;
6429 	if (accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
6430 		dev_dbg(adev->dev,
6431 			"Dropping stale UALink interrupt, accel_state %d\n",
6432 			accel_state);
6433 		return handled;
6434 	}
6435 
6436 	dev_dbg(adev->dev, "%s client_id 0x%x src_id 0x%x ih\n",
6437 		entry->ih == &adev->irq.ih ? "ring" : "ualink soft ring",
6438 		entry->client_id, entry->src_id);
6439 
6440 	/* ContextID 4 dwords */
6441 	src_acc_id = entry->pasid;
6442 	dw0 = entry->src_data[0];
6443 	dw1 = entry->src_data[1];
6444 	dw2 = entry->src_data[2];
6445 	dw3 = entry->src_data[3];
6446 
6447 	/* Copy IH entry into ualink soft ring. */
6448 	if (entry->ih == &adev->irq.ih) {
6449 		dev_dbg(adev->dev, "src accel_id %u context id 0x%x 0x%x 0x%x 0x%x\n",
6450 			src_acc_id, dw0, dw1, dw2, dw3);
6451 		dev_dbg(adev->dev, "delegate to ualink irq soft ring\n");
6452 		amdgpu_irq_ualink_delegate(adev, entry, 8);
6453 		return handled;
6454 	}
6455 
6456 	msg_type = dw0 & AMDGPU_UALINK_MESSAGE_HEADER_MASK;
6457 	local_acc_id = adev->ualink.info->ppod.accel_id;
6458 	dev_dbg(adev->dev, "Got MSG: remote acc_id %u msg_type %u\n",
6459 		src_acc_id, msg_type);
6460 
6461 	/* Debug hook: if the bit corresponding to this msg_type is set in
6462 	 * drop_msg_bitmap, drop this single packet and clear the bit so that
6463 	 * any subsequent incoming messages are processed normally. Used to
6464 	 * exercise the connection reset/recovery paths via debugfs.
6465 	 */
6466 	if (msg_type < BITS_PER_LONG &&
6467 	    test_and_clear_bit(msg_type, &adev->ualink.drop_msg_bitmap)) {
6468 		dev_warn(adev->dev,
6469 			 "DROP MSG (debugfs): src acc_id %u msg_type %u dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
6470 			 src_acc_id, msg_type, dw0, dw1, dw2, dw3);
6471 		return handled;
6472 	}
6473 
6474 	switch (msg_type) {
6475 	case AMDGPU_UALINK_HELLO_MSG:
6476 		receiver_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT) &
6477 					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
6478 		sender_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT) &
6479 					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
6480 		dev_dbg(adev->dev,
6481 			"Got HELLO MSG: src acc_id %u receiver_acc_id %u sender_acc_id %u\n",
6482 			src_acc_id, receiver_acc_id, sender_acc_id);
6483 		amdgpu_ualink_process_hello_msg(adev, receiver_acc_id, sender_acc_id,
6484 						src_acc_id);
6485 		break;
6486 	case AMDGPU_UALINK_HELLO_ACK_MSG:
6487 		dev_dbg(adev->dev,
6488 			"Got HELLO-ACK MSG: remote acc_id %u\n", src_acc_id);
6489 		amdgpu_ualink_process_hello_ack_msg(adev, src_acc_id);
6490 		break;
6491 	case AMDGPU_UALINK_NPA_REQ_MSG:
6492 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6493 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6494 								   local_acc_id);
6495 		dev_dbg(adev->dev,
6496 			"Got NPA-REQ MSG: remote acc_id %u handle %llx:%llx\n",
6497 			src_acc_id, handle.handle_hi, handle.handle_lo);
6498 		amdgpu_ualink_process_npa_req_msg(adev, src_acc_id, handle);
6499 		break;
6500 	case AMDGPU_UALINK_NPA_RSP_MSG:
6501 		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
6502 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6503 								   src_acc_id);
6504 		size = entry->src_data[2];
6505 		npa_addr = entry->src_data[3];
6506 		dev_dbg(adev->dev,
6507 			"Got NPA-RSP MSG: remote acc_id %u handle_lo %llx size 0x%x npa_addr 0x%x\n",
6508 			src_acc_id, handle.handle_lo, size, npa_addr);
6509 		amdgpu_ualink_process_npa_rsp_msg(adev, src_acc_id,
6510 						  handle.handle_lo,
6511 						  npa_addr, size);
6512 		break;
6513 	case AMDGPU_UALINK_NPA_FAIL_MSG:
6514 		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
6515 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6516 								   src_acc_id);
6517 		fail_reason = entry->src_data[2] &
6518 				AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK;
6519 		dev_dbg(adev->dev,
6520 			"Got NPA-FAIL MSG: remote acc_id %u handle_lo %llx fail_reason %u\n",
6521 			src_acc_id, handle.handle_lo, fail_reason);
6522 		amdgpu_ualink_process_npa_fail_msg(adev, src_acc_id,
6523 						   handle.handle_lo,
6524 						   fail_reason);
6525 		break;
6526 	case AMDGPU_UALINK_NPA_REVOKE_MSG:
6527 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6528 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6529 								   src_acc_id);
6530 		dev_dbg(adev->dev,
6531 			"Got NPA-REVOKE MSG: remote acc_id %u handle %llx:%llx\n",
6532 			src_acc_id, handle.handle_hi, handle.handle_lo);
6533 		amdgpu_ualink_process_npa_revoke_msg(adev, src_acc_id, handle);
6534 		break;
6535 	case AMDGPU_UALINK_NPA_RELEASE_MSG:
6536 		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
6537 		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
6538 								   local_acc_id);
6539 		dev_dbg(adev->dev,
6540 			"Got NPA-RELEASE MSG: remote acc_id %u handle %llx:%llx\n",
6541 			src_acc_id, handle.handle_hi, handle.handle_lo);
6542 		amdgpu_ualink_process_npa_release_msg(adev, src_acc_id, handle);
6543 		break;
6544 	default:
6545 		dev_err(adev->dev, "Unknown message type (%u)\n", msg_type);
6546 		break;
6547 	}
6548 
6549 	return handled;
6550 }
6551 
6552 static int amdgpu_ualink_set_irq_state(struct amdgpu_device *adev,
6553 					struct amdgpu_irq_src *source,
6554 					u32 type,
6555 					enum amdgpu_interrupt_state state)
6556 {
6557 	/*
6558 	 * Don't set register to enable/disable nHT controller interrupt.
6559 	 *
6560 	 * F/W running on MP2, which can always send cookie to IH block to
6561 	 * interrupt driver.
6562 	 */
6563 	dev_dbg(adev->dev, "ualink interrupt %s\n",
6564 		state == AMDGPU_IRQ_STATE_ENABLE ? "enable" : "disable");
6565 	return 0;
6566 }
6567 
6568 static const struct amdgpu_irq_src_funcs ualink_irq_funcs = {
6569 	.set = amdgpu_ualink_set_irq_state,
6570 	.process = amdgpu_ualink_process_irq,
6571 };
6572 
6573 /**
6574  * amdgpu_ualink_init_interrupt - register the UALink IRQ source
6575  * @adev: amdgpu device pointer
6576  * @client_id: IH client ID for the interrupt source
6577  * @src_id: source ID for the interrupt source
6578  *
6579  * Registers the UALink interrupt source with the IH subsystem.
6580  *
6581  * Return: 0 on success, negative error code on failure
6582  */
6583 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev,
6584 				 unsigned int client_id, unsigned int src_id)
6585 {
6586 	int r;
6587 
6588 	dev_dbg(adev->dev, "init ualink irq client_id 0x%x src_id 0x%x\n",
6589 		client_id, src_id);
6590 
6591 	adev->ualink.irq.num_types = 1;
6592 	adev->ualink.irq.funcs = &ualink_irq_funcs;
6593 
6594 	r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->ualink.irq);
6595 	return r;
6596 }
6597 
6598