xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h (revision 15d1c306d8291a9d66f900b10c31e87591169ecd)
1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /*
3  * Copyright 2026 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #ifndef AMDGPU_UALINK_H
26 #define AMDGPU_UALINK_H
27 
28 #include <linux/uuid.h>
29 #include <linux/bitmap.h>
30 #include <linux/kobject.h>
31 
32 struct amdgpu_device;
33 
34 struct amdgpu_xcp;
35 
36 #define AMDGPU_UALINK_ACCEL_MAX 256
37 #define AMDGPU_UALINK_LOCAL_ACCELS_MAX 8
38 #define AMDGPU_UALINK_STATIONS_MAX 64
39 
40 /* A vpod_id of 0 is reserved and treated as invalid/no vPod */
41 #define AMDGPU_UALINK_VPOD_ID_INVALID 0
42 
43 /* nHT firmware status */
44 #define AMDGPU_NHT_FW_ST_PREINIT	0xA0
45 #define AMDGPU_NHT_FW_ST_READY	0xA1
46 #define AMDGPU_NHT_FW_ST_HALT	0xA2
47 #define AMDGPU_NHT_FW_ST_ERROR	0xA3
48 #define AMDGPU_NHT_FW_ST_FATAL	0xF0
49 
50 #define AMDGPU_UALINK_RESP_TIMEOUT			5000 /* 5s timeout */
51 
52 #define AMDGPU_UALINK_HANDLE_ACCID_MASK			GENMASK_ULL(9, 0)
53 #define AMDGPU_UALINK_MESSAGE_HEADER_MASK		GENMASK_ULL(9, 0)
54 #define AMDGPU_UALINK_HELLO_MSG_ACCID_MASK		GENMASK_U32(9, 0)
55 #define AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT	10
56 #define AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT	20
57 #define AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK	GENMASK_U32(7, 0)
58 
59 /* GPU-ID is stored in bits 41-50 of the NPA address. However, we
60  * store NPA address is GPU PAGE aligned so bottom 12 bits are not used.
61  * As a result, we need the GPU-ID shift to be 41 - 12 = 29.
62  */
63 #define AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT		29
64 #define AMDGPU_UALINK_NPA_ADDR_GPUID_MASK		GENMASK_ULL(38, 29)
65 /* Reserve 2M in each 2TB range for ring buffer allocations for
66  * remote interrupts. In terms of GPU pages, this is 2M / 4K = 512 pages.
67  * So we reserve 512 pages in each 2TB range.
68  */
69 #define AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED		(1U << 9)
70 #define AMDGPU_UALINK_NPA_ADDR_RANGE_MASK		GENMASK_ULL(28, 0)
71 
72 enum AMDGPU_UALINK_NPA_FAIL_REASON {
73 	AMDGPU_UALINK_NPA_FAIL_NOSPACE			= 1,
74 	AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE		= 2,
75 	AMDGPU_UALINK_NPA_FAIL_DUPLICATE		= 3,
76 	AMDGPU_UALINK_NPA_FAIL_ERROR			= 4,
77 };
78 
79 enum AMDGPU_UALINK_NODE_STATE {
80 	AMDGPU_UALINK_NODE_NOT_READY			= 0,
81 	AMDGPU_UALINK_NODE_PENDING			= 1,
82 	AMDGPU_UALINK_NODE_READY			= 2,
83 	AMDGPU_UALINK_NODE_TEARDOWN			= 3
84 };
85 
86 enum AMDGPU_UALINK_PROTOCOL_MESSAGES {
87 	AMDGPU_UALINK_HELLO_MSG				= 1,
88 	AMDGPU_UALINK_HELLO_ACK_MSG			= 2,
89 	AMDGPU_UALINK_NPA_REQ_MSG			= 3,
90 	AMDGPU_UALINK_NPA_RSP_MSG			= 4,
91 	AMDGPU_UALINK_NPA_FAIL_MSG			= 5,
92 	AMDGPU_UALINK_NPA_REVOKE_MSG			= 6,
93 	AMDGPU_UALINK_NPA_RELEASE_MSG			= 7,
94 	AMDGPU_UALINK_MAX_PROTOCOL_MSG
95 };
96 
97 #define AMDGPU_UALINK_LIGHTWEIGHT_TLB_SHOOTDOWN		0x1
98 #define AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN		0x2
99 
100 enum amdgpu_ualink_conn_state {
101 	AMDGPU_UALINK_CONN_NOT_READY			= 0,
102 	AMDGPU_UALINK_CONN_IN_PROGRESS			= 1,
103 	AMDGPU_UALINK_CONN_PENDING			= 2,
104 	AMDGPU_UALINK_CONN_ESTABLISHED			= 3
105 };
106 
107 enum amdgpu_ualink_type {
108 	AMDGPU_UALINK_TYPE_UALOE = 0,
109 	AMDGPU_UALINK_TYPE_UALINK = 1,
110 	AMDGPU_UALINK_TYPE_MAX
111 };
112 
113 enum amdgpu_ualink_accel_state {
114 	AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED = 0,
115 	AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED,
116 	AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED,
117 	AMDGPU_UALINK_ACCEL_STATE_READY,
118 	AMDGPU_UALINK_ACCEL_STATE_ACTIVE,
119 	AMDGPU_UALINK_ACCEL_STATE_ERROR,
120 	AMDGPU_UALINK_ACCEL_STATE_MAX
121 };
122 
123 enum amdgpu_ualink_addr_mode {
124 	AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS = 0,
125 	AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT = 1,
126 	AMDGPU_UALINK_ADDR_MODE_MAX
127 };
128 
129 enum amdgpu_ualink_mgr_state {
130 	AMDGPU_UALINK_INIT_NONE = 0,
131 	AMDGPU_UALINK_INIT_HW,
132 	AMDGPU_UALINK_INIT_COMPLETE,
133 	AMDGPU_UALINK_INIT_ERROR
134 };
135 
136 /* Physical pod info shared between query and setup API */
137 struct amdgpu_ualink_ppod_info {
138 	u32 accel_id;
139 	u32 bandwidth;
140 	u32 latency;
141 	u32 size;
142 	uuid_t id;
143 };
144 
145 /* Virtual pod info shared between query and config API */
146 struct amdgpu_ualink_vpod_info {
147 	u32 id;
148 	u32 size;
149 	u32 addr_mode;
150 	DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
151 };
152 
153 /* Query current UALink physical and virtual pod info */
154 struct amdgpu_ualink_info {
155 	struct kobject kobj;
156 	enum amdgpu_ualink_type link_type;
157 	enum amdgpu_ualink_accel_state accel_state;
158 	struct amdgpu_ualink_ppod_info ppod;
159 	struct amdgpu_ualink_vpod_info vpod;
160 	/* Local accelerator array in the vpod derived from
161 	 * vpod.active_accel_bits, in arbitrary order
162 	 */
163 	u32 n_local_accels;
164 	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
165 };
166 #define to_ualink_info(ko) container_of(ko, struct amdgpu_ualink_info, kobj)
167 
168 /* UALink physical pod setup */
169 struct amdgpu_ualink_ppod_setup {
170 	struct kobject kobj;
171 	struct amdgpu_ualink_ppod_info ppod;
172 	/* Local accelerator array indexed by socket ID */
173 	u32 n_local_accels;
174 	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
175 };
176 #define to_ualink_ppod_setup(ko) container_of(ko, struct amdgpu_ualink_ppod_setup, kobj)
177 
178 /* UAlink virtual pod config */
179 struct amdgpu_ualink_vpod_config {
180 	struct kobject kobj;
181 	struct amdgpu_ualink_vpod_info vpod;
182 };
183 #define to_ualink_vpod_config(ko) container_of(ko, struct amdgpu_ualink_vpod_config, kobj)
184 
185 /* UALink station configuration */
186 struct amdgpu_ualink_station_config {
187 	struct kobject kobj;
188 	/* Station configuration flags
189 	 * bits [3:0]: PortPerStation (PPS) - 1, 2, or 4
190 	 * bits [7:4]: Reserved
191 	 */
192 	u8 flags;
193 	u8 n_stations;
194 	/* bitmap or enabled lanes for each station in logical station order */
195 	u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX];
196 };
197 #define to_ualink_station_config(ko) container_of(ko, struct amdgpu_ualink_station_config, kobj)
198 
199 struct amdgpu_ualink_handle {
200 	union {
201 		struct {
202 			u64 handle_lo;
203 			u64 handle_hi;
204 		};
205 		u64 handle[2];
206 	};
207 };
208 
209 struct amdgpu_ualink_imp_xa_node {
210 	struct amdgpu_device		*adev;
211 
212 	/* 128-bit handle for the BO */
213 	struct amdgpu_ualink_handle	handle;
214 
215 	/* Use to signal NPA-RSP arrival */
216 	struct completion		npa_done;
217 
218 	/* NPA address received in the NPA-RSP message */
219 	u64				npa_addr;
220 	u64				size;
221 
222 	/* GEM handle for the NPA BO */
223 	u32				gem_handle;
224 
225 	/* Fail reason received in NPA-FAIL message */
226 	int				fail_reason;
227 
228 	/* Node state to signal if node setup is in progress
229 	 * or is already completed. Node state goes back to
230 	 * in progress if a HELLO message is received in
231 	 * response to NPA-REQ message.
232 	 */
233 	enum AMDGPU_UALINK_NODE_STATE	node_state;
234 
235 	/* Used to connect all importer XA nodes from a particular
236 	 * exporter.
237 	 */
238 	struct list_head		list;
239 
240 	/* Dmabuf corresponding to the NPA BO */
241 	struct dma_buf			*dmabuf;
242 
243 	/* Refcount to track lifetime of this node */
244 	struct kref			refcount;
245 };
246 
247 struct amdgpu_ualink_npa_mm {
248 	struct drm_mm			mm;
249 	u64				va_start;
250 	u64				va_size;
251 	struct mutex			mm_lock;
252 };
253 
254 struct amdgpu_ualink_importer_entry {
255 	struct drm_mm_node			*mm_node;
256 	u64					npa_addr;
257 
258 	/* Keep track of if the connection got reset */
259 	u32					generation_count;
260 
261 	/* Used to connect all handles exported to a particular importer */
262 	struct list_head			list;
263 
264 	/* Pointer to the parent XA node. */
265 	struct amdgpu_ualink_exp_xa_node	*parent;
266 };
267 
268 struct amdgpu_ualink_exp_xa_node {
269 	/* 128-bit handle for the BO */
270 	struct amdgpu_ualink_handle		handle;
271 
272 	/* Pointer to the BO thats exported.*/
273 	struct amdgpu_bo			*bo;
274 
275 	/* Dmabuf corresponding to the BO */
276 	struct dma_buf				*dmabuf;
277 
278 	/* Mutex to protect the node from concurrent access */
279 	struct mutex				node_lock;
280 
281 	/* Used for storing importer info in source identification mode */
282 	struct amdgpu_ualink_importer_entry	importer_entries[AMDGPU_UALINK_ACCEL_MAX];
283 
284 	/* Used to track all importers of this BO. This is set when the
285 	 * exporter sends back the NPA-RSP message.
286 	 */
287 	DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
288 
289 	/* This bitmap is used to send NPA-REVOKE messages to all the importers.
290 	 * And to track the NPA-RELEASE response for each NPA-REVOKE message sent.
291 	 * A bit is cleared in this bitmap when the NPA RELEASE message is
292 	 * received in response to the NPA-REVOKE message.
293 	 */
294 	DECLARE_BITMAP(npa_release_bitmap, AMDGPU_UALINK_ACCEL_MAX);
295 
296 	/* Use to signal responses received from all importers */
297 	struct completion			npa_done;
298 
299 	/* Refcount to track lifetime of this node */
300 	struct kref				refcount;
301 
302 	/* Work to cleanup the node. */
303 	struct work_struct			cleanup_work;
304 };
305 
306 struct amdgpu_ualink_connection {
307 	struct completion hello_done;
308 	struct mutex lock;
309 	u32 generation_count;
310 	enum amdgpu_ualink_conn_state state;
311 };
312 
313 struct amdgpu_ualink_remote;
314 
315 struct amdgpu_ualink_msg_ctl {
316 	u32 (*check_status)(struct amdgpu_device *adev);
317 	int (*send_metadata)(struct amdgpu_device *adev, u64 metadata_gpu_addr,
318 			     u32 accel_data);
319 	int (*send_halt)(struct amdgpu_device *adev);
320 };
321 
322 struct amdgpu_ualink_mgr {
323 	u64 npa_size;
324 	u32 psp_if_ver;
325 	const struct amdgpu_ualink_msg_ctl *msg_ctl;
326 	struct amdgpu_ualink_info *info;
327 	struct amdgpu_ualink_ppod_setup *setup;
328 	struct amdgpu_ualink_vpod_config *config;
329 	struct amdgpu_ualink_station_config *stations;
330 	bool sysfs_init;
331 	enum amdgpu_ualink_mgr_state mgr_state;
332 
333 	/* For remote interrupt and shootdown */
334 	struct amdgpu_ualink_remote *remote;
335 
336 	/* handle irq from ualink client of remote GPUs */
337 	struct amdgpu_irq_src irq;
338 
339 	/* Xarray to store info about exported BOs */
340 	struct xarray	exp_xa;
341 
342 	/* Xarray to store info about imported ualink handles */
343 	struct xarray	imp_xa;
344 
345 	/* Xarray to store handles to be deleted */
346 	struct xarray	handle_invalid_xa;
347 
348 	/* Array to store connection state per remote GPU */
349 	struct amdgpu_ualink_connection conn_state[AMDGPU_UALINK_ACCEL_MAX];
350 
351 	/* List of exported ualink handles per GPU. Used to quickly traverse
352 	 * the list of all handles exported to a remote GPU.
353 	 */
354 	struct list_head exp_handles_list[AMDGPU_UALINK_ACCEL_MAX];
355 
356 	/* List of imported ualink handles per GPU. Used to quickly traverse
357 	 * the list of all handles imported from a remote GPU.
358 	 */
359 	struct list_head imp_handles_list[AMDGPU_UALINK_ACCEL_MAX];
360 
361 	/* WQ to manage revocation of exported memory. */
362 	struct workqueue_struct *npa_wq;
363 
364 	/* Memory-manager for managing NPA address space. */
365 	struct amdgpu_ualink_npa_mm npa_mm;
366 
367 	/* NPA-VM used on the exporter.*/
368 	struct amdgpu_vm npa_vm;
369 
370 	/* DRM client for UALink to manage GEM handles for NPA BOs */
371 	struct drm_client_dev client;
372 
373 	/* Sequence number to track the need for TLB flushes */
374 	atomic64_t last_flushed_tlb_seq;
375 
376 	/* Debug-only: bitmap of incoming UALink protocol messages to drop.
377 	 *
378 	 * Each bit position corresponds to a value from
379 	 * enum AMDGPU_UALINK_PROTOCOL_MESSAGES:
380 	 *   BIT(AMDGPU_UALINK_NPA_REQ_MSG) drops one incoming NPA-REQ.
381 	 *   BIT(AMDGPU_UALINK_NPA_RSP_MSG) drops one incoming NPA-RSP.
382 	 *
383 	 *   On reception of a message whose corresponding bit is set, the bit
384 	 *   is atomically cleared and the message is silently dropped. This
385 	 *   means at most one packet per set bit is dropped; any further
386 	 *   incoming packets of the same type are processed normally. This is
387 	 *   intended to exercise the connection reset / recovery paths from a
388 	 *   debugfs handle.
389 	 */
390 	unsigned long drop_msg_bitmap;
391 };
392 
393 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev,
394 				 unsigned int client_id, unsigned int src_id);
395 
396 int amdgpu_ualink_sw_init(struct amdgpu_device *adev);
397 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev);
398 
399 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id);
400 
401 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev);
402 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev);
403 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev);
404 
405 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev);
406 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev);
407 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp);
408 int amdgpu_ualink_manager_start(struct amdgpu_device *adev);
409 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev);
410 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp,
411 				u32 gem_handle,
412 				struct amdgpu_ualink_handle *handle_out);
413 int amdgpu_ualink_import_handle(struct drm_device *dev,
414 				const struct amdgpu_ualink_handle *ualink_handle,
415 				int *fd_out);
416 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data,
417 				   struct drm_file *filp);
418 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo);
419 
420 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block);
421 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block);
422 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block);
423 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block);
424 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block);
425 
426 extern const struct amdgpu_ip_block_version ualink_v1_0_ip_block;
427 #endif
428