1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /* 3 * Copyright 2026 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef AMDGPU_UALINK_H 26 #define AMDGPU_UALINK_H 27 28 #include <linux/uuid.h> 29 #include <linux/bitmap.h> 30 #include <linux/kobject.h> 31 32 struct amdgpu_device; 33 34 struct amdgpu_xcp; 35 36 #define AMDGPU_UALINK_ACCEL_MAX 256 37 #define AMDGPU_UALINK_LOCAL_ACCELS_MAX 8 38 #define AMDGPU_UALINK_STATIONS_MAX 64 39 40 /* A vpod_id of 0 is reserved and treated as invalid/no vPod */ 41 #define AMDGPU_UALINK_VPOD_ID_INVALID 0 42 43 /* nHT firmware status */ 44 #define AMDGPU_NHT_FW_ST_PREINIT 0xA0 45 #define AMDGPU_NHT_FW_ST_READY 0xA1 46 #define AMDGPU_NHT_FW_ST_HALT 0xA2 47 #define AMDGPU_NHT_FW_ST_ERROR 0xA3 48 #define AMDGPU_NHT_FW_ST_FATAL 0xF0 49 50 #define AMDGPU_UALINK_RESP_TIMEOUT 5000 /* 5s timeout */ 51 52 #define AMDGPU_UALINK_HANDLE_ACCID_MASK GENMASK_ULL(9, 0) 53 #define AMDGPU_UALINK_MESSAGE_HEADER_MASK GENMASK_ULL(9, 0) 54 #define AMDGPU_UALINK_HELLO_MSG_ACCID_MASK GENMASK_U32(9, 0) 55 #define AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT 10 56 #define AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT 20 57 #define AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK GENMASK_U32(7, 0) 58 59 /* GPU-ID is stored in bits 41-50 of the NPA address. However, we 60 * store NPA address is GPU PAGE aligned so bottom 12 bits are not used. 61 * As a result, we need the GPU-ID shift to be 41 - 12 = 29. 62 */ 63 #define AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT 29 64 #define AMDGPU_UALINK_NPA_ADDR_GPUID_MASK GENMASK_ULL(38, 29) 65 /* Reserve 2M in each 2TB range for ring buffer allocations for 66 * remote interrupts. In terms of GPU pages, this is 2M / 4K = 512 pages. 67 * So we reserve 512 pages in each 2TB range. 68 */ 69 #define AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED (1U << 9) 70 #define AMDGPU_UALINK_NPA_ADDR_RANGE_MASK GENMASK_ULL(28, 0) 71 72 enum AMDGPU_UALINK_NPA_FAIL_REASON { 73 AMDGPU_UALINK_NPA_FAIL_NOSPACE = 1, 74 AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE = 2, 75 AMDGPU_UALINK_NPA_FAIL_DUPLICATE = 3, 76 AMDGPU_UALINK_NPA_FAIL_ERROR = 4, 77 }; 78 79 enum AMDGPU_UALINK_NODE_STATE { 80 AMDGPU_UALINK_NODE_NOT_READY = 0, 81 AMDGPU_UALINK_NODE_PENDING = 1, 82 AMDGPU_UALINK_NODE_READY = 2, 83 AMDGPU_UALINK_NODE_TEARDOWN = 3 84 }; 85 86 enum AMDGPU_UALINK_PROTOCOL_MESSAGES { 87 AMDGPU_UALINK_HELLO_MSG = 1, 88 AMDGPU_UALINK_HELLO_ACK_MSG = 2, 89 AMDGPU_UALINK_NPA_REQ_MSG = 3, 90 AMDGPU_UALINK_NPA_RSP_MSG = 4, 91 AMDGPU_UALINK_NPA_FAIL_MSG = 5, 92 AMDGPU_UALINK_NPA_REVOKE_MSG = 6, 93 AMDGPU_UALINK_NPA_RELEASE_MSG = 7, 94 AMDGPU_UALINK_MAX_PROTOCOL_MSG 95 }; 96 97 #define AMDGPU_UALINK_LIGHTWEIGHT_TLB_SHOOTDOWN 0x1 98 #define AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN 0x2 99 100 enum amdgpu_ualink_conn_state { 101 AMDGPU_UALINK_CONN_NOT_READY = 0, 102 AMDGPU_UALINK_CONN_IN_PROGRESS = 1, 103 AMDGPU_UALINK_CONN_PENDING = 2, 104 AMDGPU_UALINK_CONN_ESTABLISHED = 3 105 }; 106 107 enum amdgpu_ualink_type { 108 AMDGPU_UALINK_TYPE_UALOE = 0, 109 AMDGPU_UALINK_TYPE_UALINK = 1, 110 AMDGPU_UALINK_TYPE_MAX 111 }; 112 113 enum amdgpu_ualink_accel_state { 114 AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED = 0, 115 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED, 116 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED, 117 AMDGPU_UALINK_ACCEL_STATE_READY, 118 AMDGPU_UALINK_ACCEL_STATE_ACTIVE, 119 AMDGPU_UALINK_ACCEL_STATE_ERROR, 120 AMDGPU_UALINK_ACCEL_STATE_MAX 121 }; 122 123 enum amdgpu_ualink_addr_mode { 124 AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS = 0, 125 AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT = 1, 126 AMDGPU_UALINK_ADDR_MODE_MAX 127 }; 128 129 enum amdgpu_ualink_mgr_state { 130 AMDGPU_UALINK_INIT_NONE = 0, 131 AMDGPU_UALINK_INIT_HW, 132 AMDGPU_UALINK_INIT_COMPLETE, 133 AMDGPU_UALINK_INIT_ERROR 134 }; 135 136 /* Physical pod info shared between query and setup API */ 137 struct amdgpu_ualink_ppod_info { 138 u32 accel_id; 139 u32 bandwidth; 140 u32 latency; 141 u32 size; 142 uuid_t id; 143 }; 144 145 /* Virtual pod info shared between query and config API */ 146 struct amdgpu_ualink_vpod_info { 147 u32 id; 148 u32 size; 149 u32 addr_mode; 150 DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); 151 }; 152 153 /* Query current UALink physical and virtual pod info */ 154 struct amdgpu_ualink_info { 155 struct kobject kobj; 156 enum amdgpu_ualink_type link_type; 157 enum amdgpu_ualink_accel_state accel_state; 158 struct amdgpu_ualink_ppod_info ppod; 159 struct amdgpu_ualink_vpod_info vpod; 160 /* Local accelerator array in the vpod derived from 161 * vpod.active_accel_bits, in arbitrary order 162 */ 163 u32 n_local_accels; 164 u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX]; 165 }; 166 #define to_ualink_info(ko) container_of(ko, struct amdgpu_ualink_info, kobj) 167 168 /* UALink physical pod setup */ 169 struct amdgpu_ualink_ppod_setup { 170 struct kobject kobj; 171 struct amdgpu_ualink_ppod_info ppod; 172 /* Local accelerator array indexed by socket ID */ 173 u32 n_local_accels; 174 u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX]; 175 }; 176 #define to_ualink_ppod_setup(ko) container_of(ko, struct amdgpu_ualink_ppod_setup, kobj) 177 178 /* UAlink virtual pod config */ 179 struct amdgpu_ualink_vpod_config { 180 struct kobject kobj; 181 struct amdgpu_ualink_vpod_info vpod; 182 }; 183 #define to_ualink_vpod_config(ko) container_of(ko, struct amdgpu_ualink_vpod_config, kobj) 184 185 /* UALink station configuration */ 186 struct amdgpu_ualink_station_config { 187 struct kobject kobj; 188 /* Station configuration flags 189 * bits [3:0]: PortPerStation (PPS) - 1, 2, or 4 190 * bits [7:4]: Reserved 191 */ 192 u8 flags; 193 u8 n_stations; 194 /* bitmap or enabled lanes for each station in logical station order */ 195 u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX]; 196 }; 197 #define to_ualink_station_config(ko) container_of(ko, struct amdgpu_ualink_station_config, kobj) 198 199 struct amdgpu_ualink_handle { 200 union { 201 struct { 202 u64 handle_lo; 203 u64 handle_hi; 204 }; 205 u64 handle[2]; 206 }; 207 }; 208 209 struct amdgpu_ualink_imp_xa_node { 210 struct amdgpu_device *adev; 211 212 /* 128-bit handle for the BO */ 213 struct amdgpu_ualink_handle handle; 214 215 /* Use to signal NPA-RSP arrival */ 216 struct completion npa_done; 217 218 /* NPA address received in the NPA-RSP message */ 219 u64 npa_addr; 220 u64 size; 221 222 /* GEM handle for the NPA BO */ 223 u32 gem_handle; 224 225 /* Fail reason received in NPA-FAIL message */ 226 int fail_reason; 227 228 /* Node state to signal if node setup is in progress 229 * or is already completed. Node state goes back to 230 * in progress if a HELLO message is received in 231 * response to NPA-REQ message. 232 */ 233 enum AMDGPU_UALINK_NODE_STATE node_state; 234 235 /* Used to connect all importer XA nodes from a particular 236 * exporter. 237 */ 238 struct list_head list; 239 240 /* Dmabuf corresponding to the NPA BO */ 241 struct dma_buf *dmabuf; 242 243 /* Refcount to track lifetime of this node */ 244 struct kref refcount; 245 }; 246 247 struct amdgpu_ualink_npa_mm { 248 struct drm_mm mm; 249 u64 va_start; 250 u64 va_size; 251 struct mutex mm_lock; 252 }; 253 254 struct amdgpu_ualink_importer_entry { 255 struct drm_mm_node *mm_node; 256 u64 npa_addr; 257 258 /* Keep track of if the connection got reset */ 259 u32 generation_count; 260 261 /* Used to connect all handles exported to a particular importer */ 262 struct list_head list; 263 264 /* Pointer to the parent XA node. */ 265 struct amdgpu_ualink_exp_xa_node *parent; 266 }; 267 268 struct amdgpu_ualink_exp_xa_node { 269 /* 128-bit handle for the BO */ 270 struct amdgpu_ualink_handle handle; 271 272 /* Pointer to the BO thats exported.*/ 273 struct amdgpu_bo *bo; 274 275 /* Dmabuf corresponding to the BO */ 276 struct dma_buf *dmabuf; 277 278 /* Mutex to protect the node from concurrent access */ 279 struct mutex node_lock; 280 281 /* Used for storing importer info in source identification mode */ 282 struct amdgpu_ualink_importer_entry importer_entries[AMDGPU_UALINK_ACCEL_MAX]; 283 284 /* Used to track all importers of this BO. This is set when the 285 * exporter sends back the NPA-RSP message. 286 */ 287 DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 288 289 /* This bitmap is used to send NPA-REVOKE messages to all the importers. 290 * And to track the NPA-RELEASE response for each NPA-REVOKE message sent. 291 * A bit is cleared in this bitmap when the NPA RELEASE message is 292 * received in response to the NPA-REVOKE message. 293 */ 294 DECLARE_BITMAP(npa_release_bitmap, AMDGPU_UALINK_ACCEL_MAX); 295 296 /* Use to signal responses received from all importers */ 297 struct completion npa_done; 298 299 /* Refcount to track lifetime of this node */ 300 struct kref refcount; 301 302 /* Work to cleanup the node. */ 303 struct work_struct cleanup_work; 304 }; 305 306 struct amdgpu_ualink_connection { 307 struct completion hello_done; 308 struct mutex lock; 309 u32 generation_count; 310 enum amdgpu_ualink_conn_state state; 311 }; 312 313 struct amdgpu_ualink_remote; 314 315 struct amdgpu_ualink_msg_ctl { 316 u32 (*check_status)(struct amdgpu_device *adev); 317 int (*send_metadata)(struct amdgpu_device *adev, u64 metadata_gpu_addr, 318 u32 accel_data); 319 int (*send_halt)(struct amdgpu_device *adev); 320 }; 321 322 struct amdgpu_ualink_mgr { 323 u64 npa_size; 324 u32 psp_if_ver; 325 const struct amdgpu_ualink_msg_ctl *msg_ctl; 326 struct amdgpu_ualink_info *info; 327 struct amdgpu_ualink_ppod_setup *setup; 328 struct amdgpu_ualink_vpod_config *config; 329 struct amdgpu_ualink_station_config *stations; 330 bool sysfs_init; 331 enum amdgpu_ualink_mgr_state mgr_state; 332 333 /* For remote interrupt and shootdown */ 334 struct amdgpu_ualink_remote *remote; 335 336 /* handle irq from ualink client of remote GPUs */ 337 struct amdgpu_irq_src irq; 338 339 /* Xarray to store info about exported BOs */ 340 struct xarray exp_xa; 341 342 /* Xarray to store info about imported ualink handles */ 343 struct xarray imp_xa; 344 345 /* Xarray to store handles to be deleted */ 346 struct xarray handle_invalid_xa; 347 348 /* Array to store connection state per remote GPU */ 349 struct amdgpu_ualink_connection conn_state[AMDGPU_UALINK_ACCEL_MAX]; 350 351 /* List of exported ualink handles per GPU. Used to quickly traverse 352 * the list of all handles exported to a remote GPU. 353 */ 354 struct list_head exp_handles_list[AMDGPU_UALINK_ACCEL_MAX]; 355 356 /* List of imported ualink handles per GPU. Used to quickly traverse 357 * the list of all handles imported from a remote GPU. 358 */ 359 struct list_head imp_handles_list[AMDGPU_UALINK_ACCEL_MAX]; 360 361 /* WQ to manage revocation of exported memory. */ 362 struct workqueue_struct *npa_wq; 363 364 /* Memory-manager for managing NPA address space. */ 365 struct amdgpu_ualink_npa_mm npa_mm; 366 367 /* NPA-VM used on the exporter.*/ 368 struct amdgpu_vm npa_vm; 369 370 /* DRM client for UALink to manage GEM handles for NPA BOs */ 371 struct drm_client_dev client; 372 373 /* Sequence number to track the need for TLB flushes */ 374 atomic64_t last_flushed_tlb_seq; 375 376 /* Debug-only: bitmap of incoming UALink protocol messages to drop. 377 * 378 * Each bit position corresponds to a value from 379 * enum AMDGPU_UALINK_PROTOCOL_MESSAGES: 380 * BIT(AMDGPU_UALINK_NPA_REQ_MSG) drops one incoming NPA-REQ. 381 * BIT(AMDGPU_UALINK_NPA_RSP_MSG) drops one incoming NPA-RSP. 382 * 383 * On reception of a message whose corresponding bit is set, the bit 384 * is atomically cleared and the message is silently dropped. This 385 * means at most one packet per set bit is dropped; any further 386 * incoming packets of the same type are processed normally. This is 387 * intended to exercise the connection reset / recovery paths from a 388 * debugfs handle. 389 */ 390 unsigned long drop_msg_bitmap; 391 }; 392 393 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev, 394 unsigned int client_id, unsigned int src_id); 395 396 int amdgpu_ualink_sw_init(struct amdgpu_device *adev); 397 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev); 398 399 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id); 400 401 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev); 402 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev); 403 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev); 404 405 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev); 406 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev); 407 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp); 408 int amdgpu_ualink_manager_start(struct amdgpu_device *adev); 409 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev); 410 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp, 411 u32 gem_handle, 412 struct amdgpu_ualink_handle *handle_out); 413 int amdgpu_ualink_import_handle(struct drm_device *dev, 414 const struct amdgpu_ualink_handle *ualink_handle, 415 int *fd_out); 416 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data, 417 struct drm_file *filp); 418 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo); 419 420 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block); 421 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block); 422 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block); 423 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block); 424 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block); 425 426 extern const struct amdgpu_ip_block_version ualink_v1_0_ip_block; 427 #endif 428