1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2026 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/xarray.h> 26 #include <drm/drm_mm.h> 27 #include <linux/random.h> 28 #include "amdgpu.h" 29 #include "amdgpu_sdma.h" 30 #include "amdgpu_ualink.h" 31 #include "amdgpu_xgmi.h" 32 #include "amdgpu_dma_buf.h" 33 #include "psp_gfx_if.h" 34 #include <linux/math.h> 35 #include <linux/sysfs.h> 36 #include <linux/string.h> 37 38 static void deactivate_accelerator(struct amdgpu_device *adev); 39 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev, 40 u32 remote_accel_id, u32 dw0, u32 dw1, 41 u32 dw2, u32 dw3); 42 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, 43 u32 flush_type); 44 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev, 45 struct amdgpu_bo *bos[], u32 n_bos, 46 struct drm_exec *exec, 47 bool interruptible); 48 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev, 49 struct drm_exec *exec); 50 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev, 51 u32 remote_accel_id, u32 state, 52 u32 generation_count); 53 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo); 54 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev, 55 u32 remote_accel_id, u64 addr, 56 u32 size_in_pages, u32 flush_type); 57 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev); 58 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev); 59 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev, 60 u32 remote_accel_id); 61 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev, 62 u32 remote_accel_id); 63 64 #define STRIP_NPA(addr) \ 65 (((u64)(addr) & ~AMDGPU_UALINK_NPA_ADDR_GPUID_MASK)) 66 67 #define GENERATE_NPA(addr, remote_acc_id) \ 68 ((u64)(((u64)(addr)) | \ 69 ((u64)(remote_acc_id) << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT))) 70 71 static const struct drm_client_funcs ualink_client_funcs = { 72 .unregister = drm_client_release, 73 }; 74 75 static int amdgpu_ualink_drm_client_create(struct amdgpu_device *adev) 76 { 77 int ret; 78 79 ret = drm_client_init(&adev->ddev, &adev->ualink.client, "ualink", 80 &ualink_client_funcs); 81 if (ret) { 82 dev_err(adev->dev, "Failed to init UALink DRM client: %d\n", 83 ret); 84 return ret; 85 } 86 87 drm_client_register(&adev->ualink.client); 88 89 return 0; 90 } 91 92 static void amdgpu_ualink_object_fini(struct amdgpu_device *adev) 93 { 94 if (!adev->ualink.info) 95 return; 96 97 kobject_put(&adev->ualink.stations->kobj); 98 kobject_put(&adev->ualink.config->kobj); 99 kobject_put(&adev->ualink.setup->kobj); 100 kobject_put(&adev->ualink.info->kobj); 101 adev->ualink.stations = NULL; 102 adev->ualink.config = NULL; 103 adev->ualink.setup = NULL; 104 adev->ualink.info = NULL; 105 } 106 107 static bool __check_ppod_info(struct amdgpu_device *adev, 108 const struct amdgpu_ualink_info *info) 109 { 110 const struct amdgpu_ualink_ppod_info *ppod = &info->ppod; 111 112 if (ppod->size <= 0 || ppod->size > AMDGPU_UALINK_ACCEL_MAX) { 113 dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n", 114 ppod->size, AMDGPU_UALINK_ACCEL_MAX); 115 return false; 116 } 117 if (ppod->accel_id >= ppod->size) { 118 dev_dbg(adev->dev, 119 "Accelerator ID %u greater or equal pPod size %u\n", 120 ppod->accel_id, ppod->size); 121 return false; 122 } 123 124 return true; 125 } 126 127 static bool __check_vpod_info(struct amdgpu_device *adev, 128 const struct amdgpu_ualink_info *info) 129 { 130 const struct amdgpu_ualink_ppod_info *ppod = &info->ppod; 131 const struct amdgpu_ualink_vpod_info *vpod = &info->vpod; 132 unsigned int weight; 133 134 if (vpod->size == 0 || vpod->size > ppod->size) { 135 dev_warn(adev->dev, 136 "UALINK: vPod size %u out of range [1..%u]\n", 137 vpod->size, ppod->size); 138 return false; 139 } 140 if (vpod->addr_mode >= AMDGPU_UALINK_ADDR_MODE_MAX) { 141 dev_warn(adev->dev, 142 "UALINK: invalid addr mode %u\n", 143 vpod->addr_mode); 144 return false; 145 } 146 weight = 147 bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); 148 if (weight != vpod->size) { 149 dev_warn(adev->dev, 150 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n", 151 vpod->size, weight); 152 return false; 153 } 154 if (!test_bit(ppod->accel_id, vpod->active_accel_bits)) { 155 dev_warn(adev->dev, 156 "UALINK: accelerator ID %u not listed in vpod_active_accels\n", 157 ppod->accel_id); 158 return false; 159 } 160 161 return true; 162 } 163 164 static void 165 amdgpu_ualink_info_set_accel_state(struct amdgpu_device *adev, 166 struct amdgpu_ualink_info *info, 167 enum psp_gfx_ual_config_state cfg_state) 168 { 169 enum amdgpu_ualink_accel_state cur; 170 enum amdgpu_ualink_accel_state target; 171 bool ppod_validated; 172 bool vpod_validated; 173 174 if (!info) 175 return; 176 177 cur = info->accel_state; 178 179 ppod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED && 180 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 181 vpod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED && 182 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 183 184 switch (cfg_state) { 185 case UAL_CFG_IDLE: 186 return; 187 case UAL_CFG_PPOD: 188 target = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 189 break; 190 case UAL_CFG_VPOD: 191 case UAL_CFG_STATION: 192 target = AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED; 193 break; 194 case UAL_CFG_COMPLETE: 195 target = AMDGPU_UALINK_ACCEL_STATE_READY; 196 break; 197 default: 198 dev_dbg(adev->dev, "invalid configuration state %u", cfg_state); 199 return; 200 } 201 202 /* ppod stage: should be part of a ppod first */ 203 if (!ppod_validated && !__check_ppod_info(adev, info)) { 204 if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 205 info->accel_state = 206 AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 207 } else { 208 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR; 209 dev_err(adev->dev, 210 "ppod configuration is invalid, setting to error state"); 211 } 212 return; 213 } 214 if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 215 if (!ppod_validated) 216 info->accel_state = 217 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 218 return; 219 } 220 221 /* A vpod_id of 0 means the GPU is not part of any vPod */ 222 if (info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) { 223 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 224 return; 225 } 226 227 /* vpod stage: required to reach vpod_configured or ready */ 228 if (!vpod_validated && !__check_vpod_info(adev, info)) { 229 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR; 230 dev_err(adev->dev, 231 "vpod configuration is invalid, setting to error state"); 232 return; 233 } 234 if (target == AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) { 235 if (!vpod_validated) 236 info->accel_state = 237 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED; 238 return; 239 } 240 241 /* complete stage: advance to ready unless already ready/active */ 242 if (cur < AMDGPU_UALINK_ACCEL_STATE_READY || 243 cur > AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 244 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_READY; 245 } 246 247 static int amdgpu_ualink_update_vpod_config(struct amdgpu_device *adev) 248 { 249 /* TBD: Do updates/cleanup based on updated vpod configuration */ 250 return 0; 251 } 252 253 static int amdgpu_ualink_update_accel_state( 254 struct amdgpu_device *adev, enum amdgpu_ualink_accel_state prev_state, 255 u32 prev_vpod_id, enum psp_gfx_ual_config_state cfg_state) 256 { 257 dev_info(adev->dev, 258 "UALINK: update_accel_state: prev_state=%d prev_vpod_id=%u new_vpod_id=%u cfg_state=%d\n", 259 prev_state, prev_vpod_id, adev->ualink.info->vpod.id, 260 cfg_state); 261 262 /* If the device is already active and its vpod_id is unchanged, the 263 * update does not affect vpod membership. Skip the local vpod 264 * integrity check and re-activation. 265 */ 266 if (prev_state == AMDGPU_UALINK_ACCEL_STATE_ACTIVE && 267 adev->ualink.info->vpod.id == prev_vpod_id) { 268 amdgpu_ualink_update_vpod_config(adev); 269 270 /* Same vpod_id, but the member set may have changed (vPod grown 271 * or shrunk while this GPU stayed ACTIVE). If so, run the local 272 * vpod activation path: once every local peer has committed the 273 * new config (integrity passes), it bounces the affected ACTIVE 274 * peers to rebuild links/GART for the new member set. 275 */ 276 if (amdgpu_ualink_vpod_membership_changed(adev)) { 277 dev_info(adev->dev, 278 "UALINK: update_accel_state: ACTIVE vpod_id=%u membership changed, reconfiguring\n", 279 adev->ualink.info->vpod.id); 280 scoped_guard(mutex, &mgpu_info.mutex) 281 __amdgpu_ualink_activate_vpod_locked(adev); 282 } else { 283 dev_info(adev->dev, 284 "UALINK: update_accel_state: already ACTIVE, vpod_id/membership unchanged\n"); 285 } 286 return 0; 287 } 288 289 /* A new vpod_id of 0 means this GPU was removed from the vPod. */ 290 if (adev->ualink.info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) { 291 dev_info(adev->dev, 292 "UALINK: update_accel_state: vpod_id=0, removing accelerator from vPod\n"); 293 amdgpu_ualink_update_vpod_config(adev); 294 scoped_guard(mutex, &mgpu_info.mutex) 295 deactivate_accelerator(adev); 296 return 0; 297 } 298 299 /* GPU joining a new vpod should be with invalid vpod id*/ 300 scoped_guard(mutex, &mgpu_info.mutex) { 301 if (prev_vpod_id == AMDGPU_UALINK_VPOD_ID_INVALID) { 302 dev_info(adev->dev, 303 "UALINK: update_accel_state: joining vpod_id=%u\n", 304 adev->ualink.info->vpod.id); 305 amdgpu_ualink_info_set_accel_state( 306 adev, adev->ualink.info, cfg_state); 307 __amdgpu_ualink_activate_vpod_locked(adev); 308 } else { 309 /* GPU should first get removal which will set invalid vpod_id 310 * and then join a new vpod 311 */ 312 dev_err(adev->dev, 313 "Invalid vpod transition from %u to %u\n", 314 prev_vpod_id, adev->ualink.info->vpod.id); 315 return -EINVAL; 316 } 317 } 318 319 return 0; 320 } 321 322 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev) 323 { 324 enum amdgpu_ualink_accel_state prev_state; 325 enum psp_gfx_ual_config_state cfg_state; 326 u32 prev_vpod_id; 327 int r, qerr; 328 329 /* TBD: Stop ASP interrupts if driver faced an issue */ 330 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 331 u32 status; 332 333 dev_dbg(adev->dev, 334 "UALink not initialized, skipping config update\n"); 335 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 336 return psp_ual_send_completion( 337 &adev->psp, adev->ualink.psp_if_ver, 338 PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, status); 339 } 340 341 prev_state = adev->ualink.info->accel_state; 342 prev_vpod_id = adev->ualink.info->vpod.id; 343 344 qerr = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, 345 adev->ualink.info, &cfg_state); 346 347 /*TBD: find the right value of status to be sent to ASP*/ 348 r = psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 349 PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, 0); 350 if (r || qerr) 351 goto err; 352 353 r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id, 354 cfg_state); 355 if (r) 356 goto err; 357 358 return 0; 359 360 err: 361 scoped_guard(mutex, &mgpu_info.mutex) { 362 deactivate_accelerator(adev); 363 adev->ualink.info->accel_state = 364 AMDGPU_UALINK_ACCEL_STATE_ERROR; 365 } 366 dev_err(adev->dev, 367 "UALink config update failed, setting to error state"); 368 369 return r; 370 } 371 372 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev) 373 { 374 u32 status = 0; 375 376 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 377 dev_dbg(adev->dev, 378 "UALink not initialized, skipping pause update\n"); 379 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 380 goto out; 381 } 382 383 dev_dbg(adev->dev, "UALink pause command is not handled\n"); 384 385 out: 386 return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 387 PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID, 388 status); 389 } 390 391 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev) 392 { 393 u32 status = 0; 394 395 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 396 dev_dbg(adev->dev, 397 "UALink not initialized, skipping pause update\n"); 398 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 399 goto out; 400 } 401 402 dev_dbg(adev->dev, "UALink resume command is not handled\n"); 403 404 out: 405 return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 406 PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID, 407 status); 408 } 409 410 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block) 411 { 412 struct amdgpu_device *adev = ip_block->adev; 413 int r; 414 415 if (!adev->ualink.info) 416 return 0; 417 418 adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 419 r = psp_ual_get_interface_version(&adev->psp, &adev->ualink.psp_if_ver); 420 if (r) { 421 adev->ualink.psp_if_ver = 0xffffffff; 422 goto disable; 423 } 424 dev_info(adev->dev, "Found UALink interface version 0x%x\n", 425 adev->ualink.psp_if_ver); 426 427 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_HW; 428 429 return 0; 430 disable: 431 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR; 432 return 0; 433 } 434 435 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block) 436 { 437 struct amdgpu_device *adev = ip_block->adev; 438 439 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) 440 return 0; 441 442 scoped_guard(mutex, &mgpu_info.mutex) 443 deactivate_accelerator(adev); 444 445 return 0; 446 } 447 448 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block) 449 { 450 struct amdgpu_device *adev = ip_block->adev; 451 enum psp_gfx_ual_config_state cfg_state; 452 int r; 453 454 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_HW) 455 return 0; 456 457 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, 458 adev->ualink.info, &cfg_state); 459 if (r) 460 return r; 461 462 scoped_guard(mutex, &mgpu_info.mutex) { 463 amdgpu_ualink_info_set_accel_state(adev, adev->ualink.info, 464 cfg_state); 465 __amdgpu_ualink_activate_vpod_locked(adev); 466 } 467 468 r = amdgpu_ualink_drm_client_create(adev); 469 if (r) { 470 dev_err(adev->dev, "Failed to create UALink DRM client: %d\n", 471 r); 472 goto error; 473 } 474 475 /* Consider UALink initialized only at this stage */ 476 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_COMPLETE; 477 478 return 0; 479 480 error: 481 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR; 482 return r; 483 } 484 485 /**************************************************************************** 486 * UALink info and configuration in sysfs 487 * 488 * Using kobj_attribute and not device_attribute here because the UALink 489 * attributes are part of their own nested kobjects and not the amdgpu 490 * device itself or a subdevice. 491 ****************************************************************************/ 492 493 /* UALink configuration APIs are being deprecated before upstreaming and are 494 * not enabled by default. They are replaced by a unified in-band/side-band 495 * approach through the IFoE driver. Querying of the current configuration 496 * will continue to be supported in sysfs. 497 * 498 * Uncomment the following line to enable the deprecated configuration APIs 499 * for testing purposes only. The code to support these APIs will be removed 500 * in a future patch. 501 */ 502 /* #define UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS */ 503 504 #define UALINK_VALUE_SHOW(prefix, name, field, format) \ 505 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 506 struct kobj_attribute *attr,\ 507 char *buf) \ 508 { \ 509 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 510 \ 511 return sysfs_emit(buf, format"\n", info->field); \ 512 } 513 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 514 #define UALINK_VALUE_STORE(prefix, name, field, type, base) \ 515 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 516 struct kobj_attribute *attr,\ 517 const char *buf, size_t count)\ 518 { \ 519 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 520 int r; \ 521 \ 522 r = kstrto##type(buf, base, &info->field); \ 523 if (r < 0) \ 524 return r; \ 525 return count; \ 526 } 527 #endif 528 529 static ssize_t show_idbits(unsigned long *bits, unsigned int nbits, char *buf) 530 { 531 ssize_t len = 0; 532 unsigned int id; 533 534 for_each_set_bit(id, bits, nbits) 535 len += sysfs_emit_at(buf, len, "%u ", id); 536 if (len) 537 len--; 538 return len + sysfs_emit_at(buf, len, "\n"); 539 } 540 #define UALINK_IDBITS_SHOW(prefix, name, field) \ 541 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 542 struct kobj_attribute *attr,\ 543 char *buf) \ 544 { \ 545 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 546 \ 547 return show_idbits(info->field, sizeof(info->field)*8, buf); \ 548 } 549 550 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 551 static ssize_t store_idbits(unsigned long *bits, unsigned int nbits, 552 const char *buf, size_t count) 553 { 554 char *dup_buf, *str, *tok; 555 ssize_t r; 556 u32 id; 557 558 bitmap_zero(bits, nbits); 559 if (!count) 560 return 0; 561 562 dup_buf = kstrndup(buf, count, GFP_KERNEL); 563 if (unlikely(!dup_buf)) 564 return -ENOMEM; 565 str = dup_buf; 566 do { 567 str += strspn(str, " "); 568 tok = strsep(&str, ", "); 569 r = kstrtou32(tok, 10, &id); 570 if (r < 0) 571 goto err; 572 if (id >= nbits) { 573 r = -ERANGE; 574 goto err; 575 } 576 set_bit(id, bits); 577 } while (str); 578 kfree(dup_buf); 579 580 return count; 581 582 err: 583 kfree(dup_buf); 584 return r; 585 } 586 #define UALINK_IDBITS_STORE(prefix, name, field, nbits) \ 587 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 588 struct kobj_attribute *attr,\ 589 const char *buf, size_t count)\ 590 { \ 591 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 592 DECLARE_BITMAP(bits, nbits); \ 593 ssize_t r = store_idbits(bits, nbits, buf, count); \ 594 \ 595 if (r >= 0) \ 596 bitmap_copy(info->field, bits, nbits); \ 597 return r; \ 598 } 599 #endif 600 601 static ssize_t show_idarray(const u32 *array, u32 size, char *buf) 602 { 603 ssize_t len = 0; 604 unsigned int i; 605 606 for (i = 0; i < size; i++) 607 len += sysfs_emit_at(buf, len, "%u ", array[i]); 608 if (len) 609 len--; 610 return len + sysfs_emit_at(buf, len, "\n"); 611 } 612 #define UALINK_IDARRAY_SHOW(prefix, name, field, size) \ 613 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 614 struct kobj_attribute *attr,\ 615 char *buf) \ 616 { \ 617 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 618 \ 619 return show_idarray(info->field, info->size, buf); \ 620 } 621 622 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 623 static ssize_t store_idarray(u32 *array, u32 *size, u32 max_id, u32 max_size, 624 const char *buf, size_t count) 625 { 626 char *dup_buf, *str, *tok; 627 unsigned int index = 0; 628 ssize_t r; 629 u32 id; 630 631 *size = 0; 632 if (!count) 633 return 0; 634 635 dup_buf = kstrndup(buf, count, GFP_KERNEL); 636 if (unlikely(!dup_buf)) 637 return -ENOMEM; 638 str = dup_buf; 639 do { 640 str += strspn(str, " "); 641 tok = strsep(&str, ", "); 642 r = kstrtou32(tok, 10, &id); 643 if (r < 0) 644 goto err; 645 if (index >= max_size) { 646 r = -ENOSPC; 647 goto err; 648 } 649 if (id >= max_id) { 650 r = -ERANGE; 651 goto err; 652 } 653 array[index++] = id; 654 } while (str); 655 kfree(dup_buf); 656 657 *size = index; 658 return count; 659 660 err: 661 kfree(dup_buf); 662 return r; 663 } 664 #define UALINK_IDARRAY_STORE(prefix, name, field, size, max_id) \ 665 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 666 struct kobj_attribute *attr,\ 667 const char *buf, size_t count)\ 668 { \ 669 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 670 u32 new_array[ARRAY_SIZE(info->field)]; \ 671 u32 new_size; \ 672 ssize_t r = store_idarray(new_array, &new_size, max_id, \ 673 ARRAY_SIZE(new_array), buf, count); \ 674 \ 675 if (r >= 0) { \ 676 memcpy(info->field, new_array, sizeof(new_array)); \ 677 info->size = new_size; \ 678 } \ 679 return r; \ 680 } 681 #endif 682 683 #define UALINK_UUID_SHOW(prefix, name, field) \ 684 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 685 struct kobj_attribute *attr,\ 686 char *buf) \ 687 { \ 688 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 689 \ 690 return sysfs_emit(buf, "%pU\n", &info->field); \ 691 } 692 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 693 #define UALINK_UUID_STORE(prefix, name, field) \ 694 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 695 struct kobj_attribute *attr,\ 696 const char *buf, size_t count)\ 697 { \ 698 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 699 int r; \ 700 \ 701 r = uuid_parse(buf, &info->field); \ 702 return r ? r : count; \ 703 } 704 #endif 705 706 static ssize_t show_enum(u32 x, const char * const values[], unsigned int n, 707 char *buf) 708 { 709 return sysfs_emit(buf, "%s\n", x < n && values[x] ? values[x] : "invalid"); 710 } 711 #define UALINK_ENUM_SHOW(prefix, name, field) \ 712 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 713 struct kobj_attribute *attr,\ 714 char *buf) \ 715 { \ 716 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 717 \ 718 return show_enum(info->field, ualink_##name##_values, \ 719 ARRAY_SIZE(ualink_##name##_values), buf); \ 720 } 721 722 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 723 static ssize_t store_enum(u32 *x, const char * const values[], unsigned int n, 724 const char *buf, size_t count) 725 { 726 unsigned int i; 727 728 for (i = 0; i < n; i++) { 729 if (values[i] && sysfs_streq(buf, values[i])) { 730 *x = i; 731 return count; 732 } 733 } 734 return -EINVAL; 735 } 736 #define UALINK_ENUM_STORE(prefix, name, field) \ 737 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 738 struct kobj_attribute *attr,\ 739 const char *buf, size_t count)\ 740 { \ 741 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 742 \ 743 return store_enum(&info->field, ualink_##name##_values, \ 744 ARRAY_SIZE(ualink_##name##_values), \ 745 buf, count); \ 746 } 747 #endif 748 static const char * const ualink_link_type_values[] = { 749 "UALoE", "UALink" 750 }; 751 static const char * const ualink_addr_mode_values[] = { 752 "source-aliasing", "source-identification" 753 }; 754 static const char * const ualink_accel_state_values[] = { 755 [AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED] = "unconfigured", 756 [AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED] = "ppod_configured", 757 [AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED] = "vpod_configured", 758 [AMDGPU_UALINK_ACCEL_STATE_READY] = "ready", 759 [AMDGPU_UALINK_ACCEL_STATE_ACTIVE] = "active", 760 [AMDGPU_UALINK_ACCEL_STATE_ERROR] = "error", 761 }; 762 static_assert(ARRAY_SIZE(ualink_accel_state_values) == 763 AMDGPU_UALINK_ACCEL_STATE_MAX); 764 765 UALINK_ENUM_SHOW(info, link_type, link_type); 766 UALINK_VALUE_SHOW(info, accel_id, ppod.accel_id, "%u"); 767 UALINK_VALUE_SHOW(info, bandwidth, ppod.bandwidth, "%u"); 768 UALINK_VALUE_SHOW(info, latency, ppod.latency, "%u"); 769 UALINK_UUID_SHOW(info, ppod_id, ppod.id); 770 UALINK_VALUE_SHOW(info, ppod_size, ppod.size, "%u"); 771 UALINK_VALUE_SHOW(info, vpod_id, vpod.id, "%u"); 772 UALINK_VALUE_SHOW(info, vpod_size, vpod.size, "%u"); 773 UALINK_IDBITS_SHOW(info, vpod_active_accels, vpod.active_accel_bits); 774 UALINK_ENUM_SHOW(info, addr_mode, vpod.addr_mode); 775 UALINK_ENUM_SHOW(info, accel_state, accel_state); 776 UALINK_IDARRAY_SHOW(info, local_accels, local_accels, n_local_accels); 777 778 static struct amdgpu_device *ualink_xcp_kobj_to_adev(struct kobject *kobj) 779 { 780 struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp, 781 ualink.kobj); 782 783 return xcp->xcp_mgr->adev; 784 } 785 786 #define UALINK_XCP_INFO_SHOW(name) \ 787 static ssize_t ualink_xcp_info_##name##_show(struct kobject *kobj, \ 788 struct kobj_attribute *attr, char *buf) \ 789 { \ 790 struct amdgpu_device *adev = ualink_xcp_kobj_to_adev(kobj); \ 791 \ 792 return ualink_info_##name##_show(&adev->ualink.info->kobj, \ 793 attr, buf); \ 794 } 795 796 UALINK_XCP_INFO_SHOW(link_type) 797 UALINK_XCP_INFO_SHOW(accel_id) 798 UALINK_XCP_INFO_SHOW(bandwidth) 799 UALINK_XCP_INFO_SHOW(latency) 800 UALINK_XCP_INFO_SHOW(ppod_id) 801 UALINK_XCP_INFO_SHOW(ppod_size) 802 UALINK_XCP_INFO_SHOW(vpod_id) 803 UALINK_XCP_INFO_SHOW(vpod_size) 804 UALINK_XCP_INFO_SHOW(vpod_active_accels) 805 UALINK_XCP_INFO_SHOW(addr_mode) 806 UALINK_XCP_INFO_SHOW(accel_state) 807 UALINK_XCP_INFO_SHOW(local_accels) 808 809 #define UALINK_INFO_ATTR(name) __ATTR(name, 0444, ualink_info_##name##_show, NULL) 810 static struct kobj_attribute ualink_info_link_type = UALINK_INFO_ATTR(link_type); 811 static struct kobj_attribute ualink_info_accel_id = UALINK_INFO_ATTR(accel_id); 812 static struct kobj_attribute ualink_info_bandwidth = UALINK_INFO_ATTR(bandwidth); 813 static struct kobj_attribute ualink_info_latency = UALINK_INFO_ATTR(latency); 814 static struct kobj_attribute ualink_info_ppod_id = UALINK_INFO_ATTR(ppod_id); 815 static struct kobj_attribute ualink_info_ppod_size = UALINK_INFO_ATTR(ppod_size); 816 static struct kobj_attribute ualink_info_vpod_id = UALINK_INFO_ATTR(vpod_id); 817 static struct kobj_attribute ualink_info_vpod_size = UALINK_INFO_ATTR(vpod_size); 818 static struct kobj_attribute ualink_info_vpod_active_accels = UALINK_INFO_ATTR(vpod_active_accels); 819 static struct kobj_attribute ualink_info_addr_mode = UALINK_INFO_ATTR(addr_mode); 820 static struct kobj_attribute ualink_info_accel_state = UALINK_INFO_ATTR(accel_state); 821 static struct kobj_attribute ualink_info_local_accels = UALINK_INFO_ATTR(local_accels); 822 823 static const struct attribute *ualink_info_attrs[] = { 824 &ualink_info_link_type.attr, 825 &ualink_info_accel_id.attr, 826 &ualink_info_bandwidth.attr, 827 &ualink_info_latency.attr, 828 &ualink_info_ppod_id.attr, 829 &ualink_info_ppod_size.attr, 830 &ualink_info_vpod_id.attr, 831 &ualink_info_vpod_size.attr, 832 &ualink_info_vpod_active_accels.attr, 833 &ualink_info_addr_mode.attr, 834 &ualink_info_accel_state.attr, 835 &ualink_info_local_accels.attr, 836 NULL 837 }; 838 839 #define UALINK_XCP_INFO_ATTR(name) \ 840 __ATTR(name, 0444, ualink_xcp_info_##name##_show, NULL) 841 static struct kobj_attribute ualink_xcp_info_link_type = UALINK_XCP_INFO_ATTR(link_type); 842 static struct kobj_attribute ualink_xcp_info_accel_id = UALINK_XCP_INFO_ATTR(accel_id); 843 static struct kobj_attribute ualink_xcp_info_bandwidth = UALINK_XCP_INFO_ATTR(bandwidth); 844 static struct kobj_attribute ualink_xcp_info_latency = UALINK_XCP_INFO_ATTR(latency); 845 static struct kobj_attribute ualink_xcp_info_ppod_id = UALINK_XCP_INFO_ATTR(ppod_id); 846 static struct kobj_attribute ualink_xcp_info_ppod_size = UALINK_XCP_INFO_ATTR(ppod_size); 847 static struct kobj_attribute ualink_xcp_info_vpod_id = UALINK_XCP_INFO_ATTR(vpod_id); 848 static struct kobj_attribute ualink_xcp_info_vpod_size = UALINK_XCP_INFO_ATTR(vpod_size); 849 static struct kobj_attribute ualink_xcp_info_vpod_active_accels = UALINK_XCP_INFO_ATTR(vpod_active_accels); 850 static struct kobj_attribute ualink_xcp_info_addr_mode = UALINK_XCP_INFO_ATTR(addr_mode); 851 static struct kobj_attribute ualink_xcp_info_accel_state = UALINK_XCP_INFO_ATTR(accel_state); 852 static struct kobj_attribute ualink_xcp_info_local_accels = UALINK_XCP_INFO_ATTR(local_accels); 853 854 static struct attribute *ualink_xcp_info_attrs[] = { 855 &ualink_xcp_info_link_type.attr, 856 &ualink_xcp_info_accel_id.attr, 857 &ualink_xcp_info_bandwidth.attr, 858 &ualink_xcp_info_latency.attr, 859 &ualink_xcp_info_ppod_id.attr, 860 &ualink_xcp_info_ppod_size.attr, 861 &ualink_xcp_info_vpod_id.attr, 862 &ualink_xcp_info_vpod_size.attr, 863 &ualink_xcp_info_vpod_active_accels.attr, 864 &ualink_xcp_info_addr_mode.attr, 865 &ualink_xcp_info_accel_state.attr, 866 &ualink_xcp_info_local_accels.attr, 867 NULL 868 }; 869 870 static umode_t ualink_xcp_info_is_visible(struct kobject *kobj, 871 struct attribute *attr, int n) 872 { 873 struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp, 874 ualink.kobj); 875 876 if (!xcp->valid) 877 return 0; 878 879 return attr->mode; 880 } 881 882 static const struct attribute_group ualink_xcp_info_group = { 883 .attrs = ualink_xcp_info_attrs, 884 .is_visible = ualink_xcp_info_is_visible, 885 }; 886 887 static const struct kobj_type ualink_xcp_info_ktype = { 888 .sysfs_ops = &kobj_sysfs_ops 889 }; 890 891 static void ualink_info_release(struct kobject *kobj) 892 { 893 kfree(to_ualink_info(kobj)); 894 } 895 896 static const struct kobj_type ualink_info_ktype = { 897 .release = ualink_info_release, 898 .sysfs_ops = &kobj_sysfs_ops 899 }; 900 901 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 902 UALINK_VALUE_SHOW(ppod_setup, accel_id, ppod.accel_id, "%u"); 903 UALINK_VALUE_SHOW(ppod_setup, bandwidth, ppod.bandwidth, "%u"); 904 UALINK_VALUE_SHOW(ppod_setup, latency, ppod.latency, "%u"); 905 UALINK_UUID_SHOW(ppod_setup, ppod_id, ppod.id); 906 UALINK_VALUE_SHOW(ppod_setup, ppod_size, ppod.size, "%u"); 907 UALINK_IDARRAY_SHOW(ppod_setup, local_accels, local_accels, n_local_accels); 908 909 UALINK_VALUE_STORE(ppod_setup, accel_id, ppod.accel_id, u32, 10); 910 UALINK_VALUE_STORE(ppod_setup, bandwidth, ppod.bandwidth, u32, 10); 911 UALINK_VALUE_STORE(ppod_setup, latency, ppod.latency, u32, 10); 912 UALINK_UUID_STORE(ppod_setup, ppod_id, ppod.id); 913 UALINK_VALUE_STORE(ppod_setup, ppod_size, ppod.size, u32, 10); 914 UALINK_IDARRAY_STORE(ppod_setup, local_accels, local_accels, n_local_accels, 915 AMDGPU_UALINK_ACCEL_MAX); 916 917 static enum amdgpu_ualink_accel_state 918 check_ppod_state(struct amdgpu_device *adev, 919 const struct amdgpu_ualink_ppod_setup *setup) 920 { 921 if (setup->ppod.size <= 0 || setup->ppod.size > AMDGPU_UALINK_ACCEL_MAX) { 922 dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n", 923 setup->ppod.size, AMDGPU_UALINK_ACCEL_MAX); 924 return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 925 } 926 if (setup->ppod.accel_id >= setup->ppod.size) { 927 dev_dbg(adev->dev, "Accelerator ID %u greater or equal pPod size %u\n", 928 setup->ppod.accel_id, setup->ppod.size); 929 return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 930 } 931 return AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 932 } 933 934 static ssize_t ualink_ppod_setup_commit_store(struct kobject *kobj, 935 struct kobj_attribute *attr, 936 const char *buf, size_t count) 937 { 938 struct amdgpu_ualink_ppod_setup *setup = to_ualink_ppod_setup(kobj); 939 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 940 struct device *dev = kobj_to_dev(info->kobj.parent); 941 struct drm_device *ddev = dev_get_drvdata(dev); 942 struct amdgpu_device *adev = drm_to_adev(ddev); 943 int r; 944 945 if (!sysfs_streq(buf, "true")) 946 return -EINVAL; 947 948 r = psp_ual_set_ppod_config(&adev->psp, adev->ualink.psp_if_ver, 949 setup); 950 if (r) 951 return r; 952 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL); 953 if (r) 954 return r; 955 956 /* 957 * Hold mgpu_info.mutex to serialize with activate_local_vpod() in 958 * vpod_config_commit_store which also holds this lock. Without it, 959 * deactivate_accelerator() can race with activate_accelerator(), 960 * causing concurrent vm_fini / vm_init on the same NPA VM. 961 */ 962 mutex_lock(&mgpu_info.mutex); 963 if (info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_READY) 964 deactivate_accelerator(adev); 965 info->accel_state = check_ppod_state(adev, setup); 966 /* PPOD is expected to be configured first */ 967 info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID; 968 mutex_unlock(&mgpu_info.mutex); 969 970 /* TODO: If accel_state was ACTIVE, reset all connections */ 971 972 return count; 973 } 974 975 #define UALINK_PPOD_SETUP_ATTR(name) __ATTR(name, 0600, \ 976 ualink_ppod_setup_##name##_show, \ 977 ualink_ppod_setup_##name##_store) 978 static struct kobj_attribute ualink_ppod_setup_accel_id = UALINK_PPOD_SETUP_ATTR(accel_id); 979 static struct kobj_attribute ualink_ppod_setup_bandwidth = UALINK_PPOD_SETUP_ATTR(bandwidth); 980 static struct kobj_attribute ualink_ppod_setup_latency = UALINK_PPOD_SETUP_ATTR(latency); 981 static struct kobj_attribute ualink_ppod_setup_ppod_id = UALINK_PPOD_SETUP_ATTR(ppod_id); 982 static struct kobj_attribute ualink_ppod_setup_ppod_size = UALINK_PPOD_SETUP_ATTR(ppod_size); 983 static struct kobj_attribute ualink_ppod_setup_local_accels = UALINK_PPOD_SETUP_ATTR(local_accels); 984 static struct kobj_attribute ualink_ppod_setup_commit = __ATTR(commit, 0200, NULL, 985 ualink_ppod_setup_commit_store); 986 987 static const struct attribute *ualink_ppod_setup_attrs[] = { 988 &ualink_ppod_setup_accel_id.attr, 989 &ualink_ppod_setup_bandwidth.attr, 990 &ualink_ppod_setup_latency.attr, 991 &ualink_ppod_setup_ppod_id.attr, 992 &ualink_ppod_setup_ppod_size.attr, 993 &ualink_ppod_setup_local_accels.attr, 994 &ualink_ppod_setup_commit.attr, 995 NULL 996 }; 997 #endif 998 999 static void ualink_ppod_setup_release(struct kobject *kobj) 1000 { 1001 kfree(to_ualink_ppod_setup(kobj)); 1002 } 1003 1004 static const struct kobj_type ualink_ppod_setup_ktype = { 1005 .release = ualink_ppod_setup_release, 1006 .sysfs_ops = &kobj_sysfs_ops 1007 }; 1008 1009 static struct amdgpu_device *find_peer_adev(unsigned int accel_id) 1010 { 1011 unsigned int i; 1012 1013 for (i = 0; i < mgpu_info.num_gpu; i++) { 1014 struct amdgpu_device *peer_adev = mgpu_info.gpu_ins[i].adev; 1015 1016 if (peer_adev->ualink.info && 1017 peer_adev->ualink.info->ppod.accel_id == accel_id) 1018 return peer_adev; 1019 } 1020 1021 return NULL; 1022 } 1023 1024 static bool amdgpu_ualink_is_local_accel(struct amdgpu_device *adev, 1025 u32 accel_id) 1026 { 1027 struct amdgpu_ualink_info *info = adev->ualink.info; 1028 unsigned int i; 1029 1030 if (!info) 1031 return false; 1032 1033 for (i = 0; i < info->n_local_accels; i++) { 1034 if (info->local_accels[i] == accel_id) 1035 return true; 1036 } 1037 return false; 1038 } 1039 1040 static void activate_accelerator(struct amdgpu_device *adev) 1041 { 1042 int r; 1043 1044 if (adev->ualink.info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 1045 return; 1046 1047 dev_info(adev->dev, 1048 "UALINK: activating accelerator accel_id=%u (accel_state=%d)\n", 1049 adev->ualink.info->ppod.accel_id, 1050 adev->ualink.info->accel_state); 1051 1052 /* Enable incoming NPA address translation with NPA VMID */ 1053 r = psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver, 1054 adev->vm_manager.npa_vmid, true); 1055 if (r) { 1056 dev_err(adev->dev, "Failed to set NPA config\n"); 1057 return; 1058 } 1059 1060 r = amdgpu_ualink_manager_start(adev); 1061 if (r) { 1062 dev_err(adev->dev, "Failed to start UALink manager\n"); 1063 return; 1064 } 1065 1066 r = amdgpu_ualink_sw_init(adev); 1067 if (r) { 1068 dev_err(adev->dev, "Failed to init UALink sw r=%d\n", r); 1069 amdgpu_ualink_manager_stop(adev); 1070 return; 1071 } 1072 1073 adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 1074 dev_info(adev->dev, 1075 "UALINK: accelerator accel_id=%u is now ACTIVE\n", 1076 adev->ualink.info->ppod.accel_id); 1077 } 1078 1079 static void deactivate_accelerator(struct amdgpu_device *adev) 1080 { 1081 if (adev->ualink.info->accel_state < AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 1082 return; 1083 1084 dev_info(adev->dev, 1085 "UALINK: deactivating accelerator accel_id=%u (accel_state=%d -> PPOD_CONFIGURED)\n", 1086 adev->ualink.info->ppod.accel_id, 1087 adev->ualink.info->accel_state); 1088 1089 /* Disable incoming NPA address translation with NPA VMID */ 1090 psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver, 1091 adev->vm_manager.npa_vmid, false); 1092 /* ignore return value */ 1093 adev->ualink.info->accel_state = 1094 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 1095 1096 amdgpu_ualink_sw_fini(adev); 1097 amdgpu_ualink_manager_stop(adev); 1098 } 1099 1100 static void activate_local_vpod(struct amdgpu_device *adev) 1101 { 1102 struct amdgpu_ualink_info *info = adev->ualink.info; 1103 struct amdgpu_device *peer_adev; 1104 unsigned int accel_id; 1105 unsigned int i; 1106 1107 for (i = 0; i < info->n_local_accels; i++) { 1108 accel_id = info->local_accels[i]; 1109 1110 peer_adev = find_peer_adev(accel_id); 1111 if (WARN_ON(!peer_adev || !peer_adev->ualink.info)) 1112 /* info->local_accels is corrupted? */ 1113 continue; 1114 1115 /* Bring the peer up to match the current vPod membership. 1116 * 1117 * A peer that is not yet ACTIVE (or whose ACTIVE membership 1118 * changed on a grow/shrink) has its remote metadata, links and 1119 * GART mappings built for a stale member set (or none at all). 1120 * Firmware only accepts a full metadata reload while halted, so 1121 * surgical per-peer deltas are not possible: fully bounce the 1122 * accelerator. deactivate_accelerator() is a no-op when the peer 1123 * is not ACTIVE, so this handles first-time bring-up too. 1124 * Unchanged ACTIVE peers are left untouched. 1125 */ 1126 if (amdgpu_ualink_vpod_membership_changed(peer_adev)) { 1127 dev_info(peer_adev->dev, 1128 "UALINK: (re)configuring vpod for accel_id=%u\n", 1129 peer_adev->ualink.info->ppod.accel_id); 1130 deactivate_accelerator(peer_adev); 1131 activate_accelerator(peer_adev); 1132 } 1133 } 1134 } 1135 1136 static inline bool __is_vpod_peer(struct amdgpu_ualink_info *info, 1137 struct amdgpu_ualink_info *peer_info) 1138 { 1139 return peer_info->vpod.id == info->vpod.id && 1140 uuid_equal(&peer_info->ppod.id, &info->ppod.id); 1141 } 1142 1143 static int __check_local_vpod_integrity(struct amdgpu_device *adev) 1144 { 1145 DECLARE_BITMAP(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX); 1146 struct amdgpu_ualink_info *info = adev->ualink.info; 1147 u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX]; 1148 struct amdgpu_ualink_info *peer_info; 1149 struct amdgpu_device *peer_adev; 1150 unsigned int i, n_local_accels; 1151 unsigned int accel_id; 1152 /* Check that all local accelerators listed in vpod_active_accels have 1153 * matching pod IDs 1154 */ 1155 bitmap_zero(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX); 1156 n_local_accels = 0; 1157 __set_bit(info->ppod.accel_id, local_accel_ids); 1158 local_accels[n_local_accels++] = info->ppod.accel_id; 1159 1160 for (i = 0; i < mgpu_info.num_gpu; i++) { 1161 peer_adev = mgpu_info.gpu_ins[i].adev; 1162 if (peer_adev == adev || !peer_adev->ualink.info) 1163 continue; 1164 1165 peer_info = peer_adev->ualink.info; 1166 /* peer device ppod not configured */ 1167 if (peer_info->accel_state < 1168 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 1169 dev_info(adev->dev, 1170 "UALINK: integrity EAGAIN: peer accel_id=%u not ppod-configured (accel_state=%d)\n", 1171 peer_info->ppod.accel_id, 1172 peer_info->accel_state); 1173 return -EAGAIN; 1174 } 1175 1176 accel_id = peer_info->ppod.accel_id; 1177 if (!test_bit(accel_id, info->vpod.active_accel_bits)) 1178 continue; 1179 /* peer device vpod not configured */ 1180 if (peer_info->accel_state < 1181 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) { 1182 dev_info(adev->dev, 1183 "UALINK: integrity EAGAIN: vpod peer accel_id=%u not vpod-configured (accel_state=%d)\n", 1184 accel_id, peer_info->accel_state); 1185 return -EAGAIN; 1186 } 1187 1188 if (!uuid_equal(&peer_info->ppod.id, &info->ppod.id)) { 1189 dev_warn(adev->dev, 1190 "UALINK: integrity fail: peer %u ppod_id doesn't match: %pU != %pU\n", 1191 peer_info->ppod.accel_id, &peer_info->ppod.id, 1192 &info->ppod.id); 1193 return -EINVAL; 1194 } 1195 1196 if (peer_info->ppod.size != info->ppod.size) { 1197 dev_warn(adev->dev, 1198 "UALINK: integrity fail: peer %u ppod_size doesn't match: %u != %u\n", 1199 accel_id, peer_info->ppod.size, 1200 info->ppod.size); 1201 return -EINVAL; 1202 } 1203 1204 if (peer_info->vpod.id != info->vpod.id) { 1205 dev_warn(adev->dev, 1206 "UALINK: integrity fail: peer %u vpod_id doesn't match: %u != %u\n", 1207 accel_id, peer_info->vpod.id, info->vpod.id); 1208 return -EINVAL; 1209 } 1210 if (peer_info->vpod.size != info->vpod.size) { 1211 dev_warn(adev->dev, 1212 "UALINK: integrity fail: peer %u vpod_size doesn't match: %u != %u\n", 1213 accel_id, peer_info->vpod.size, 1214 info->vpod.size); 1215 return -EINVAL; 1216 } 1217 if (peer_info->vpod.addr_mode != info->vpod.addr_mode) { 1218 dev_warn(adev->dev, 1219 "UALINK: integrity fail: peer %u addr_mode doesn't match: %u != %u\n", 1220 accel_id, peer_info->vpod.addr_mode, 1221 info->vpod.addr_mode); 1222 return -EINVAL; 1223 } 1224 if (!bitmap_equal(peer_info->vpod.active_accel_bits, 1225 info->vpod.active_accel_bits, 1226 AMDGPU_UALINK_ACCEL_MAX)) { 1227 dev_warn(adev->dev, 1228 "UALINK: integrity fail: peer %u vpod_active_accels don't match\n", 1229 accel_id); 1230 return -EINVAL; 1231 } 1232 1233 if (__test_and_set_bit(accel_id, local_accel_ids)) { 1234 dev_warn(adev->dev, 1235 "UALINK: integrity fail: duplicate accel_id %u among local vpod peers\n", 1236 accel_id); 1237 return -EINVAL; 1238 } 1239 local_accels[n_local_accels++] = accel_id; 1240 } 1241 1242 for (i = 0; i < mgpu_info.num_gpu; i++) { 1243 peer_adev = mgpu_info.gpu_ins[i].adev; 1244 peer_info = peer_adev->ualink.info; 1245 1246 if (!peer_info) 1247 continue; 1248 if (peer_adev != adev && !__is_vpod_peer(info, peer_info)) 1249 continue; 1250 1251 peer_info->n_local_accels = n_local_accels; 1252 memcpy(peer_info->local_accels, local_accels, 1253 sizeof(local_accels)); 1254 } 1255 1256 return 0; 1257 } 1258 1259 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev) 1260 { 1261 int ret; 1262 1263 if (adev->ualink.info->accel_state < 1264 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED || 1265 adev->ualink.info->accel_state == 1266 AMDGPU_UALINK_ACCEL_STATE_ERROR) 1267 return; 1268 1269 ret = __check_local_vpod_integrity(adev); 1270 if (ret && ret != -EAGAIN) { 1271 dev_err(adev->dev, 1272 "Local vpod integrity check failed: %d\n", ret); 1273 return; 1274 } 1275 if (ret == -EAGAIN) { 1276 dev_info(adev->dev, 1277 "UALINK: activate deferred, waiting for local vpod peers to reach VPOD_CONFIGURED\n"); 1278 return; 1279 } 1280 1281 dev_info(adev->dev, 1282 "UALINK: integrity OK, applying local vpod (%u local accels)\n", 1283 adev->ualink.info->n_local_accels); 1284 activate_local_vpod(adev); 1285 } 1286 1287 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1288 UALINK_VALUE_SHOW(vpod_config, vpod_id, vpod.id, "%u"); 1289 UALINK_VALUE_SHOW(vpod_config, vpod_size, vpod.size, "%u"); 1290 UALINK_IDBITS_SHOW(vpod_config, vpod_active_accels, vpod.active_accel_bits); 1291 UALINK_ENUM_SHOW(vpod_config, addr_mode, vpod.addr_mode); 1292 1293 UALINK_VALUE_STORE(vpod_config, vpod_id, vpod.id, u32, 10); 1294 UALINK_VALUE_STORE(vpod_config, vpod_size, vpod.size, u32, 10); 1295 UALINK_IDBITS_STORE(vpod_config, vpod_active_accels, vpod.active_accel_bits, 1296 AMDGPU_UALINK_ACCEL_MAX); 1297 UALINK_ENUM_STORE(vpod_config, addr_mode, vpod.addr_mode); 1298 1299 static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj, 1300 struct kobj_attribute *attr, 1301 const char *buf, size_t count) 1302 { 1303 struct amdgpu_ualink_vpod_config *config = to_ualink_vpod_config(kobj); 1304 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 1305 struct device *dev = kobj_to_dev(info->kobj.parent); 1306 struct drm_device *ddev = dev_get_drvdata(dev); 1307 struct amdgpu_device *adev = drm_to_adev(ddev); 1308 enum amdgpu_ualink_accel_state prev_state; 1309 u32 prev_vpod_id; 1310 int r; 1311 1312 if (!sysfs_streq(buf, "true")) 1313 return -EINVAL; 1314 1315 dev_dbg(adev->dev, 1316 "UALINK: vpod-commit enter: accel_state=%d staged vpod_id=%u vpod_size=%u addr_mode=%u\n", 1317 info->accel_state, config->vpod.id, config->vpod.size, 1318 config->vpod.addr_mode); 1319 1320 if (info->accel_state < AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 1321 dev_warn(adev->dev, 1322 "UALINK: vpod-commit rejected, ppod not configured (accel_state=%d)\n", 1323 info->accel_state); 1324 return -EINVAL; 1325 } 1326 1327 prev_state = info->accel_state; 1328 prev_vpod_id = info->vpod.id; 1329 r = psp_ual_set_vpod_config(&adev->psp, adev->ualink.psp_if_ver, 1330 config); 1331 if (r) { 1332 dev_warn(adev->dev, 1333 "UALINK: vpod-commit psp_ual_set_vpod_config failed: %d\n", 1334 r); 1335 return r; 1336 } 1337 1338 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL); 1339 if (r) { 1340 dev_warn(adev->dev, 1341 "UALINK: vpod-commit psp_ual_query_info failed: %d\n", 1342 r); 1343 return r; 1344 } 1345 1346 dev_dbg(adev->dev, 1347 "UALINK: vpod-commit fw read-back: vpod_id=%u vpod_size=%u accel_id=%u ppod.size=%u addr_mode=%u\n", 1348 info->vpod.id, info->vpod.size, info->ppod.accel_id, 1349 info->ppod.size, info->vpod.addr_mode); 1350 1351 if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID && 1352 !__check_vpod_info(adev, info)) { 1353 dev_err(adev->dev, 1354 "UALINK: vpod-commit __check_vpod_info() failed: %d\n", 1355 r); 1356 return -EINVAL; 1357 } 1358 1359 /* The integrity check makes sure each new GPU is consistent with the 1360 * other GPUs already in the vPod. All known local GPUs can become 1361 * "ready" at the same time. 1362 * 1363 * Misconfiguration of one GPU does not reduce the state of other GPUs 1364 * already in the vPod. GPU is intentionally not put to ERROR state if 1365 * misconfiguration occurs. 1366 */ 1367 r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id, 1368 UAL_CFG_VPOD); 1369 if (r) 1370 return r; 1371 1372 return count; 1373 } 1374 1375 #define UALINK_VPOD_CONFIG_ATTR(name) __ATTR(name, 0600, \ 1376 ualink_vpod_config_##name##_show, \ 1377 ualink_vpod_config_##name##_store) 1378 static struct kobj_attribute ualink_vpod_config_vpod_id = UALINK_VPOD_CONFIG_ATTR(vpod_id); 1379 static struct kobj_attribute ualink_vpod_config_vpod_size = UALINK_VPOD_CONFIG_ATTR(vpod_size); 1380 static struct kobj_attribute ualink_vpod_config_vpod_active_accels = 1381 UALINK_VPOD_CONFIG_ATTR(vpod_active_accels); 1382 static struct kobj_attribute ualink_vpod_config_addr_mode = UALINK_VPOD_CONFIG_ATTR(addr_mode); 1383 static struct kobj_attribute ualink_vpod_config_commit = __ATTR(commit, 0200, NULL, 1384 ualink_vpod_config_commit_store); 1385 1386 static const struct attribute *ualink_vpod_config_attrs[] = { 1387 &ualink_vpod_config_vpod_id.attr, 1388 &ualink_vpod_config_vpod_size.attr, 1389 &ualink_vpod_config_vpod_active_accels.attr, 1390 &ualink_vpod_config_addr_mode.attr, 1391 &ualink_vpod_config_commit.attr, 1392 NULL 1393 }; 1394 #endif 1395 1396 static void ualink_vpod_config_release(struct kobject *kobj) 1397 { 1398 kfree(to_ualink_vpod_config(kobj)); 1399 } 1400 1401 static const struct kobj_type ualink_vpod_config_ktype = { 1402 .release = ualink_vpod_config_release, 1403 .sysfs_ops = &kobj_sysfs_ops 1404 }; 1405 1406 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1407 UALINK_VALUE_SHOW(station_config, flags, flags, "0x%x"); 1408 1409 UALINK_VALUE_STORE(station_config, flags, flags, u8, 16); 1410 1411 static size_t lane_bitmap_to_string(char *str, 1412 const struct amdgpu_ualink_station_config *stations) 1413 { 1414 size_t i; 1415 1416 for (i = 0; i < stations->n_stations; i++) 1417 str[i] = stations->lane_en_bitmap[i] + 1418 (stations->lane_en_bitmap[i] < 10 ? '0' : 'a' - 10); 1419 str[i] = '\0'; 1420 1421 return i; 1422 } 1423 static ssize_t 1424 ualink_station_config_lane_en_bitmap_show(struct kobject *kobj, 1425 struct kobj_attribute *attr, 1426 char *buf) 1427 { 1428 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1429 char str[AMDGPU_UALINK_STATIONS_MAX + 1]; 1430 1431 lane_bitmap_to_string(str, stations); 1432 return sysfs_emit(buf, "%s\n", str); 1433 } 1434 static ssize_t 1435 ualink_station_config_lane_en_bitmap_store(struct kobject *kobj, 1436 struct kobj_attribute *attr, 1437 const char *buf, size_t count) 1438 { 1439 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1440 u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX]; 1441 u32 n_stations = 0; 1442 bool end = false; 1443 size_t i; 1444 1445 for (i = 0; i < count; i++) { 1446 /* Accept any number of \n in the end */ 1447 if (buf[i] == '\n') { 1448 end = true; 1449 continue; 1450 } 1451 /* Don't accept any more data after \n */ 1452 if (end) 1453 return -EINVAL; 1454 /* Don't accept more data than the array size */ 1455 if (i >= sizeof(lane_en_bitmap)) 1456 return -ENOSPC; 1457 /* Accept hex digits */ 1458 if (buf[i] >= '0' && buf[i] <= '9') 1459 lane_en_bitmap[n_stations++] = buf[i] - '0'; 1460 else if (buf[i] >= 'a' && buf[i] <= 'f') 1461 lane_en_bitmap[n_stations++] = buf[i] + 10 - 'a'; 1462 else if (buf[i] >= 'A' && buf[i] <= 'F') 1463 lane_en_bitmap[n_stations++] = buf[i] + 10 - 'A'; 1464 else 1465 return -EINVAL; 1466 } 1467 1468 memcpy(stations->lane_en_bitmap, lane_en_bitmap, sizeof(lane_en_bitmap)); 1469 stations->n_stations = n_stations; 1470 return i; 1471 } 1472 1473 static ssize_t ualink_station_config_commit_store(struct kobject *kobj, 1474 struct kobj_attribute *attr, 1475 const char *buf, size_t count) 1476 { 1477 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1478 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 1479 struct device *dev = kobj_to_dev(info->kobj.parent); 1480 struct drm_device *ddev = dev_get_drvdata(dev); 1481 struct amdgpu_device *adev = drm_to_adev(ddev); 1482 int r; 1483 1484 /* DF reconfiguration does not interact with accelerator state */ 1485 1486 if (!sysfs_streq(buf, "true")) 1487 return -EINVAL; 1488 1489 r = psp_ual_set_station_config(&adev->psp, adev->ualink.psp_if_ver, stations); 1490 if (r) 1491 return r; 1492 1493 return count; 1494 } 1495 1496 #define UALINK_STATION_CONFIG_ATTR(name) __ATTR(name, 0600, \ 1497 ualink_station_config_##name##_show, \ 1498 ualink_station_config_##name##_store) 1499 static struct kobj_attribute ualink_station_config_flags = UALINK_STATION_CONFIG_ATTR(flags); 1500 static struct kobj_attribute ualink_station_config_lane_en_bitmap = 1501 UALINK_STATION_CONFIG_ATTR(lane_en_bitmap); 1502 static struct kobj_attribute ualink_station_config_commit = 1503 __ATTR(commit, 0200, NULL, ualink_station_config_commit_store); 1504 1505 static const struct attribute *ualink_station_config_attrs[] = { 1506 &ualink_station_config_flags.attr, 1507 &ualink_station_config_lane_en_bitmap.attr, 1508 &ualink_station_config_commit.attr, 1509 NULL 1510 }; 1511 #endif 1512 1513 static void ualink_station_config_release(struct kobject *kobj) 1514 { 1515 kfree(to_ualink_station_config(kobj)); 1516 } 1517 1518 static const struct kobj_type ualink_station_config_ktype = { 1519 .release = ualink_station_config_release, 1520 .sysfs_ops = &kobj_sysfs_ops 1521 }; 1522 1523 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block) 1524 { 1525 struct amdgpu_ualink_station_config *stations; 1526 struct amdgpu_ualink_vpod_config *vpod_config; 1527 struct amdgpu_ualink_ppod_setup *ppod_setup; 1528 struct amdgpu_device *adev = ip_block->adev; 1529 struct amdgpu_ualink_info *info; 1530 1531 info = kzalloc(sizeof(*info), GFP_KERNEL); 1532 ppod_setup = kzalloc(sizeof(*ppod_setup), GFP_KERNEL); 1533 vpod_config = kzalloc(sizeof(*vpod_config), GFP_KERNEL); 1534 stations = kzalloc(sizeof(*stations), GFP_KERNEL); 1535 if (!info || !ppod_setup || !vpod_config || !stations) { 1536 kfree(info); 1537 kfree(ppod_setup); 1538 kfree(vpod_config); 1539 kfree(stations); 1540 return -ENOMEM; 1541 } 1542 1543 info->ppod.accel_id = 0xffffffff; 1544 info->ppod.bandwidth = 0xffffffff; 1545 info->ppod.latency = 0xffffffff; 1546 info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID; 1547 info->vpod.addr_mode = AMDGPU_UALINK_ADDR_MODE_MAX; 1548 1549 /* 1550 * Initialize the kobjects here so their lifetime is tied to the UALink 1551 * manager software state. amdgpu_ualink_object_fini() drops the final 1552 * reference via kobject_put(). 1553 */ 1554 kobject_init(&info->kobj, &ualink_info_ktype); 1555 kobject_init(&ppod_setup->kobj, &ualink_ppod_setup_ktype); 1556 kobject_init(&vpod_config->kobj, &ualink_vpod_config_ktype); 1557 kobject_init(&stations->kobj, &ualink_station_config_ktype); 1558 1559 adev->ualink.info = info; 1560 adev->ualink.setup = ppod_setup; 1561 adev->ualink.config = vpod_config; 1562 adev->ualink.stations = stations; 1563 1564 return 0; 1565 } 1566 1567 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block) 1568 { 1569 amdgpu_ualink_object_fini(ip_block->adev); 1570 return 0; 1571 } 1572 1573 static int ualink_kobj_add(struct kobject *kobj, struct kobject *parent, 1574 const char *name, const struct attribute **attrs) 1575 { 1576 int r; 1577 1578 r = kobject_add(kobj, parent, "%s", name); 1579 if (r) 1580 return r; 1581 r = sysfs_create_files(kobj, attrs); 1582 if (r) 1583 kobject_del(kobj); 1584 1585 return r; 1586 } 1587 1588 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev); 1589 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev); 1590 1591 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev) 1592 { 1593 struct amdgpu_ualink_info *info = adev->ualink.info; 1594 int r; 1595 1596 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) 1597 return 0; 1598 1599 /* ualink parent node */ 1600 r = ualink_kobj_add(&info->kobj, &adev->dev->kobj, "ualink", 1601 ualink_info_attrs); 1602 if (r) 1603 goto err; 1604 1605 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1606 r = ualink_kobj_add(&adev->ualink.setup->kobj, &info->kobj, "setup", 1607 ualink_ppod_setup_attrs); 1608 if (r) 1609 goto err_info; 1610 1611 r = ualink_kobj_add(&adev->ualink.config->kobj, &info->kobj, "config", 1612 ualink_vpod_config_attrs); 1613 if (r) 1614 goto err_setup; 1615 1616 r = ualink_kobj_add(&adev->ualink.stations->kobj, &info->kobj, 1617 "stations", ualink_station_config_attrs); 1618 if (r) 1619 goto err_config; 1620 #else 1621 r = kobject_add(&adev->ualink.setup->kobj, &info->kobj, "%s", 1622 "setup"); 1623 if (r) 1624 goto err_info; 1625 1626 r = kobject_add(&adev->ualink.config->kobj, &info->kobj, "%s", 1627 "config"); 1628 if (r) 1629 goto err_setup; 1630 1631 r = kobject_add(&adev->ualink.stations->kobj, &info->kobj, "%s", 1632 "stations"); 1633 if (r) 1634 goto err_config; 1635 #endif 1636 1637 amdgpu_ualink_xcp_sysfs_init(adev); 1638 adev->ualink.sysfs_init = true; 1639 1640 return 0; 1641 1642 err_config: 1643 kobject_del(&adev->ualink.config->kobj); 1644 err_setup: 1645 kobject_del(&adev->ualink.setup->kobj); 1646 err_info: 1647 kobject_del(&info->kobj); 1648 err: 1649 dev_warn(adev->dev, "Failed to create UALink sysfs: %d\n", r); 1650 return 0; 1651 } 1652 1653 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev) 1654 { 1655 if (!adev->ualink.sysfs_init) 1656 return; 1657 1658 amdgpu_ualink_xcp_sysfs_fini(adev); 1659 kobject_del(&adev->ualink.stations->kobj); 1660 kobject_del(&adev->ualink.config->kobj); 1661 kobject_del(&adev->ualink.setup->kobj); 1662 kobject_del(&adev->ualink.info->kobj); 1663 adev->ualink.sysfs_init = false; 1664 } 1665 1666 static int amdgpu_ualink_xcp_sysfs_add(struct amdgpu_xcp *xcp) 1667 { 1668 struct amdgpu_device *adev = xcp->xcp_mgr->adev; 1669 int r; 1670 1671 r = kobject_init_and_add(&xcp->ualink.kobj, &ualink_xcp_info_ktype, 1672 &xcp->ddev->dev->kobj, "ualink"); 1673 if (r) 1674 goto err; 1675 1676 r = sysfs_create_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1677 if (r) 1678 goto err; 1679 1680 /* pin info so it outlives this node regardless of teardown order */ 1681 kobject_get(&adev->ualink.info->kobj); 1682 xcp->ualink.sysfs = true; 1683 return 0; 1684 err: 1685 kobject_put(&xcp->ualink.kobj); 1686 return r; 1687 } 1688 1689 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp) 1690 { 1691 if (!xcp->ualink.sysfs) 1692 return; 1693 1694 sysfs_update_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1695 } 1696 1697 static void amdgpu_ualink_xcp_sysfs_remove(struct amdgpu_xcp *xcp) 1698 { 1699 struct amdgpu_device *adev = xcp->xcp_mgr->adev; 1700 1701 if (!xcp->ualink.sysfs) 1702 return; 1703 1704 /* group is only populated for valid partitions */ 1705 if (xcp->valid) 1706 sysfs_remove_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1707 kobject_put(&adev->ualink.info->kobj); 1708 kobject_put(&xcp->ualink.kobj); 1709 xcp->ualink.sysfs = false; 1710 } 1711 1712 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev) 1713 { 1714 struct amdgpu_xcp *xcp; 1715 int i; 1716 1717 if (!adev->xcp_mgr) 1718 return; 1719 1720 for (i = 0; i < MAX_XCP; i++) { 1721 xcp = &adev->xcp_mgr->xcp[i]; 1722 if (!xcp->ddev || amdgpu_xcp_is_primary(xcp)) 1723 continue; 1724 amdgpu_ualink_xcp_sysfs_add(xcp); 1725 } 1726 } 1727 1728 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev) 1729 { 1730 struct amdgpu_xcp *xcp; 1731 int i; 1732 1733 if (!adev->xcp_mgr) 1734 return; 1735 1736 for (i = 0; i < MAX_XCP; i++) { 1737 xcp = &adev->xcp_mgr->xcp[i]; 1738 amdgpu_ualink_xcp_sysfs_remove(xcp); 1739 } 1740 } 1741 1742 static int amdgpu_ualink_npa_alloc_va(struct amdgpu_device *adev, 1743 struct drm_mm_node *mm_node, 1744 u64 va, u64 range_start, 1745 u64 range_end, int size) 1746 { 1747 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1748 u64 alignment = 0; 1749 int ret; 1750 1751 mutex_lock(&npa_mm->mm_lock); 1752 if (va) { 1753 mm_node->start = va; 1754 mm_node->size = size; 1755 ret = drm_mm_reserve_node(&npa_mm->mm, mm_node); 1756 } else { 1757 if (!range_start && !range_end) { 1758 range_start = npa_mm->va_start; 1759 range_end = npa_mm->va_start + npa_mm->va_size; 1760 } 1761 1762 /* If size is greater than 2MB (or 512 pages), 1763 * align it to 2MB (or 512 pages) granularity. 1764 */ 1765 if (size >= 0x200) 1766 alignment = 0x200; 1767 1768 ret = drm_mm_insert_node_in_range(&npa_mm->mm, mm_node, size, 1769 alignment, 0, range_start, 1770 range_end, 0); 1771 } 1772 mutex_unlock(&npa_mm->mm_lock); 1773 1774 if (ret) 1775 dev_err(adev->dev, "Failed to allocate NPA address\n"); 1776 1777 return ret; 1778 } 1779 1780 static void amdgpu_ualink_npa_free_va(struct amdgpu_device *adev, 1781 struct drm_mm_node *mm_node) 1782 { 1783 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1784 1785 mutex_lock(&npa_mm->mm_lock); 1786 drm_mm_remove_node(mm_node); 1787 mutex_unlock(&npa_mm->mm_lock); 1788 } 1789 1790 static void amdgpu_ualink_npa_mm_init(struct amdgpu_device *adev) 1791 { 1792 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1793 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 1794 u64 va_size; 1795 1796 mutex_init(&npa_mm->mm_lock); 1797 1798 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 1799 /* 51 bit address space in Source-Identification mode. 1800 * 39 bits in page granularity. 1801 */ 1802 npa_mm->va_start = 0; 1803 va_size = GENMASK_ULL(38, 0); 1804 } else { 1805 /* 41 bit address space in Source-Alias mode. 1806 * 29 bits in page granularity. 1807 */ 1808 /* First 8MB is reserved for Metadata NPAs */ 1809 npa_mm->va_start = 0x800; 1810 va_size = GENMASK_ULL(28, 0); 1811 } 1812 1813 npa_mm->va_size = va_size - npa_mm->va_start; 1814 drm_mm_init(&npa_mm->mm, npa_mm->va_start, npa_mm->va_size); 1815 } 1816 1817 static void amdgpu_ualink_npa_mm_fini(struct amdgpu_device *adev) 1818 { 1819 mutex_destroy(&adev->ualink.npa_mm.mm_lock); 1820 drm_mm_takedown(&adev->ualink.npa_mm.mm); 1821 } 1822 1823 /* The caller of this function is expected to hold the XA lock when calling 1824 * this function. 1825 */ 1826 static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev, 1827 struct amdgpu_ualink_handle *handle) 1828 { 1829 bool unique = false; 1830 1831 do { 1832 handle->handle_lo = get_random_u64(); 1833 /* Replace bottom 10 bits in handle_lo with accId */ 1834 handle->handle_lo &= ~AMDGPU_UALINK_HANDLE_ACCID_MASK; 1835 handle->handle_lo |= adev->ualink.info->ppod.accel_id; 1836 1837 /* Don't generate/store a Handle with value 0. */ 1838 if (!handle->handle_lo) 1839 continue; 1840 1841 /* Find if the handle already exists in the exporter xarray. 1842 * If it already exists, then regenerate the handle since we 1843 * want the handle to be unique. 1844 */ 1845 unique = !xa_load(&adev->ualink.exp_xa, handle->handle_lo); 1846 } while (!unique); 1847 1848 handle->handle_hi = get_random_u64(); 1849 dev_dbg(adev->dev, "GENERATE-HANDLE: generated handle: %llx:%llx\n", 1850 handle->handle_hi, handle->handle_lo); 1851 } 1852 1853 static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref) 1854 { 1855 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 1856 struct amdgpu_device *adev; 1857 1858 exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node, 1859 refcount); 1860 adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev); 1861 queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work); 1862 } 1863 1864 static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref) 1865 { 1866 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 1867 struct amdgpu_device *adev; 1868 1869 imp_xa_node = container_of(ref, struct amdgpu_ualink_imp_xa_node, 1870 refcount); 1871 adev = imp_xa_node->adev; 1872 1873 dev_dbg(adev->dev, "IMP-CLEANUP: Remove entry from XA for handle:%llx:%llx\n", 1874 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1875 1876 /* Remove node from the Xarray */ 1877 if (!xa_erase(&adev->ualink.imp_xa, imp_xa_node->handle.handle_lo)) 1878 dev_err(adev->dev, "IMP-CLEANUP: Failed to find entry in XA for handle:%llx:%llx\n", 1879 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1880 1881 if (imp_xa_node->npa_addr && imp_xa_node->size && imp_xa_node->dmabuf) { 1882 dev_dbg(adev->dev, 1883 "IMP-CLEANUP: dmabuf free for NPA:%llx handle:%llx:%llx, fc:%lu\n", 1884 imp_xa_node->npa_addr, imp_xa_node->handle.handle_hi, 1885 imp_xa_node->handle.handle_lo, 1886 file_count(imp_xa_node->dmabuf->file)); 1887 dma_buf_put(imp_xa_node->dmabuf); 1888 drm_gem_handle_delete(adev->ualink.client.file, 1889 imp_xa_node->gem_handle); 1890 1891 } 1892 1893 dev_dbg(adev->dev, "IMP-CLEANUP: Freeing XA entry for handle:%llx:%llx\n", 1894 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1895 1896 kfree(imp_xa_node); 1897 } 1898 1899 static int amdgpu_ualink_exp_xa_entry_get(struct amdgpu_ualink_exp_xa_node *exp_xa_node) 1900 { 1901 return kref_get_unless_zero(&exp_xa_node->refcount); 1902 } 1903 1904 static void amdgpu_ualink_exp_xa_entry_put(struct amdgpu_ualink_exp_xa_node *exp_xa_node) 1905 { 1906 kref_put(&exp_xa_node->refcount, amdgpu_ualink_cleanup_exp_xa_node); 1907 } 1908 1909 static int amdgpu_ualink_imp_xa_entry_get(struct amdgpu_ualink_imp_xa_node *imp_xa_node) 1910 { 1911 return kref_get_unless_zero(&imp_xa_node->refcount); 1912 } 1913 1914 static void amdgpu_ualink_imp_xa_entry_put(struct amdgpu_ualink_imp_xa_node *imp_xa_node) 1915 { 1916 kref_put(&imp_xa_node->refcount, amdgpu_ualink_cleanup_imp_xa_node); 1917 } 1918 1919 static int amdgpu_ualink_send_npa_release_msg(struct amdgpu_device *adev, 1920 u32 remote_acc_id, 1921 struct amdgpu_ualink_handle handle) 1922 { 1923 u32 dw0, dw1, dw2, dw3; 1924 1925 dw0 = lower_32_bits(handle.handle_lo); 1926 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1927 dw0 |= AMDGPU_UALINK_NPA_RELEASE_MSG; 1928 1929 dw1 = upper_32_bits(handle.handle_lo); 1930 dw2 = lower_32_bits(handle.handle_hi); 1931 dw3 = upper_32_bits(handle.handle_hi); 1932 1933 dev_dbg(adev->dev, "SEND NPA-RELEASE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1934 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1935 1936 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1937 dw2, dw3); 1938 } 1939 1940 static int amdgpu_ualink_send_npa_revoke_msg(struct amdgpu_device *adev, 1941 u32 remote_acc_id, 1942 struct amdgpu_ualink_handle handle) 1943 { 1944 u32 dw0, dw1, dw2, dw3; 1945 1946 dw0 = lower_32_bits(handle.handle_lo); 1947 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1948 dw0 |= AMDGPU_UALINK_NPA_REVOKE_MSG; 1949 1950 dw1 = upper_32_bits(handle.handle_lo); 1951 dw2 = lower_32_bits(handle.handle_hi); 1952 dw3 = upper_32_bits(handle.handle_hi); 1953 1954 dev_dbg(adev->dev, "SEND NPA-REVOKE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1955 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1956 1957 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1958 dw2, dw3); 1959 } 1960 1961 static int amdgpu_ualink_send_npa_fail_msg(struct amdgpu_device *adev, 1962 u32 remote_acc_id, 1963 struct amdgpu_ualink_handle handle, 1964 u32 fail_reason) 1965 { 1966 u32 dw0, dw1, dw2, dw3; 1967 1968 dw0 = lower_32_bits(handle.handle_lo); 1969 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1970 dw0 |= AMDGPU_UALINK_NPA_FAIL_MSG; 1971 1972 dw1 = upper_32_bits(handle.handle_lo); 1973 dw2 = fail_reason & 0xFF; 1974 dw3 = 0; 1975 1976 dev_dbg(adev->dev, "SEND NPA-FAIL: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1977 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1978 1979 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1980 dw2, dw3); 1981 } 1982 1983 static int amdgpu_ualink_send_npa_rsp_msg(struct amdgpu_device *adev, 1984 u32 remote_acc_id, 1985 struct amdgpu_ualink_handle handle, 1986 u32 npa_addr, u32 size) 1987 { 1988 u32 dw0, dw1, dw2, dw3; 1989 1990 dw0 = lower_32_bits(handle.handle_lo); 1991 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1992 dw0 |= AMDGPU_UALINK_NPA_RSP_MSG; 1993 1994 dw1 = upper_32_bits(handle.handle_lo); 1995 dw2 = size; 1996 dw3 = npa_addr; 1997 1998 dev_dbg(adev->dev, "SEND NPA-RSP: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1999 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 2000 2001 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 2002 dw2, dw3); 2003 } 2004 2005 static int amdgpu_ualink_send_npa_req_msg(struct amdgpu_device *adev, 2006 u32 remote_acc_id, 2007 struct amdgpu_ualink_handle handle) 2008 { 2009 u32 dw0, dw1, dw2, dw3; 2010 2011 dw0 = lower_32_bits(handle.handle_lo); 2012 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 2013 dw0 |= AMDGPU_UALINK_NPA_REQ_MSG; 2014 2015 dw1 = upper_32_bits(handle.handle_lo); 2016 dw2 = lower_32_bits(handle.handle_hi); 2017 dw3 = upper_32_bits(handle.handle_hi); 2018 2019 dev_dbg(adev->dev, "SEND NPA-REQ: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 2020 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 2021 2022 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 2023 dw2, dw3); 2024 } 2025 2026 static u64 amdgpu_ualink_get_export_pte_flags(struct amdgpu_device *adev, 2027 struct amdgpu_bo *bo, 2028 u64 mapping_flags) 2029 { 2030 u64 pte_flags = adev->gmc.init_pte_flags; 2031 2032 pte_flags |= (AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE | 2033 AMDGPU_PTE_WRITEABLE); 2034 mapping_flags |= AMDGPU_VM_MTYPE_DEFAULT; 2035 2036 amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, bo, mapping_flags, 2037 &pte_flags); 2038 2039 return pte_flags; 2040 } 2041 2042 static int amdgpu_ualink_unmap_npa_addr(struct amdgpu_device *adev, 2043 struct amdgpu_bo *bo, 2044 u64 npa_addr, u64 size) 2045 { 2046 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2047 uint64_t pte_value = adev->gmc.noretry_flags; 2048 struct amdgpu_bo *bos[] = { bo }; 2049 struct dma_fence *fence = NULL; 2050 struct drm_exec exec; 2051 int r; 2052 2053 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false); 2054 if (unlikely(r)) { 2055 dev_err(adev->dev, "Failed to reserve VM and BO in unmap_npa_addr\n"); 2056 return r; 2057 } 2058 2059 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2060 false, NULL, npa_addr, npa_addr + size - 1, 2061 pte_value, 0, 0, NULL, NULL, &vm->last_update); 2062 if (r) { 2063 dev_err(adev->dev, 2064 "Failed to unmap NPA addr (%llx) from NPA VM\n", npa_addr); 2065 goto out; 2066 } 2067 2068 r = amdgpu_vm_update_pdes(adev, vm, false); 2069 if (r) { 2070 dev_err(adev->dev, 2071 "Failed %d to update page directories during unmapping NPA: 0x%llx\n", 2072 r, npa_addr); 2073 goto out; 2074 } 2075 2076 fence = dma_fence_get(vm->last_update); 2077 if (fence) { 2078 r = dma_fence_wait(fence, false); 2079 dma_fence_put(fence); 2080 fence = NULL; 2081 if (r) { 2082 dev_dbg(adev->dev, 2083 "UNMAP-NPA: dma fence wait failed, error: %d\n", r); 2084 goto out; 2085 } 2086 } 2087 2088 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2089 out: 2090 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2091 2092 return r; 2093 } 2094 2095 static int amdgpu_ualink_map_npa_addr(struct amdgpu_device *adev, u64 npa_addr, 2096 u64 size, struct amdgpu_bo *bo, u64 offset, 2097 u64 pte_flags) 2098 { 2099 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2100 struct amdgpu_bo *bos[] = { bo }; 2101 struct dma_fence *fence = NULL; 2102 struct drm_exec exec; 2103 int r; 2104 2105 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false); 2106 if (unlikely(r)) { 2107 dev_err(adev->dev, "Failed to reserve VM and BO in map_npa_addr\n"); 2108 return r; 2109 } 2110 2111 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2112 false, NULL, npa_addr, npa_addr + size - 1, 2113 pte_flags, offset, adev->vm_manager.vram_base_offset, 2114 bo->tbo.resource, NULL, &vm->last_update); 2115 if (r) { 2116 dev_warn(adev->dev, 2117 "Failed to map NPA addr (%llx) into NPA VM\n", npa_addr); 2118 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2119 goto out; 2120 } 2121 2122 r = amdgpu_vm_update_pdes(adev, vm, false); 2123 if (r) { 2124 dev_err(adev->dev, 2125 "failed %d to update page directories for NPA: 0x%llx\n", 2126 r, npa_addr); 2127 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2128 goto unmap_npa; 2129 } 2130 2131 fence = dma_fence_get(vm->last_update); 2132 if (fence) { 2133 r = dma_fence_wait(fence, false); 2134 dma_fence_put(fence); 2135 fence = NULL; 2136 if (r) { 2137 pr_debug("failed %d to dma fence wait\n", r); 2138 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2139 goto unmap_npa; 2140 } 2141 } 2142 2143 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2144 2145 /* TLB flush may be needed after updated page directories */ 2146 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2147 2148 return 0; 2149 2150 unmap_npa: 2151 amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size); 2152 out: 2153 return r; 2154 } 2155 2156 static int amdgpu_ualink_send_hello_ack_msg(struct amdgpu_device *adev, 2157 u32 remote_acc_id) 2158 { 2159 dev_dbg(adev->dev, "SEND HELLO-ACK: HELLO-ACK message to remote AccId:%u\n", 2160 remote_acc_id); 2161 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, 2162 AMDGPU_UALINK_HELLO_ACK_MSG, 2163 0, 0, 0); 2164 } 2165 2166 static int amdgpu_ualink_send_hello_msg(struct amdgpu_device *adev, 2167 u32 remote_acc_id) 2168 { 2169 u32 dw0; 2170 2171 dw0 = AMDGPU_UALINK_HELLO_MSG; 2172 dw0 |= (remote_acc_id << AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT); 2173 dw0 |= (adev->ualink.info->ppod.accel_id << 2174 AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT); 2175 2176 dev_dbg(adev->dev, "SEND HELLO: HELLO message to remote AccId:%u\n", 2177 remote_acc_id); 2178 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, 0, 2179 0, 0); 2180 } 2181 2182 static u32 amdgpu_ualink_check_conn_ready(struct amdgpu_device *adev, 2183 u32 remote_acc_id, u32 gen_count) 2184 { 2185 struct amdgpu_ualink_connection *conn_state; 2186 u32 current_gen_count = 0; 2187 2188 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2189 2190 /* Check if the connection is established. */ 2191 mutex_lock(&conn_state->lock); 2192 if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) && 2193 (!gen_count || conn_state->generation_count == gen_count)) 2194 current_gen_count = conn_state->generation_count; 2195 mutex_unlock(&conn_state->lock); 2196 2197 return current_gen_count; 2198 } 2199 2200 static void amdgpu_ualink_process_hello_ack_msg(struct amdgpu_device *adev, 2201 u32 sender_acc_id) 2202 { 2203 struct amdgpu_ualink_connection *conn_state; 2204 2205 if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 2206 dev_err(adev->dev, 2207 "HELLO-ACK: sender AccId out of range:%u\n", 2208 sender_acc_id); 2209 return; 2210 } 2211 2212 conn_state = &adev->ualink.conn_state[sender_acc_id]; 2213 2214 /* If we are in IN_PROGRESS state, then transition the connection 2215 * state to established and signal that the HELLO ACK is received. 2216 * Otherwise, ignore the HELLO ACK. 2217 */ 2218 mutex_lock(&conn_state->lock); 2219 2220 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) { 2221 conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED; 2222 conn_state->generation_count++; 2223 complete(&conn_state->hello_done); 2224 } else { 2225 dev_dbg(adev->dev, 2226 "HELLO-ACK: already connected, ignoring from AccId:%u\n", 2227 sender_acc_id); 2228 } 2229 mutex_unlock(&conn_state->lock); 2230 } 2231 2232 static void amdgpu_ualink_process_hello_msg(struct amdgpu_device *adev, 2233 u32 receiver_acc_id, 2234 u32 sender_acc_id, 2235 u32 src_acc_id) 2236 { 2237 struct amdgpu_ualink_connection *conn_state; 2238 u32 generation_count; 2239 int r; 2240 2241 if (receiver_acc_id != adev->ualink.info->ppod.accel_id) { 2242 dev_err(adev->dev, 2243 "HELLO: receiver AccId mismatch got:%u self:%u\n", 2244 receiver_acc_id, adev->ualink.info->ppod.accel_id); 2245 return; 2246 } 2247 2248 /* src_acc_id is the AccId received in IH cookie. Confirm it 2249 * matches with the sender AccId. 2250 */ 2251 if (sender_acc_id != src_acc_id) { 2252 dev_err(adev->dev, 2253 "HELLO: sender AccId mismatch sender:%u IH cookie:%u\n", 2254 sender_acc_id, src_acc_id); 2255 return; 2256 } 2257 2258 if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 2259 dev_err(adev->dev, 2260 "HELLO: sender AccId out of range:%u\n", 2261 sender_acc_id); 2262 return; 2263 } 2264 2265 conn_state = &adev->ualink.conn_state[sender_acc_id]; 2266 2267 /* Check if connection is already established. If yes, then receiving HELLO msg 2268 * triggers a reset handling scenario. 2269 * If the connection is not ready, and we receive a HELLO msg, then 2270 * transition the state to PENDING. 2271 * Otherwise, leave it IN_PROGRESS. 2272 */ 2273 mutex_lock(&conn_state->lock); 2274 if (conn_state->state != AMDGPU_UALINK_CONN_ESTABLISHED) { 2275 if (conn_state->state == AMDGPU_UALINK_CONN_NOT_READY) 2276 conn_state->state = AMDGPU_UALINK_CONN_PENDING; 2277 /* otherwise, leave it IN_PROGRESS to signal the completion below */ 2278 mutex_unlock(&conn_state->lock); 2279 } else { 2280 /* Leave state ESTABLISHED so handle_connection_reset() 2281 * performs the transition and imp/exp XA cleanup. 2282 */ 2283 generation_count = conn_state->generation_count; 2284 mutex_unlock(&conn_state->lock); 2285 2286 amdgpu_ualink_handle_connection_reset(adev, sender_acc_id, 2287 AMDGPU_UALINK_CONN_PENDING, 2288 generation_count); 2289 } 2290 2291 /* Peer may have rebooted. */ 2292 r = amdgpu_ualink_ring_resync_ptrs(adev, sender_acc_id); 2293 if (r < 0) { 2294 dev_err(adev->dev, 2295 "HELLO: resync sender AccId %u ptrs failed %d\n", 2296 sender_acc_id, r); 2297 goto out_set_state; 2298 } 2299 if (r) 2300 amdgpu_ualink_reset_peer_rings(adev, sender_acc_id); 2301 2302 r = amdgpu_ualink_send_hello_ack_msg(adev, sender_acc_id); 2303 if (r) 2304 dev_err(adev->dev, "HELLO-ACK: send failed to remote AccId:%u\n", 2305 sender_acc_id); 2306 2307 out_set_state: 2308 mutex_lock(&conn_state->lock); 2309 if (r) { 2310 conn_state->state = AMDGPU_UALINK_CONN_NOT_READY; 2311 } else { 2312 /* If we are in IN_PROGRESS state and we receivied the HELLO message, 2313 * upon receiving the HELLO message, transition the state to ESTABLISHED 2314 * and signal the completion. 2315 */ 2316 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) 2317 complete(&conn_state->hello_done); 2318 conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED; 2319 conn_state->generation_count++; 2320 } 2321 mutex_unlock(&conn_state->lock); 2322 } 2323 2324 /* True if nothing is exported to, or imported from, this remote accelerator. */ 2325 static bool amdgpu_ualink_peer_idle(struct amdgpu_device *adev, u32 remote_accel_id) 2326 { 2327 /* Unlocked: a racing import at worst costs one redundant ring resync. */ 2328 return list_empty(&adev->ualink.imp_handles_list[remote_accel_id]) && 2329 list_empty(&adev->ualink.exp_handles_list[remote_accel_id]); 2330 } 2331 2332 static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev, 2333 u32 remote_acc_id) 2334 { 2335 struct amdgpu_ualink_connection *conn_state; 2336 int r; 2337 2338 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2339 2340 /* Connection state management goes through different states. 2341 * The states are: 2342 * - NOT_READY: The connection is not ready. 2343 * - IN_PROGRESS: GPU sent HELLO message and is waiting for the HELLO_ACK. 2344 * - PENDING: GPU received HELLO message and is in the process of sending 2345 * the HELLO_ACK. 2346 * - ESTABLISHED: The connection is established. 2347 */ 2348 2349 /* Grab the lock and check if connection establishment was 2350 * already done by another thread. 2351 */ 2352 mutex_lock(&conn_state->lock); 2353 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS || 2354 conn_state->state == AMDGPU_UALINK_CONN_PENDING) { 2355 r = -EAGAIN; 2356 goto out; 2357 } 2358 2359 if (conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED && 2360 !amdgpu_ualink_peer_idle(adev, remote_acc_id)) { 2361 r = 0; 2362 goto out; 2363 } 2364 2365 /* Peer is idle or not connected, it may have rebooted. Skip the 2366 * handshake if the rings are still in sync, it would reset the remote. 2367 */ 2368 r = amdgpu_ualink_ring_resync_ptrs(adev, remote_acc_id); 2369 if (r < 0) 2370 goto out; 2371 if (!r && conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) 2372 goto out; 2373 2374 amdgpu_ualink_reset_peer_rings(adev, remote_acc_id); 2375 2376 conn_state->state = AMDGPU_UALINK_CONN_IN_PROGRESS; 2377 2378 mutex_unlock(&conn_state->lock); 2379 2380 /* Send HELLO message */ 2381 r = amdgpu_ualink_send_hello_msg(adev, remote_acc_id); 2382 if (r) { 2383 dev_warn(adev->dev, 2384 "HELLO: Send failed to remote AccId:%u\n", 2385 remote_acc_id); 2386 goto reset_state; 2387 } 2388 2389 /* Wait for the HELLO_ACK to come back */ 2390 /* complete(conn_state->hello_done) should be called from the IRQ 2391 * handler when the HELLO_ACK is received. 2392 */ 2393 r = wait_for_completion_interruptible_timeout(&conn_state->hello_done, 2394 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 2395 if (r == -ERESTARTSYS) { 2396 dev_err_ratelimited(adev->dev, 2397 "HELLO-ACK: interrupted by signal\n"); 2398 goto reset_state; 2399 } else if (r == 0) { 2400 dev_warn(adev->dev, 2401 "HELLO-ACK: Timeout from remote AccId:%u\n", 2402 remote_acc_id); 2403 r = -ETIMEDOUT; 2404 goto reset_state; 2405 } 2406 2407 return 0; 2408 2409 reset_state: 2410 mutex_lock(&conn_state->lock); 2411 conn_state->state = AMDGPU_UALINK_CONN_NOT_READY; 2412 out: 2413 mutex_unlock(&conn_state->lock); 2414 2415 return r; 2416 } 2417 2418 /* Invalidate an importer node's mappings and drop its last ref (frees the 2419 * node, dma-buf and GEM handle). Caller must have unlinked it from the 2420 * per-remote list and must not hold the imp_xa lock. 2421 */ 2422 static void amdgpu_ualink_release_imp_xa_node(struct amdgpu_device *adev, 2423 struct amdgpu_ualink_imp_xa_node *imp_xa_node) 2424 { 2425 struct amdgpu_bo *bo; 2426 2427 dev_dbg(adev->dev, 2428 "IMP-CLEANUP: handle:%llx:%llx npa:%llx size:%llx\n", 2429 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 2430 imp_xa_node->npa_addr, imp_xa_node->size); 2431 2432 if (imp_xa_node->dmabuf) { 2433 bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv); 2434 amdgpu_ualink_invalidate_import_mappings(bo); 2435 } 2436 2437 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 2438 } 2439 2440 static void amdgpu_ualink_cleanup_imp_xa_entries(struct amdgpu_device *adev, 2441 u32 remote_acc_id) 2442 { 2443 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 2444 struct list_head *imp_handles_list; 2445 2446 dev_dbg(adev->dev, 2447 "IMP-RESET: Cleaning up all XA entries for remote:%u\n", 2448 remote_acc_id); 2449 2450 imp_handles_list = &adev->ualink.imp_handles_list[remote_acc_id]; 2451 2452 xa_lock(&adev->ualink.imp_xa); 2453 while (!list_empty(imp_handles_list)) { 2454 imp_xa_node = list_first_entry(imp_handles_list, 2455 struct amdgpu_ualink_imp_xa_node, list); 2456 list_del_init(&imp_xa_node->list); 2457 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 2458 xa_unlock(&adev->ualink.imp_xa); 2459 2460 amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node); 2461 2462 xa_lock(&adev->ualink.imp_xa); 2463 } 2464 xa_unlock(&adev->ualink.imp_xa); 2465 } 2466 2467 static void amdgpu_ualink_cleanup_exp_xa_entries(struct amdgpu_device *adev, 2468 u32 remote_acc_id) 2469 { 2470 struct amdgpu_ualink_importer_entry *importer_entry; 2471 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2472 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 2473 struct list_head *exp_handles_list; 2474 struct drm_mm_node *mm_node; 2475 u64 npa_addr, size; 2476 2477 /* Get the list head for the list containing all the handles 2478 * exported to this remote GPU. 2479 */ 2480 exp_handles_list = &adev->ualink.exp_handles_list[remote_acc_id]; 2481 2482 dev_dbg(adev->dev, 2483 "EXP-RESET: Cleaning up all XA entries for remote:%u\n", 2484 remote_acc_id); 2485 2486 xa_lock(&adev->ualink.exp_xa); 2487 while (!list_empty(exp_handles_list)) { 2488 importer_entry = list_first_entry(exp_handles_list, 2489 struct amdgpu_ualink_importer_entry, list); 2490 list_del_init(&importer_entry->list); 2491 2492 exp_xa_node = importer_entry->parent; 2493 if (!amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) 2494 continue; 2495 2496 xa_unlock(&adev->ualink.exp_xa); 2497 /* Clear the bit corresponding to this remote GPU in 2498 * the importer bitmap. 2499 */ 2500 if (!test_and_clear_bit(remote_acc_id, 2501 exp_xa_node->importers_bitmap)) { 2502 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 2503 xa_lock(&adev->ualink.exp_xa); 2504 continue; 2505 } 2506 2507 mutex_lock(&exp_xa_node->node_lock); 2508 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2509 /* Unpin the BO */ 2510 if (likely(!amdgpu_bo_reserve(exp_xa_node->bo, true))) { 2511 amdgpu_bo_unpin(exp_xa_node->bo); 2512 amdgpu_bo_unreserve(exp_xa_node->bo); 2513 } else { 2514 dev_warn(adev->dev, 2515 "EXP-RESET: BO reserve to unpin failed handle:%llx:%llx\n", 2516 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo); 2517 } 2518 2519 dev_dbg(adev->dev, 2520 "EXP-RESET: handle:%llx:%llx pin_count:%d, importers:%d\n", 2521 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 2522 exp_xa_node->bo->tbo.pin_count, 2523 bitmap_weight(exp_xa_node->importers_bitmap, 2524 AMDGPU_UALINK_ACCEL_MAX)); 2525 2526 WARN_ON(exp_xa_node->bo->tbo.pin_count < 2527 bitmap_weight(exp_xa_node->importers_bitmap, 2528 AMDGPU_UALINK_ACCEL_MAX)); 2529 2530 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 2531 npa_addr = importer_entry->npa_addr; 2532 mm_node = importer_entry->mm_node; 2533 mutex_unlock(&exp_xa_node->node_lock); 2534 2535 dev_dbg(adev->dev, 2536 "EXP-RESET: Unmap NPA:%llx size: %llx remote:%u handle:%llx:%llx\n", 2537 npa_addr, size, remote_acc_id, 2538 exp_xa_node->handle.handle_hi, 2539 exp_xa_node->handle.handle_lo); 2540 amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo, 2541 npa_addr, size); 2542 amdgpu_ualink_npa_free_va(adev, mm_node); 2543 kfree(mm_node); 2544 2545 /* Reset the NPA addr and mm_node */ 2546 mutex_lock(&exp_xa_node->node_lock); 2547 importer_entry->npa_addr = 0; 2548 importer_entry->mm_node = NULL; 2549 } else if (bitmap_empty(exp_xa_node->importers_bitmap, 2550 AMDGPU_UALINK_ACCEL_MAX)) { 2551 /* In Source-Aliasing mode, if there are no importers 2552 * for this handle, then we can unmap and free the 2553 * NPA address. 2554 */ 2555 mm_node = exp_xa_node->importer_entries[0].mm_node; 2556 npa_addr = exp_xa_node->importer_entries[0].npa_addr; 2557 mutex_unlock(&exp_xa_node->node_lock); 2558 2559 dev_dbg(adev->dev, 2560 "EXP-RESET: Unmap NPA:%llx size: %llx handle:%llx:%llx\n", 2561 npa_addr, size, exp_xa_node->handle.handle_hi, 2562 exp_xa_node->handle.handle_lo); 2563 amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo, 2564 npa_addr, size); 2565 amdgpu_ualink_npa_free_va(adev, mm_node); 2566 kfree(mm_node); 2567 mutex_lock(&exp_xa_node->node_lock); 2568 exp_xa_node->importer_entries[0].npa_addr = 0; 2569 exp_xa_node->importer_entries[0].mm_node = NULL; 2570 } 2571 2572 mutex_unlock(&exp_xa_node->node_lock); 2573 xa_lock(&adev->ualink.exp_xa); 2574 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 2575 } 2576 xa_unlock(&adev->ualink.exp_xa); 2577 } 2578 2579 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev, 2580 u32 remote_acc_id, u32 state, 2581 u32 generation_count) 2582 { 2583 struct amdgpu_ualink_connection *conn_state; 2584 2585 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2586 2587 mutex_lock(&conn_state->lock); 2588 if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) && 2589 (conn_state->generation_count == generation_count)) { 2590 conn_state->state = state; 2591 mutex_unlock(&conn_state->lock); 2592 2593 amdgpu_ualink_cleanup_imp_xa_entries(adev, remote_acc_id); 2594 amdgpu_ualink_cleanup_exp_xa_entries(adev, remote_acc_id); 2595 } else { 2596 mutex_unlock(&conn_state->lock); 2597 } 2598 } 2599 2600 /* Set PTE.X = 1 for all importer entries to retry RPCs. */ 2601 static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev, 2602 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2603 { 2604 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2605 struct amdgpu_ualink_importer_entry *imp_entry; 2606 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2607 u64 pte_flags, npa_addr, size; 2608 struct dma_fence *fence = NULL; 2609 struct amdgpu_bo *bo; 2610 struct drm_exec exec; 2611 u32 remote_acc_id; 2612 int r; 2613 2614 bo = exp_xa_node->bo; 2615 size = amdgpu_bo_ngpu_pages(bo); 2616 2617 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false); 2618 if (unlikely(r)) { 2619 dev_err(adev->dev, "Failed to reserve VM and BO in force_retry_rpcs\n"); 2620 return; 2621 } 2622 2623 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2624 AMDGPU_UALINK_ACCEL_MAX) { 2625 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2626 imp_entry = &exp_xa_node->importer_entries[0]; 2627 else 2628 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2629 npa_addr = imp_entry->npa_addr; 2630 2631 /* If the connection state changed while we are freeing 2632 * the BO, then ignore this importer. We will unmap this 2633 * address eventually in amdgpu_ualink_unmap_all_npa_addr() 2634 * function. Also, we will clear the corresponding bit in 2635 * the importer_bitmap in the same function. 2636 */ 2637 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2638 imp_entry->generation_count)) 2639 continue; 2640 2641 /* Set PTE.X = 1 */ 2642 pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 2643 AMDGPU_VM_PAGE_EXECUTABLE); 2644 dev_dbg(adev->dev, 2645 "RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n", 2646 npa_addr, remote_acc_id, pte_flags); 2647 2648 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2649 false, NULL, npa_addr, npa_addr + size - 1, 2650 pte_flags, 0, adev->vm_manager.vram_base_offset, 2651 bo->tbo.resource, NULL, &vm->last_update); 2652 2653 if (r) 2654 dev_warn(adev->dev, 2655 "RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n", 2656 npa_addr, remote_acc_id, r); 2657 2658 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2659 break; 2660 } 2661 2662 r = amdgpu_vm_update_pdes(adev, vm, false); 2663 if (r) { 2664 dev_err(adev->dev, 2665 "Failed %d to update page directories during force retry rpcs\n", 2666 r); 2667 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2668 return; 2669 } 2670 2671 fence = dma_fence_get(vm->last_update); 2672 if (fence) { 2673 r = dma_fence_wait(fence, false); 2674 dma_fence_put(fence); 2675 fence = NULL; 2676 if (r) 2677 dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r); 2678 } 2679 2680 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2681 2682 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2683 } 2684 2685 static void amdgpu_ualink_send_tlb_shootdown(struct amdgpu_device *adev, 2686 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2687 { 2688 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2689 u32 accel_id = adev->ualink.info->ppod.accel_id; 2690 struct amdgpu_ualink_importer_entry *imp_entry; 2691 u64 npa_addr, size; 2692 u32 remote_acc_id; 2693 int r; 2694 2695 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2696 2697 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2698 AMDGPU_UALINK_ACCEL_MAX) { 2699 /* 2700 * Replace the GPU-id bits with local accel_id 2701 */ 2702 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2703 imp_entry = &exp_xa_node->importer_entries[0]; 2704 else 2705 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2706 2707 npa_addr = STRIP_NPA(imp_entry->npa_addr); 2708 npa_addr = GENERATE_NPA(npa_addr, accel_id); 2709 2710 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2711 imp_entry->generation_count)) { 2712 clear_bit(remote_acc_id, 2713 exp_xa_node->importers_bitmap); 2714 continue; 2715 } 2716 2717 dev_dbg(adev->dev, 2718 "EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n", 2719 remote_acc_id); 2720 r = amdgpu_ualink_remote_shootdown(adev, remote_acc_id, 2721 npa_addr, size, 2722 AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN); 2723 if (r) { 2724 dev_err(adev->dev, 2725 "EXP-CLEANUP: TLB shootdown send failed to remote:%u\n", 2726 remote_acc_id); 2727 2728 /* Send failed: mark the connection NOT_READY so the 2729 * rest of this cleanup (and other cleanups) skip the 2730 * unreachable peer via amdgpu_ualink_check_conn_ready(). 2731 */ 2732 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 2733 AMDGPU_UALINK_CONN_NOT_READY, 2734 imp_entry->generation_count); 2735 } 2736 } 2737 } 2738 2739 /* Unmap all NPA addresses associated with a BO (UALink handle). This function is used 2740 * only in Source Identification mode. 2741 */ 2742 static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev, 2743 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2744 { 2745 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2746 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2747 struct amdgpu_ualink_importer_entry *imp_entry; 2748 u64 pte_value = adev->gmc.noretry_flags; 2749 struct dma_fence *fence = NULL; 2750 struct drm_exec exec; 2751 u64 npa_addr, size; 2752 u32 remote_acc_id; 2753 int r; 2754 2755 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2756 2757 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1, 2758 &exec, false); 2759 if (unlikely(r)) { 2760 dev_err(adev->dev, "Failed to reserve VM and BO in unmap_all_npa_addr\n"); 2761 return; 2762 } 2763 2764 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2765 AMDGPU_UALINK_ACCEL_MAX) { 2766 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2767 imp_entry = &exp_xa_node->importer_entries[0]; 2768 else 2769 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2770 npa_addr = imp_entry->npa_addr; 2771 2772 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2773 imp_entry->generation_count)) { 2774 clear_bit(remote_acc_id, 2775 exp_xa_node->importers_bitmap); 2776 continue; 2777 } 2778 2779 dev_dbg(adev->dev, 2780 "UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n", 2781 npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 2782 remote_acc_id, pte_value); 2783 2784 r = amdgpu_vm_update_range(adev, vm, false, 2785 false, true, false, NULL, npa_addr, 2786 npa_addr + size - 1, pte_value, 0, 2787 0, NULL, NULL, &vm->last_update); 2788 2789 if (r) 2790 dev_err(adev->dev, 2791 "UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n", 2792 npa_addr, exp_xa_node->handle.handle_hi, 2793 exp_xa_node->handle.handle_lo, remote_acc_id); 2794 2795 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2796 break; 2797 } 2798 2799 r = amdgpu_vm_update_pdes(adev, vm, false); 2800 if (r) { 2801 dev_err(adev->dev, 2802 "Failed %d to update page directories during all NPA addresses unmapping\n", 2803 r); 2804 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2805 return; 2806 } 2807 2808 fence = dma_fence_get(vm->last_update); 2809 if (fence) { 2810 r = dma_fence_wait(fence, false); 2811 dma_fence_put(fence); 2812 fence = NULL; 2813 if (r) 2814 dev_err(adev->dev, 2815 "UNMAP-NPA: dma fence wait failed\n"); 2816 } 2817 2818 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2819 2820 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2821 } 2822 2823 static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev, 2824 struct amdgpu_ualink_exp_xa_node *exp_xa_node, 2825 unsigned long *importers_bitmap) 2826 { 2827 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2828 struct drm_mm_node *mm_node; 2829 u32 remote_acc_id; 2830 2831 for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) { 2832 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2833 mm_node = exp_xa_node->importer_entries[0].mm_node; 2834 else 2835 mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node; 2836 2837 if (!mm_node) { 2838 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2839 break; 2840 continue; 2841 } 2842 2843 dev_dbg(adev->dev, 2844 "FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n", 2845 mm_node->start, exp_xa_node->handle.handle_hi, 2846 exp_xa_node->handle.handle_lo, remote_acc_id); 2847 2848 amdgpu_ualink_npa_free_va(adev, mm_node); 2849 kfree(mm_node); 2850 2851 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2852 break; 2853 } 2854 } 2855 2856 static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work) 2857 { 2858 DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 2859 struct amdgpu_ualink_importer_entry *imp_entry; 2860 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 2861 struct amdgpu_ualink_handle handle; 2862 struct amdgpu_device *adev; 2863 struct amdgpu_bo *bo; 2864 u32 remote_acc_id; 2865 int r; 2866 2867 exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node, 2868 cleanup_work); 2869 bo = exp_xa_node->bo; 2870 adev = amdgpu_ttm_adev(bo->tbo.bdev); 2871 handle = exp_xa_node->handle; 2872 2873 /* Revoking access to an exported memory follows the steps: 2874 * 1. Set PTE.X = 1 to retry for RPCs. 2875 * 2. Send Remote TLB Shootdowns to all importers. 2876 * 3. Unmap the NPA address from NPA VM. 2877 * 4. Drop the ref count for the BO. 2878 * 5. Send NPA_REVOKE to all importers. 2879 * 6. Wait for NPA_RELEASE from all importers. 2880 * 7. Free the NPA address once all responses are received. 2881 */ 2882 /* If there are no importers for this BO/handle */ 2883 if (bitmap_empty(exp_xa_node->importers_bitmap, 2884 AMDGPU_UALINK_ACCEL_MAX)) { 2885 /* Release the dma_buf created at export time (used for the 2886 * local-import shortcut). The has-importers path below drops 2887 * it too; this branch must not skip it or the dma_buf and the 2888 * BO it pins are leaked. 2889 */ 2890 dma_buf_put(exp_xa_node->dmabuf); 2891 /* Drop the BO reference so it can be freed. */ 2892 amdgpu_bo_unref(&bo); 2893 exp_xa_node->bo = NULL; 2894 goto free_node; 2895 } 2896 2897 dev_dbg(adev->dev, 2898 "EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n", 2899 handle.handle_hi, handle.handle_lo, 2900 AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap); 2901 2902 bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap, 2903 AMDGPU_UALINK_ACCEL_MAX); 2904 2905 /* Set PTE.X = 1 for NPA addresses from all importers*/ 2906 amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node); 2907 2908 /* Send TLB-shootdown to all importer GPUs */ 2909 amdgpu_ualink_send_tlb_shootdown(adev, exp_xa_node); 2910 2911 /* Unmap all NPA addresses for this BO from NPA VM */ 2912 amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node); 2913 2914 dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n", 2915 handle.handle_hi, handle.handle_lo, bo->tbo.pin_count, 2916 bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)); 2917 WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap, 2918 AMDGPU_UALINK_ACCEL_MAX)); 2919 2920 /* Unpin the BO */ 2921 if (likely(!amdgpu_bo_reserve(bo, true))) { 2922 bo->ualink_handle_lo = 0ULL; 2923 for_each_set_bit(remote_acc_id, orig_importers_bitmap, 2924 AMDGPU_UALINK_ACCEL_MAX) 2925 amdgpu_bo_unpin(bo); 2926 amdgpu_bo_unreserve(bo); 2927 } else { 2928 dev_warn(adev->dev, 2929 "EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n", 2930 handle.handle_hi, handle.handle_lo); 2931 } 2932 2933 /* Free the DMABuf */ 2934 dma_buf_put(exp_xa_node->dmabuf); 2935 /* Drop the reference to the BO so it can be freed. */ 2936 amdgpu_bo_unref(&bo); 2937 exp_xa_node->bo = NULL; 2938 2939 /* Seed npa_release_bitmap with all importers before any NPA-REVOKE, so 2940 * the release IRQ only sees it empty once every responder has replied. 2941 * Dead peers are dropped from it in the send loop below. 2942 */ 2943 bitmap_copy(exp_xa_node->npa_release_bitmap, 2944 exp_xa_node->importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 2945 2946 /* Send NPA-REVOKE to all importers which have imported this memory. 2947 * On send failure clear the bit (no response will arrive) and mark the 2948 * connection NOT_READY so other cleanups skip this unreachable peer. 2949 */ 2950 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2951 AMDGPU_UALINK_ACCEL_MAX) { 2952 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2953 2954 /* Re-check the connection right before the send: concurrent 2955 * cleanup workers all seed the bitmap while the peer is still 2956 * ESTABLISHED, so once its egress dies and the first worker 2957 * marks it NOT_READY, this skips the rest instead of flooding 2958 * the log with failed LSDMA sends to the same dead peer. 2959 */ 2960 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2961 imp_entry->generation_count)) { 2962 clear_bit(remote_acc_id, 2963 exp_xa_node->npa_release_bitmap); 2964 continue; 2965 } 2966 2967 dev_dbg(adev->dev, 2968 "EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n", 2969 remote_acc_id); 2970 r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle); 2971 if (r) { 2972 dev_err(adev->dev, 2973 "EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n", 2974 remote_acc_id); 2975 clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap); 2976 2977 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 2978 AMDGPU_UALINK_CONN_NOT_READY, 2979 imp_entry->generation_count); 2980 } 2981 } 2982 2983 /* Wait for the NPA_RELEASE to come back from all importers */ 2984 if (!bitmap_empty(exp_xa_node->npa_release_bitmap, 2985 AMDGPU_UALINK_ACCEL_MAX)) { 2986 dev_dbg(adev->dev, 2987 "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n", 2988 handle.handle_hi, handle.handle_lo, 2989 AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap); 2990 2991 /* Wait for the NPA_RELEASE to come back from all importers */ 2992 r = wait_for_completion_timeout(&exp_xa_node->npa_done, 2993 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 2994 2995 if (r == 0) 2996 dev_warn(adev->dev, 2997 "EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n", 2998 handle.handle_hi, handle.handle_lo); 2999 } 3000 3001 /* Free the NPA addresses given to all the importers */ 3002 amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, 3003 orig_importers_bitmap); 3004 3005 /* Warn about all importers that didn't respond back with 3006 * NPA-RELEASE message and trigger connection timeout handling. 3007 */ 3008 for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap, 3009 AMDGPU_UALINK_ACCEL_MAX) { 3010 dev_warn(adev->dev, 3011 "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n", 3012 handle.handle_hi, handle.handle_lo, remote_acc_id); 3013 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3014 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 3015 AMDGPU_UALINK_CONN_NOT_READY, 3016 imp_entry->generation_count); 3017 } 3018 3019 free_node: 3020 xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo); 3021 mutex_destroy(&exp_xa_node->node_lock); 3022 kfree(exp_xa_node); 3023 } 3024 3025 /* This function is a copy of amdgpu_dma_buf_move_notify() function. 3026 * amdgpu_dma_buf_move_notify is only called for import attachments. 3027 * But NPA DMABufs don't use attachments because they are imported 3028 * on the same device. So we need to invalidate the GPUVM mappings 3029 * manually. 3030 */ 3031 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo) 3032 { 3033 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 3034 struct ttm_operation_ctx ctx = { false, false }; 3035 struct ttm_placement placement = {}; 3036 struct amdgpu_vm_bo_base *bo_base; 3037 struct drm_exec exec; 3038 int r; 3039 3040 /* 3041 * Lock the BO together with every client VM page directory it is 3042 * mapped into in a single drm_exec transaction. 3043 */ 3044 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 3045 drm_exec_until_all_locked(&exec) { 3046 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 3047 drm_exec_retry_on_contention(&exec); 3048 if (unlikely(r)) 3049 goto fini; 3050 3051 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 3052 r = amdgpu_vm_lock_pd(bo_base->vm, &exec, 0); 3053 drm_exec_retry_on_contention(&exec); 3054 if (unlikely(r)) 3055 goto fini; 3056 } 3057 } 3058 3059 amdgpu_vm_bo_move(bo, NULL, false); 3060 3061 r = ttm_bo_validate(&bo->tbo, &placement, &ctx); 3062 if (r) { 3063 dev_err(adev->dev, "Failed to invalidate NPA DMA-buf import (%d)\n", 3064 r); 3065 goto fini; 3066 } 3067 3068 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 3069 struct amdgpu_vm *vm = bo_base->vm; 3070 3071 /* 3072 * Fences for the two SDMA page table updates were already 3073 * reserved by amdgpu_vm_lock_pd() above (it reserves 2 + 3074 * num_fences on the same VM root PD dma_resv). 3075 */ 3076 r = amdgpu_vm_clear_freed(adev, vm, NULL); 3077 if (!r) 3078 r = amdgpu_vm_handle_moved(adev, vm, &exec.ticket); 3079 3080 if (r && r != -EBUSY) 3081 dev_err(adev->dev, "Failed to invalidate VM page tables (%d))\n", 3082 r); 3083 } 3084 3085 fini: 3086 drm_exec_fini(&exec); 3087 } 3088 3089 static int amdgpu_ualink_map_npa_to_dmabuf(struct amdgpu_device *adev, 3090 struct amdgpu_ualink_imp_xa_node *imp_xa_node) 3091 { 3092 struct ttm_operation_ctx ctx = { false, false }; 3093 u32 initial_domain = AMDGPU_GEM_DOMAIN_NPA; 3094 struct drm_gem_object *gobj = NULL; 3095 u64 alloc_flags, npa_addr, size; 3096 struct dma_buf *dmabuf; 3097 struct amdgpu_bo *bo; 3098 u32 handle; 3099 int r; 3100 3101 npa_addr = imp_xa_node->npa_addr; 3102 size = imp_xa_node->size; 3103 3104 dev_dbg(adev->dev, "Create NPA BO addr 0x%llx size in pages 0x%llx\n", 3105 npa_addr, size); 3106 3107 /* 3108 * Create the BO directly in the NPA domain. 3109 */ 3110 alloc_flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 3111 r = amdgpu_gem_object_create(adev, size * AMDGPU_GPU_PAGE_SIZE, 1, 3112 initial_domain, alloc_flags, 3113 ttm_bo_type_device, NULL, &gobj, 0); 3114 if (r) { 3115 dev_err(adev->dev, "Failed to create NPA BO. ret %d\n", r); 3116 return r; 3117 } 3118 3119 bo = gem_to_amdgpu_bo(gobj); 3120 3121 r = amdgpu_bo_reserve(bo, false); 3122 if (unlikely(r != 0)) { 3123 dev_err(adev->dev, "Failed to reserve NPA BO, r: %d\n", r); 3124 goto err_reserve_failed; 3125 } 3126 3127 /* 3128 * Drop the arbitrarily-placed NPA node and re-create it at the exact 3129 * remote window, mirroring amdgpu_bo_create_kernel_at(). 3130 */ 3131 ttm_resource_free(&bo->tbo, &bo->tbo.resource); 3132 3133 bo->placements[0].fpfn = npa_addr; 3134 bo->placements[0].lpfn = npa_addr + size; 3135 3136 r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.resource, &ctx); 3137 amdgpu_bo_unreserve(bo); 3138 if (r) { 3139 dev_err(adev->dev, 3140 "Failed to place NPA BO at 0x%llx, r: %d\n", npa_addr, r); 3141 goto err_validate_failed; 3142 } 3143 3144 r = drm_gem_handle_create(adev->ualink.client.file, gobj, &handle); 3145 if (r) { 3146 dev_err(adev->dev, 3147 "Failed to get handle for NPA GEM object, r: %d\n", r); 3148 goto err_validate_failed; 3149 } 3150 drm_gem_object_put(gobj); 3151 3152 dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, adev->ualink.client.file, 3153 handle, DRM_CLOEXEC | DRM_RDWR); 3154 if (IS_ERR(dmabuf)) { 3155 r = PTR_ERR(dmabuf); 3156 dev_err(adev->dev, 3157 "Failed to generate DMABuf for NPA GEM object\n"); 3158 goto err_dmabuf_failed; 3159 } 3160 3161 imp_xa_node->dmabuf = dmabuf; 3162 imp_xa_node->gem_handle = handle; 3163 3164 return 0; 3165 3166 err_dmabuf_failed: 3167 drm_gem_handle_delete(adev->ualink.client.file, handle); 3168 return r; 3169 err_validate_failed: 3170 err_reserve_failed: 3171 drm_gem_object_put(gobj); 3172 3173 return r; 3174 } 3175 3176 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo) 3177 { 3178 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 3179 struct amdgpu_ualink_importer_entry *imp_entry; 3180 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3181 u32 remote_acc_id; 3182 int r; 3183 3184 if (!bo->ualink_handle_lo) 3185 return; 3186 3187 /* Remove the entry from the Xarray. */ 3188 xa_lock(&adev->ualink.exp_xa); 3189 exp_xa_node = __xa_erase(&adev->ualink.exp_xa, 3190 bo->ualink_handle_lo); 3191 if (!exp_xa_node) { 3192 xa_unlock(&adev->ualink.exp_xa); 3193 dev_warn(adev->dev, 3194 "Exp XA: handle_lo:%llx not found\n", 3195 bo->ualink_handle_lo); 3196 return; 3197 } 3198 3199 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 3200 AMDGPU_UALINK_ACCEL_MAX) { 3201 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3202 list_del_init(&imp_entry->list); 3203 } 3204 xa_unlock(&adev->ualink.exp_xa); 3205 3206 /* Add it to the Handle_Invalid xarray */ 3207 r = xa_err(xa_store(&adev->ualink.handle_invalid_xa, 3208 bo->ualink_handle_lo, 3209 exp_xa_node, GFP_KERNEL)); 3210 if (r) 3211 dev_err(adev->dev, 3212 "Handle_Invalid XA store failed handle:%llx:%llx error:%d\n", 3213 exp_xa_node->handle.handle_hi, 3214 exp_xa_node->handle.handle_lo, r); 3215 3216 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3217 } 3218 3219 static void amdgpu_ualink_process_npa_release_msg(struct amdgpu_device *adev, 3220 u32 remote_acc_id, 3221 struct amdgpu_ualink_handle handle) 3222 { 3223 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3224 3225 xa_lock(&adev->ualink.handle_invalid_xa); 3226 exp_xa_node = xa_load(&adev->ualink.handle_invalid_xa, handle.handle_lo); 3227 if (!exp_xa_node) { 3228 dev_warn(adev->dev, 3229 "NPA-RELEASE: Handle (%llx:%llx) not found\n", 3230 handle.handle_hi, handle.handle_lo); 3231 goto out; 3232 } 3233 3234 /* Confirm that the complete handle matches. */ 3235 if (handle.handle_hi != exp_xa_node->handle.handle_hi) { 3236 dev_warn(adev->dev, 3237 "NPA-RELEASE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n", 3238 exp_xa_node->handle.handle_hi, 3239 handle.handle_hi, handle.handle_lo, remote_acc_id); 3240 goto out; 3241 } 3242 3243 /* Test and clear the bit corresponding to the remote GPU id to signal 3244 * the arrival of NPA_RELEASE message from it. 3245 * If the corresponding bit wasn't set, then raise a warning and ignore 3246 * the NPA-Release message from the remote GPU. 3247 */ 3248 if (!test_and_clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap)) { 3249 dev_warn(adev->dev, 3250 "NPA-RELEASE: unexpected from remote:%u handle:%llx:%llx\n", 3251 remote_acc_id, handle.handle_hi, handle.handle_lo); 3252 goto out; 3253 } 3254 3255 /* Signal completion if NPA_Release received from all importers */ 3256 if (bitmap_empty(exp_xa_node->npa_release_bitmap, 3257 AMDGPU_UALINK_ACCEL_MAX)) 3258 complete(&exp_xa_node->npa_done); 3259 3260 out: 3261 xa_unlock(&adev->ualink.handle_invalid_xa); 3262 } 3263 3264 static void amdgpu_ualink_process_npa_revoke_msg(struct amdgpu_device *adev, 3265 u32 remote_acc_id, 3266 struct amdgpu_ualink_handle handle) 3267 { 3268 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3269 struct amdgpu_bo *bo; 3270 int r = 0; 3271 3272 /* Remove the entry from the Xarray. */ 3273 xa_lock(&adev->ualink.imp_xa); 3274 imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo); 3275 if (!imp_xa_node) { 3276 xa_unlock(&adev->ualink.imp_xa); 3277 dev_warn(adev->dev, 3278 "NPA-REVOKE: Handle (%llx:%llx) not found\n", 3279 handle.handle_hi, handle.handle_lo); 3280 return; 3281 } 3282 3283 /* Confirm that the complete handle matches. */ 3284 if (handle.handle_hi != imp_xa_node->handle.handle_hi) { 3285 xa_unlock(&adev->ualink.imp_xa); 3286 dev_warn(adev->dev, 3287 "NPA-REVOKE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n", 3288 imp_xa_node->handle.handle_hi, 3289 handle.handle_hi, handle.handle_lo, remote_acc_id); 3290 return; 3291 } 3292 3293 switch (READ_ONCE(imp_xa_node->node_state)) { 3294 case AMDGPU_UALINK_NODE_READY: 3295 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 3296 list_del_init(&imp_xa_node->list); 3297 xa_unlock(&adev->ualink.imp_xa); 3298 3299 /* Invalidate the GPUVM mappings */ 3300 bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv); 3301 amdgpu_ualink_invalidate_import_mappings(bo); 3302 3303 /* Drop the refcount for the node */ 3304 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 3305 break; 3306 case AMDGPU_UALINK_NODE_PENDING: 3307 /* The import is still building the dma-buf and nothing has 3308 * been handed to user-space yet. The importing thread sees 3309 * the teardown state and unwinds. 3310 */ 3311 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 3312 xa_unlock(&adev->ualink.imp_xa); 3313 break; 3314 default: 3315 /* NPA-REVOKE always follows NPA-RSP, so a NOT_READY node means 3316 * a stale revoke, and a node in teardown is already being 3317 * released by whoever moved it there. 3318 */ 3319 xa_unlock(&adev->ualink.imp_xa); 3320 return; 3321 } 3322 3323 r = amdgpu_ualink_send_npa_release_msg(adev, remote_acc_id, handle); 3324 if (r) 3325 dev_err(adev->dev, 3326 "NPA-Release send failed remote:%u handle:%llx:%llx error:%d\n", 3327 remote_acc_id, handle.handle_hi, handle.handle_lo, r); 3328 } 3329 3330 static void amdgpu_ualink_process_npa_fail_msg(struct amdgpu_device *adev, 3331 u32 remote_acc_id, u64 partial_handle, 3332 u32 fail_reason) 3333 { 3334 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3335 int r = 0; 3336 3337 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) { 3338 dev_warn(adev->dev, 3339 "NPA-FAIL: no connection with remote AccId:%u\n", 3340 remote_acc_id); 3341 goto conn_setup; 3342 } 3343 3344 xa_lock(&adev->ualink.imp_xa); 3345 imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle); 3346 if (!imp_xa_node) { 3347 xa_unlock(&adev->ualink.imp_xa); 3348 dev_warn(adev->dev, 3349 "NPA-FAIL: imp XA handle not found:%llx\n", 3350 partial_handle); 3351 return; 3352 } 3353 3354 imp_xa_node->fail_reason = fail_reason; 3355 /* Signal completion done to signal response received for NPA-REQ 3356 * message. 3357 * If the node is in NOT_READY state, then set the node state to 3358 * PENDING and signal the completion. If the node is not in NOT_READY 3359 * state, then it is an unsolicited NPA-FAIL message and we 3360 * log a debug message. 3361 */ 3362 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) { 3363 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING); 3364 complete(&imp_xa_node->npa_done); 3365 } else { 3366 dev_dbg(adev->dev, 3367 "NPA-FAIL: unsolicited for handle:%llx:%llx from AccId:%u\n", 3368 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 3369 remote_acc_id); 3370 } 3371 xa_unlock(&adev->ualink.imp_xa); 3372 3373 return; 3374 3375 conn_setup: 3376 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3377 if (r) 3378 dev_warn(adev->dev, 3379 "NPA-FAIL: connection setup failed with remote AccId:%u\n", 3380 remote_acc_id); 3381 } 3382 3383 static void amdgpu_ualink_process_npa_rsp_msg(struct amdgpu_device *adev, 3384 u32 remote_acc_id, u64 partial_handle, 3385 u64 npa_addr, u64 size) 3386 { 3387 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3388 int r = 0; 3389 3390 /* Check if the connection is established. If it is not, then start 3391 * connection setup. 3392 */ 3393 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) { 3394 dev_warn(adev->dev, 3395 "NPA-RSP: no connection with remote AccId:%u\n", 3396 remote_acc_id); 3397 goto conn_setup; 3398 } 3399 3400 xa_lock(&adev->ualink.imp_xa); 3401 imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle); 3402 if (!imp_xa_node) { 3403 xa_unlock(&adev->ualink.imp_xa); 3404 dev_warn(adev->dev, 3405 "NPA-RSP: imp XA handle not found:%llx\n", partial_handle); 3406 return; 3407 } 3408 3409 /* NPA addr received in NPA-RSP is page aligned and without the remote 3410 * GPU-id in Bits 41-50. Assemble back the NPA address before storing 3411 * it. 3412 */ 3413 imp_xa_node->npa_addr = GENERATE_NPA(npa_addr, remote_acc_id); 3414 /* Size is in number of GPU pages granularity. */ 3415 imp_xa_node->size = size; 3416 3417 /* Signal completion done to signal NPA_RSP received. 3418 * If the node is in NOT_READY state, then set the node state to 3419 * PENDING and signal the completion. If the node is not in NOT_READY 3420 * state, then it is an unsolicited NPA-RSP message and we 3421 * log a debug message. 3422 */ 3423 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) { 3424 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING); 3425 complete(&imp_xa_node->npa_done); 3426 } else { 3427 dev_dbg(adev->dev, 3428 "NPA-RSP: unsolicited for handle:%llx:%llx from AccId:%u\n", 3429 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 3430 remote_acc_id); 3431 } 3432 xa_unlock(&adev->ualink.imp_xa); 3433 3434 return; 3435 3436 conn_setup: 3437 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3438 if (r) 3439 dev_warn(adev->dev, 3440 "NPA-RSP: connection setup failed with remote AccId:%u\n", 3441 remote_acc_id); 3442 } 3443 3444 static void amdgpu_ualink_process_npa_req_msg(struct amdgpu_device *adev, 3445 u32 remote_acc_id, 3446 struct amdgpu_ualink_handle handle) 3447 { 3448 struct amdgpu_ualink_importer_entry *importer_entry, *npa_addr_entry; 3449 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 3450 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3451 u64 range_start, range_end, pte_flags; 3452 struct drm_mm_node *mm_node = NULL; 3453 int r = 0, fail_reason = 0; 3454 bool send_npa_fail = true; 3455 u64 npa_addr = 0, size; 3456 struct amdgpu_bo *bo; 3457 u32 gen_count; 3458 3459 /* Check if the connection is established. If it is not, then start 3460 * connection setup. 3461 */ 3462 gen_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0); 3463 if (!gen_count) { 3464 dev_warn(adev->dev, 3465 "NPA-REQ: no connection with remote AccId:%u\n", 3466 remote_acc_id); 3467 goto conn_setup; 3468 } 3469 3470 /* Check entry exists in Exporter XA. If yes, increase the refcount 3471 * for the node. 3472 */ 3473 xa_lock(&adev->ualink.exp_xa); 3474 exp_xa_node = xa_load(&adev->ualink.exp_xa, handle.handle_lo); 3475 if (!exp_xa_node || (handle.handle_hi != exp_xa_node->handle.handle_hi) || 3476 !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) { 3477 xa_unlock(&adev->ualink.exp_xa); 3478 dev_warn(adev->dev, 3479 "NPA-REQ: exp XA handle not found handle:%llx:%llx\n", 3480 handle.handle_hi, handle.handle_lo); 3481 fail_reason = AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE; 3482 goto handle_invalid_fail; 3483 } 3484 xa_unlock(&adev->ualink.exp_xa); 3485 3486 bo = exp_xa_node->bo; 3487 size = amdgpu_bo_ngpu_pages(bo); 3488 3489 /* Pin the BO */ 3490 r = amdgpu_bo_reserve(bo, true); 3491 if (unlikely(r)) { 3492 dev_warn(adev->dev, 3493 "NPA-REQ: BO reserve failed handle:%llx:%llx\n", 3494 handle.handle_hi, handle.handle_lo); 3495 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3496 goto bo_reserve_fail; 3497 } 3498 r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM); 3499 amdgpu_bo_unreserve(bo); 3500 if (r) { 3501 dev_warn(adev->dev, 3502 "NPA-REQ: BO pin failed handle:%llx:%llx\n", 3503 handle.handle_hi, handle.handle_lo); 3504 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3505 goto bo_pin_fail; 3506 } 3507 3508 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 3509 mutex_lock(&exp_xa_node->node_lock); 3510 importer_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3511 npa_addr_entry = importer_entry; 3512 mutex_unlock(&exp_xa_node->node_lock); 3513 /* Check if NPA address is already allocated for this importer. 3514 * If yes, then send the NPA-FAIL message back to the remote GPU. 3515 */ 3516 if (importer_entry->npa_addr) { 3517 fail_reason = AMDGPU_UALINK_NPA_FAIL_DUPLICATE; 3518 goto npa_duplicate_fail; 3519 } 3520 3521 range_start = ((u64)remote_acc_id << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT) | 3522 AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED; 3523 range_end = range_start | AMDGPU_UALINK_NPA_ADDR_RANGE_MASK; 3524 } else { 3525 /* We store NPA-address in importer_entries[0] in 3526 * Source-Aliasing mode. 3527 */ 3528 mutex_lock(&exp_xa_node->node_lock); 3529 npa_addr_entry = &exp_xa_node->importer_entries[0]; 3530 importer_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3531 /* Check if NPA address is already allocated for this importer. 3532 * If yes, then set the corresponding bit in the importers_bitmap, 3533 * set the generation count and send the NPA-RSP back to the remote GPU. 3534 */ 3535 if (npa_addr_entry->npa_addr) { 3536 npa_addr = npa_addr_entry->npa_addr; 3537 set_bit(remote_acc_id, exp_xa_node->importers_bitmap); 3538 importer_entry->generation_count = gen_count; 3539 mutex_unlock(&exp_xa_node->node_lock); 3540 dev_dbg(adev->dev, 3541 "NPA-REQ: NPA:%llx size:%llx handle:%llx:%llx\n", 3542 npa_addr, size, handle.handle_hi, handle.handle_lo); 3543 3544 goto send_npa_rsp; 3545 } 3546 mutex_unlock(&exp_xa_node->node_lock); 3547 3548 range_start = 0; 3549 range_end = 0; 3550 } 3551 3552 mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL); 3553 if (!mm_node) { 3554 dev_warn(adev->dev, 3555 "NPA-REQ: mm_node alloc failed handle:%llx:%llx\n", 3556 handle.handle_hi, handle.handle_lo); 3557 fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE; 3558 goto mem_alloc_fail; 3559 } 3560 3561 /* Allocate NPA address */ 3562 r = amdgpu_ualink_npa_alloc_va(adev, mm_node, 0, range_start, 3563 range_end, size); 3564 if (r) { 3565 dev_warn(adev->dev, 3566 "NPA-REQ: NPA addr alloc failed handle:%llx:%llx\n", 3567 handle.handle_hi, handle.handle_lo); 3568 fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE; 3569 goto npa_alloc_fail; 3570 } 3571 npa_addr = mm_node->start; 3572 3573 pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 0); 3574 dev_dbg(adev->dev, 3575 "NPA-REQ: Allocated NPA:%llx size:%llx PTE:%llx handle:%llx:%llx\n", 3576 npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo); 3577 3578 /* Map the NPA address into NPA VM*/ 3579 r = amdgpu_ualink_map_npa_addr(adev, npa_addr, size, bo, 0, pte_flags); 3580 if (r) { 3581 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3582 dev_warn(adev->dev, 3583 "NPA-REQ: NPA addr (%llx) map failed handle:%llx:%llx\n", 3584 npa_addr, handle.handle_hi, handle.handle_lo); 3585 goto map_npa_fail; 3586 } 3587 3588 dev_dbg(adev->dev, 3589 "NPA-REQ: Mapped NPA:%llx size:%llx pte:%llx handle:%llx:%llx\n", 3590 npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo); 3591 send_npa_rsp: 3592 /* Send NPA-RSP back to the remote GPU */ 3593 r = amdgpu_ualink_send_npa_rsp_msg(adev, remote_acc_id, handle, 3594 STRIP_NPA(npa_addr), size); 3595 if (r) { 3596 dev_warn(adev->dev, 3597 "NPA-REQ: send NPA-RSP failed remote:%u handle:%llx:%llx\n", 3598 remote_acc_id, handle.handle_hi, handle.handle_lo); 3599 send_npa_fail = false; 3600 goto send_npa_rsp_fail; 3601 } 3602 3603 dev_dbg(adev->dev, 3604 "NPA-REQ: Sent NPA-RSP with NPA:%llx size:%llx handle:%llx:%llx\n", 3605 npa_addr, size, handle.handle_hi, handle.handle_lo); 3606 3607 /* If this is the first time we are setting the bit for this importer, 3608 * then store the NPA address, mm_node and generation count. 3609 */ 3610 mutex_lock(&exp_xa_node->node_lock); 3611 if (!test_and_set_bit(remote_acc_id, exp_xa_node->importers_bitmap)) { 3612 npa_addr_entry->npa_addr = npa_addr; 3613 npa_addr_entry->mm_node = mm_node; 3614 importer_entry->generation_count = gen_count; 3615 } 3616 mutex_unlock(&exp_xa_node->node_lock); 3617 3618 dev_dbg(adev->dev, 3619 "NPA-REQ: BO pin_count:%d, importers:%d, handle:%llx:%llx\n", 3620 bo->tbo.pin_count, bitmap_weight(exp_xa_node->importers_bitmap, 3621 AMDGPU_UALINK_ACCEL_MAX), handle.handle_hi, handle.handle_lo); 3622 WARN_ON(bo->tbo.pin_count < bitmap_weight(exp_xa_node->importers_bitmap, 3623 AMDGPU_UALINK_ACCEL_MAX)); 3624 3625 /* Add this node to the exported handles list for the remote GPU, 3626 * but only if the node is still in exp_xa. If revoke already erased 3627 * it, skip the list_add to avoid a dangling list entry. The cleanup 3628 * worker is guaranteed to run after we drop our ref, so it will see 3629 * this importer in the bitmap and send NPA-REVOKE. 3630 */ 3631 xa_lock(&adev->ualink.exp_xa); 3632 if (xa_load(&adev->ualink.exp_xa, exp_xa_node->handle.handle_lo) == exp_xa_node) 3633 list_add(&importer_entry->list, &adev->ualink.exp_handles_list[remote_acc_id]); 3634 xa_unlock(&adev->ualink.exp_xa); 3635 3636 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3637 3638 return; 3639 3640 send_npa_rsp_fail: 3641 mutex_lock(&exp_xa_node->node_lock); 3642 clear_bit(remote_acc_id, exp_xa_node->importers_bitmap); 3643 mutex_unlock(&exp_xa_node->node_lock); 3644 if (mm_node) 3645 amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size); 3646 3647 map_npa_fail: 3648 if (mm_node) 3649 amdgpu_ualink_npa_free_va(adev, mm_node); 3650 3651 npa_alloc_fail: 3652 kfree(mm_node); 3653 mem_alloc_fail: 3654 npa_duplicate_fail: 3655 r = amdgpu_bo_reserve(bo, true); 3656 if (likely(!r)) { 3657 amdgpu_bo_unpin(bo); 3658 amdgpu_bo_unreserve(bo); 3659 } else { 3660 dev_warn(adev->dev, 3661 "NPA-REQ: BO reserve to unpin failed for handle:%llx:%llx\n", 3662 handle.handle_hi, handle.handle_lo); 3663 } 3664 3665 bo_pin_fail: 3666 bo_reserve_fail: 3667 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3668 3669 handle_invalid_fail: 3670 if (send_npa_fail) { 3671 r = amdgpu_ualink_send_npa_fail_msg(adev, remote_acc_id, 3672 handle, fail_reason); 3673 if (r) 3674 dev_warn(adev->dev, 3675 "NPA-REQ: send NPA-FAIL failed remote:%u handle:%llx:%llx\n", 3676 remote_acc_id, handle.handle_hi, handle.handle_lo); 3677 } 3678 return; 3679 conn_setup: 3680 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3681 if (r) 3682 dev_warn(adev->dev, 3683 "NPA-REQ: connection setup failed with remote AccId:%u\n", 3684 remote_acc_id); 3685 } 3686 3687 static int amdgpu_ualink_translate_npa_fail_reason(struct amdgpu_device *adev, 3688 u32 fail_reason) 3689 { 3690 switch (fail_reason) { 3691 case AMDGPU_UALINK_NPA_FAIL_NOSPACE: 3692 return -ENOSPC; 3693 case AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE: 3694 case AMDGPU_UALINK_NPA_FAIL_DUPLICATE: 3695 case AMDGPU_UALINK_NPA_FAIL_ERROR: 3696 return -EINVAL; 3697 default: 3698 dev_err(adev->dev, 3699 "IMPORT: invalid NPA-FAIL reason:%u\n", 3700 fail_reason); 3701 return -EINVAL; 3702 } 3703 } 3704 3705 static int amdgpu_ualink_do_import_handle(struct amdgpu_device *adev, 3706 struct amdgpu_ualink_imp_xa_node *imp_xa_node, 3707 u32 remote_acc_id) 3708 { 3709 struct amdgpu_ualink_handle handle = imp_xa_node->handle; 3710 u32 generation_count; 3711 int r; 3712 3713 /* First check if the connection is setup with the 3714 * remote GPU. If yes, then initiate the NPA protocol to 3715 * get the NPA address. 3716 * If not, then initiate the HELLO protocol to first setup 3717 * the connection and once the connection is setup, then 3718 * initiate the NPA protocol. 3719 */ 3720 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3721 if (r) { 3722 if (r != -EAGAIN) 3723 dev_warn(adev->dev, 3724 "IMPORT: connection setup failed with remote AccId:%u\n", 3725 remote_acc_id); 3726 return r; 3727 } 3728 3729 /* Send NPA_REQ message */ 3730 r = amdgpu_ualink_send_npa_req_msg(adev, remote_acc_id, handle); 3731 if (r) { 3732 dev_warn(adev->dev, 3733 "IMPORT: NPA-REQ send failed to remote AccId:%u\n", 3734 remote_acc_id); 3735 goto reset_conn; 3736 } 3737 3738 /* Wait for the NPA_RSP to come back */ 3739 r = wait_for_completion_interruptible_timeout(&imp_xa_node->npa_done, 3740 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 3741 if (r == -ERESTARTSYS) { 3742 dev_err_ratelimited(adev->dev, 3743 "IMPORT: NPA-RSP wait interrupted by signal\n"); 3744 return r; 3745 } else if (r == 0) { 3746 dev_warn(adev->dev, 3747 "IMPORT: NPA-RSP timeout from remote AccId:%u\n", 3748 remote_acc_id); 3749 r = -ETIMEDOUT; 3750 goto reset_conn; 3751 } 3752 3753 /* If the NPA addr/size isn't filled with valid values, then either 3754 * we got a NPA_FAIL or something bad happened. In either case, we 3755 * return the error back to user-space. 3756 */ 3757 if (imp_xa_node->fail_reason) { 3758 dev_warn(adev->dev, 3759 "IMPORT: NPA-REQ failed with fail_reason:%d handle:%llx:%llx\n", 3760 imp_xa_node->fail_reason, handle.handle_hi, handle.handle_lo); 3761 return amdgpu_ualink_translate_npa_fail_reason(adev, 3762 imp_xa_node->fail_reason); 3763 } 3764 3765 if (!imp_xa_node->npa_addr || !imp_xa_node->size) { 3766 dev_warn(adev->dev, 3767 "IMPORT: invalid npa:%llx or size:%llx\n", 3768 imp_xa_node->npa_addr, imp_xa_node->size); 3769 imp_xa_node->npa_addr = 0; 3770 imp_xa_node->size = 0; 3771 return -EINVAL; 3772 } 3773 3774 r = amdgpu_ualink_map_npa_to_dmabuf(adev, imp_xa_node); 3775 if (r) { 3776 dev_warn(adev->dev, 3777 "IMPORT: dmabuf creation failed npa:%llx size:%llx\n", 3778 imp_xa_node->npa_addr, imp_xa_node->size); 3779 imp_xa_node->npa_addr = 0; 3780 imp_xa_node->size = 0; 3781 return r; 3782 } 3783 3784 /* Add this node to the imported handles list for the remote GPU, 3785 * unless the exporter revoked the handle while the import was in 3786 * flight. The dmabuf is released with the last node reference. 3787 */ 3788 xa_lock(&adev->ualink.imp_xa); 3789 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_TEARDOWN) { 3790 xa_unlock(&adev->ualink.imp_xa); 3791 dev_dbg(adev->dev, 3792 "IMPORT: handle:%llx:%llx revoked during import\n", 3793 handle.handle_hi, handle.handle_lo); 3794 return -EINVAL; 3795 } 3796 list_add(&imp_xa_node->list, &adev->ualink.imp_handles_list[remote_acc_id]); 3797 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_READY); 3798 xa_unlock(&adev->ualink.imp_xa); 3799 3800 return 0; 3801 3802 reset_conn: 3803 dev_dbg(adev->dev, 3804 "IMPORT: Resetting connection for remote:%u\n", remote_acc_id); 3805 generation_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0); 3806 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 3807 AMDGPU_UALINK_CONN_NOT_READY, 3808 generation_count); 3809 return r; 3810 } 3811 3812 static int amdgpu_ualink_local_import(struct amdgpu_device *adev, 3813 u32 remote_acc_id, 3814 struct amdgpu_ualink_handle *handle, 3815 int *fd_out) 3816 { 3817 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3818 struct amdgpu_device *peer_adev; 3819 int fd; 3820 3821 mutex_lock(&mgpu_info.mutex); 3822 peer_adev = find_peer_adev(remote_acc_id); 3823 mutex_unlock(&mgpu_info.mutex); 3824 if (!peer_adev) { 3825 dev_err(adev->dev, 3826 "IMPORT LOCAL: peer adev not found for AccId:%u\n", 3827 remote_acc_id); 3828 return -ENODEV; 3829 } 3830 3831 xa_lock(&peer_adev->ualink.exp_xa); 3832 exp_xa_node = xa_load(&peer_adev->ualink.exp_xa, handle->handle_lo); 3833 if (!exp_xa_node || 3834 exp_xa_node->handle.handle_hi != handle->handle_hi || 3835 !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) { 3836 xa_unlock(&peer_adev->ualink.exp_xa); 3837 dev_err(adev->dev, 3838 "IMPORT LOCAL: handle:%llx:%llx not found in peer exp_xa\n", 3839 handle->handle_hi, handle->handle_lo); 3840 return -EINVAL; 3841 } 3842 xa_unlock(&peer_adev->ualink.exp_xa); 3843 3844 get_dma_buf(exp_xa_node->dmabuf); 3845 /* Get a new fd for the DMABuf */ 3846 fd = dma_buf_fd(exp_xa_node->dmabuf, O_CLOEXEC | O_RDWR); 3847 if (fd < 0) { 3848 dev_err(adev->dev, 3849 "IMPORT LOCAL: dma-buf fd failed handle:%llx:%llx\n", 3850 handle->handle_hi, handle->handle_lo); 3851 dma_buf_put(exp_xa_node->dmabuf); 3852 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3853 return fd; 3854 } 3855 3856 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3857 *fd_out = fd; 3858 return 0; 3859 } 3860 3861 int amdgpu_ualink_import_handle(struct drm_device *dev, 3862 const struct amdgpu_ualink_handle *ualink_handle, 3863 int *fd_out) 3864 { 3865 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3866 struct amdgpu_device *adev = drm_to_adev(dev); 3867 struct amdgpu_ualink_handle handle = *ualink_handle; 3868 u32 remote_acc_id, node_state; 3869 int r = 0, fd; 3870 3871 remote_acc_id = (handle.handle_lo & 3872 AMDGPU_UALINK_HANDLE_ACCID_MASK); 3873 3874 if (remote_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 3875 dev_err(adev->dev, 3876 "IMPORT: invalid remote AccId:%u\n", remote_acc_id); 3877 return -EINVAL; 3878 } 3879 3880 if (amdgpu_ualink_is_local_accel(adev, remote_acc_id)) 3881 return amdgpu_ualink_local_import(adev, remote_acc_id, 3882 &handle, fd_out); 3883 3884 xa_lock(&adev->ualink.imp_xa); 3885 imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo); 3886 3887 if (imp_xa_node) { 3888 /* If node state is Not_ready/Pending, then some other 3889 * thread is already trying the NPA protocol for the same 3890 * ualink handle. Back off and let the thread finish. 3891 * If the node state is Teardown, then it means this 3892 * node is about to be removed. So let user-space know 3893 * that this handle is invalid. 3894 */ 3895 node_state = READ_ONCE(imp_xa_node->node_state); 3896 if (node_state == AMDGPU_UALINK_NODE_NOT_READY || 3897 node_state == AMDGPU_UALINK_NODE_PENDING) { 3898 xa_unlock(&adev->ualink.imp_xa); 3899 r = -EAGAIN; 3900 goto out; 3901 } else if (node_state == AMDGPU_UALINK_NODE_TEARDOWN) { 3902 xa_unlock(&adev->ualink.imp_xa); 3903 r = -EINVAL; 3904 goto out; 3905 } 3906 3907 /* Increase the refcount while we are processing the request */ 3908 r = amdgpu_ualink_imp_xa_entry_get(imp_xa_node) ? 0 : -EINVAL; 3909 xa_unlock(&adev->ualink.imp_xa); 3910 3911 /* If the refcount has become 0 but the entry is not yet 3912 * removed from the Xarray, then return error to user-space. 3913 */ 3914 if (r) 3915 goto out; 3916 } else { 3917 xa_unlock(&adev->ualink.imp_xa); 3918 /* if the partial handle doesn't exist in the Importer xarray then 3919 * initiate the NPA protocol and generate the DMABuf corresponding 3920 * to the NPA address. 3921 * First store the entry in the Xarray. 3922 */ 3923 imp_xa_node = kzalloc(sizeof(*imp_xa_node), GFP_KERNEL); 3924 if (!imp_xa_node) { 3925 r = -ENOMEM; 3926 goto out; 3927 } 3928 imp_xa_node->adev = adev; 3929 imp_xa_node->node_state = AMDGPU_UALINK_NODE_NOT_READY; 3930 imp_xa_node->handle = handle; 3931 init_completion(&imp_xa_node->npa_done); 3932 INIT_LIST_HEAD(&imp_xa_node->list); 3933 kref_init(&imp_xa_node->refcount); 3934 3935 /* Take an extra reference to store in the Xarray. The error 3936 * handling paths will drop both these references, while a 3937 * successful path will drop only one reference to the Xarray 3938 * entry. 3939 */ 3940 amdgpu_ualink_imp_xa_entry_get(imp_xa_node); 3941 3942 /* Check if another thread created a node for the same handle while 3943 * we were trying to create and initialize the node. 3944 */ 3945 r = xa_insert(&adev->ualink.imp_xa, handle.handle_lo, 3946 imp_xa_node, GFP_KERNEL); 3947 if (r) { 3948 kfree(imp_xa_node); 3949 /* -EBUSY means another thread raced us and inserted a 3950 * node for the same handle. Ask user-space to retry. 3951 */ 3952 if (r == -EBUSY) 3953 r = -EAGAIN; 3954 else 3955 dev_err(adev->dev, 3956 "IMPORT: XA insert failed for handle:%llx:%llx err:%d\n", 3957 handle.handle_hi, handle.handle_lo, r); 3958 goto out; 3959 } 3960 3961 r = amdgpu_ualink_do_import_handle(adev, imp_xa_node, remote_acc_id); 3962 3963 /* If error is returned, then cleanup the xarray entry before returning 3964 * the error back to user-space 3965 */ 3966 if (r) { 3967 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 3968 if (r != -EAGAIN) 3969 dev_err(adev->dev, 3970 "IMPORT: XA import failed for handle:%llx:%llx\n", 3971 handle.handle_hi, handle.handle_lo); 3972 goto cleanup; 3973 } 3974 } 3975 3976 /* dma_buf_fd consumes a reference and assigns it to the fd. 3977 * Therefore take an extra reference to be consumed. It will be 3978 * released when user mode closes the fd. 3979 */ 3980 get_dma_buf(imp_xa_node->dmabuf); 3981 3982 fd = dma_buf_fd(imp_xa_node->dmabuf, O_CLOEXEC | O_RDWR); 3983 if (fd >= 0) { 3984 *fd_out = fd; 3985 } else { 3986 dma_buf_put(imp_xa_node->dmabuf); 3987 r = fd; 3988 dev_err(adev->dev, 3989 "IMPORT: dma-buf fd creation failed handle:%llx:%llx\n", 3990 handle.handle_hi, handle.handle_lo); 3991 } 3992 3993 cleanup: 3994 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 3995 out: 3996 return r; 3997 } 3998 3999 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp, 4000 u32 gem_handle, 4001 struct amdgpu_ualink_handle *handle_out) 4002 { 4003 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 4004 struct amdgpu_ualink_handle handle; 4005 struct drm_gem_object *gobj; 4006 struct amdgpu_device *adev; 4007 struct amdgpu_bo *robj; 4008 int r = 0, i; 4009 4010 gobj = drm_gem_object_lookup(filp, gem_handle); 4011 if (!gobj) 4012 return -ENOENT; 4013 4014 robj = gem_to_amdgpu_bo(gobj); 4015 adev = amdgpu_ttm_adev(robj->tbo.bdev); 4016 4017 if (!(robj->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { 4018 dev_err(adev->dev, "Only VRAM BOs can be exported\n"); 4019 r = -EOPNOTSUPP; 4020 goto out; 4021 } 4022 4023 if (!robj->ualink_handle_lo) { 4024 /* If no ualink handle generated for BO, then generate one and 4025 * add it to the exporter Xarray. 4026 */ 4027 exp_xa_node = kzalloc(sizeof(*exp_xa_node), GFP_KERNEL); 4028 if (!exp_xa_node) { 4029 dev_err(adev->dev, "Failed to allocate exp_xa_node\n"); 4030 r = -ENOMEM; 4031 goto out; 4032 } 4033 4034 amdgpu_bo_ref(robj); 4035 exp_xa_node->bo = robj; 4036 init_completion(&exp_xa_node->npa_done); 4037 bitmap_zero(exp_xa_node->importers_bitmap, 4038 AMDGPU_UALINK_ACCEL_MAX); 4039 bitmap_zero(exp_xa_node->npa_release_bitmap, 4040 AMDGPU_UALINK_ACCEL_MAX); 4041 kref_init(&exp_xa_node->refcount); 4042 mutex_init(&exp_xa_node->node_lock); 4043 INIT_WORK(&exp_xa_node->cleanup_work, 4044 amdgpu_ualink_exp_cleanup_worker); 4045 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4046 INIT_LIST_HEAD(&exp_xa_node->importer_entries[i].list); 4047 exp_xa_node->importer_entries[i].parent = exp_xa_node; 4048 } 4049 /* DMABuf handle for local import of fabric handles */ 4050 exp_xa_node->dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, filp, 4051 gem_handle, DRM_CLOEXEC | DRM_RDWR); 4052 if (IS_ERR(exp_xa_node->dmabuf)) { 4053 r = PTR_ERR(exp_xa_node->dmabuf); 4054 dev_err(adev->dev, "Failed to generate DMABuf for the BO\n"); 4055 goto err_unref_bo; 4056 } 4057 4058 xa_lock(&adev->ualink.exp_xa); 4059 amdgpu_generate_ualink_handle(adev, &handle); 4060 exp_xa_node->handle = handle; 4061 r = __xa_insert(&adev->ualink.exp_xa, handle.handle_lo, 4062 exp_xa_node, GFP_KERNEL); 4063 xa_unlock(&adev->ualink.exp_xa); 4064 if (r) { 4065 dev_err(adev->dev, "Failed to insert exp_xa_node into XA: %d\n", r); 4066 goto err_put_dmabuf; 4067 } 4068 4069 r = amdgpu_bo_reserve(robj, false); 4070 if (r) 4071 goto err_erase_xa; 4072 4073 if (robj->ualink_handle_lo) { 4074 amdgpu_bo_unreserve(robj); 4075 r = -EAGAIN; 4076 goto err_erase_xa; 4077 } 4078 4079 robj->ualink_handle_lo = handle.handle_lo; 4080 amdgpu_bo_unreserve(robj); 4081 4082 /* Return the generated handle back to the caller */ 4083 *handle_out = handle; 4084 } else { 4085 handle_out->handle_lo = robj->ualink_handle_lo; 4086 4087 /* Do a sanity check to ensure the handle exists in the XA */ 4088 xa_lock(&adev->ualink.exp_xa); 4089 exp_xa_node = xa_load(&adev->ualink.exp_xa, 4090 robj->ualink_handle_lo); 4091 xa_unlock(&adev->ualink.exp_xa); 4092 WARN(!exp_xa_node, "Exp XA: Handle_Lo: %llx not found", 4093 robj->ualink_handle_lo); 4094 if (exp_xa_node) 4095 handle_out->handle_hi = exp_xa_node->handle.handle_hi; 4096 } 4097 4098 drm_gem_object_put(gobj); 4099 return 0; 4100 4101 err_erase_xa: 4102 xa_erase(&adev->ualink.exp_xa, handle.handle_lo); 4103 err_put_dmabuf: 4104 dma_buf_put(exp_xa_node->dmabuf); 4105 err_unref_bo: 4106 amdgpu_bo_unref(&exp_xa_node->bo); 4107 kfree(exp_xa_node); 4108 out: 4109 drm_gem_object_put(gobj); 4110 return r; 4111 } 4112 4113 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data, 4114 struct drm_file *filp) 4115 { 4116 union drm_amdgpu_ualink_handle *args = data; 4117 struct amdgpu_device *adev = drm_to_adev(dev); 4118 struct amdgpu_ualink_handle handle = {}; 4119 u32 gem_handle; 4120 int r, fd = -1; 4121 4122 /* The ioctl is registered for every amdgpu device, but the UALink 4123 * software state only exists on ASICs that instantiate the UALink IP 4124 * block. Bail out if it is absent, and short-circuit before the 4125 * accel_state dereference below. 4126 */ 4127 if (!adev->ualink.info || 4128 adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) { 4129 dev_dbg(adev->dev, 4130 "ualink device not available or not in active state\n"); 4131 return -EOPNOTSUPP; 4132 } 4133 4134 /* The input and output members of the ioctl argument alias each other. 4135 * Latch every input field before invoking the handlers, and only write 4136 * the output fields afterwards. 4137 */ 4138 switch (args->in.op) { 4139 case DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT: 4140 gem_handle = args->in.gem_handle; 4141 r = amdgpu_ualink_export_handle(dev, filp, gem_handle, &handle); 4142 if (!r) { 4143 args->out.export_ualink_handle[0] = handle.handle_lo; 4144 args->out.export_ualink_handle[1] = handle.handle_hi; 4145 } 4146 break; 4147 case DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT: 4148 handle.handle_lo = args->in.import_ualink_handle[0]; 4149 handle.handle_hi = args->in.import_ualink_handle[1]; 4150 r = amdgpu_ualink_import_handle(dev, &handle, &fd); 4151 if (!r) 4152 args->out.import_dmabuf_handle = fd; 4153 break; 4154 default: 4155 r = -EINVAL; 4156 break; 4157 } 4158 4159 return r; 4160 } 4161 4162 int amdgpu_ualink_manager_start(struct amdgpu_device *adev) 4163 { 4164 int i, r; 4165 4166 r = amdgpu_vm_init(adev, &adev->ualink.npa_vm, 0); 4167 if (r) 4168 goto out; 4169 4170 /* For pinning page tables and using CPU for page table updates. */ 4171 r = amdgpu_vm_make_npa(adev, &adev->ualink.npa_vm); 4172 if (r) 4173 goto uninit_vm; 4174 4175 adev->ualink.npa_wq = alloc_workqueue("NPA WQ", WQ_UNBOUND, 0); 4176 if (unlikely(!adev->ualink.npa_wq)) { 4177 dev_err(adev->dev, "Failed to allocate NPA WQ\n"); 4178 r = -ENOMEM; 4179 goto uninit_vm; 4180 } 4181 4182 /* Map this VM to NPA VMID */ 4183 adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 4184 amdgpu_gmc_pd_addr(adev->ualink.npa_vm.root.bo)); 4185 4186 xa_init(&adev->ualink.exp_xa); 4187 xa_init(&adev->ualink.imp_xa); 4188 xa_init(&adev->ualink.handle_invalid_xa); 4189 4190 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4191 init_completion(&adev->ualink.conn_state[i].hello_done); 4192 mutex_init(&adev->ualink.conn_state[i].lock); 4193 adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY; 4194 INIT_LIST_HEAD(&adev->ualink.exp_handles_list[i]); 4195 INIT_LIST_HEAD(&adev->ualink.imp_handles_list[i]); 4196 } 4197 4198 amdgpu_ualink_npa_mm_init(adev); 4199 4200 return 0; 4201 4202 uninit_vm: 4203 amdgpu_vm_fini(adev, &adev->ualink.npa_vm); 4204 out: 4205 return r; 4206 } 4207 4208 /* Free every importer entry left in imp_xa at manager stop. */ 4209 static void amdgpu_ualink_teardown_imp_xa_entries(struct amdgpu_device *adev) 4210 { 4211 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 4212 unsigned long index; 4213 4214 xa_for_each(&adev->ualink.imp_xa, index, imp_xa_node) { 4215 list_del_init(&imp_xa_node->list); 4216 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 4217 amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node); 4218 } 4219 } 4220 4221 /* Free a single exporter node at manager stop. This is the message-free 4222 * portion of amdgpu_ualink_exp_cleanup_worker(). 4223 */ 4224 static void amdgpu_ualink_teardown_exp_xa_node(struct amdgpu_device *adev, 4225 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 4226 { 4227 DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 4228 struct amdgpu_bo *bo = exp_xa_node->bo; 4229 u32 remote_acc_id; 4230 4231 bitmap_copy(importers_bitmap, exp_xa_node->importers_bitmap, 4232 AMDGPU_UALINK_ACCEL_MAX); 4233 4234 dev_dbg(adev->dev, 4235 "EXP-STOP: handle:%llx:%llx importers bitmap: %*pbl\n", 4236 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 4237 AMDGPU_UALINK_ACCEL_MAX, importers_bitmap); 4238 4239 if (!bitmap_empty(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)) { 4240 amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node); 4241 4242 /* Unlink importers and unpin the BO once per importer. */ 4243 if (likely(!amdgpu_bo_reserve(bo, true))) { 4244 bo->ualink_handle_lo = 0ULL; 4245 for_each_set_bit(remote_acc_id, importers_bitmap, 4246 AMDGPU_UALINK_ACCEL_MAX) { 4247 list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list); 4248 amdgpu_bo_unpin(bo); 4249 } 4250 amdgpu_bo_unreserve(bo); 4251 } else { 4252 dev_warn(adev->dev, 4253 "EXP-STOP: BO reserve to unpin failed handle:%llx:%llx\n", 4254 exp_xa_node->handle.handle_hi, 4255 exp_xa_node->handle.handle_lo); 4256 for_each_set_bit(remote_acc_id, importers_bitmap, 4257 AMDGPU_UALINK_ACCEL_MAX) 4258 list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list); 4259 } 4260 4261 amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, importers_bitmap); 4262 } 4263 4264 /* Release the export dma-buf and drop the BO ref. */ 4265 dma_buf_put(exp_xa_node->dmabuf); 4266 amdgpu_bo_unref(&bo); 4267 exp_xa_node->bo = NULL; 4268 4269 mutex_destroy(&exp_xa_node->node_lock); 4270 kfree(exp_xa_node); 4271 } 4272 4273 /* Free every exporter entry left in exp_xa at manager stop. These were 4274 * exported but never revoked, so no cleanup worker was queued for them. 4275 */ 4276 static void amdgpu_ualink_teardown_exp_xa_entries(struct amdgpu_device *adev) 4277 { 4278 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 4279 unsigned long index; 4280 4281 xa_for_each(&adev->ualink.exp_xa, index, exp_xa_node) { 4282 xa_erase(&adev->ualink.exp_xa, index); 4283 amdgpu_ualink_teardown_exp_xa_node(adev, exp_xa_node); 4284 } 4285 } 4286 4287 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev) 4288 { 4289 int i; 4290 4291 adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 0); 4292 4293 /* Mark connections down so the drained workers and the teardown below 4294 * take the message-free path in amdgpu_ualink_check_conn_ready(). 4295 */ 4296 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4297 mutex_lock(&adev->ualink.conn_state[i].lock); 4298 adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY; 4299 mutex_unlock(&adev->ualink.conn_state[i].lock); 4300 } 4301 4302 /* Drain in-flight exporter cleanup work so revoked nodes in 4303 * handle_invalid_xa free themselves and empty that xarray. 4304 */ 4305 drain_workqueue(adev->ualink.npa_wq); 4306 4307 /* Free live entries before the NPA allocator is torn down, so NPA 4308 * addresses can still be unmapped and freed. 4309 */ 4310 amdgpu_ualink_teardown_imp_xa_entries(adev); 4311 amdgpu_ualink_teardown_exp_xa_entries(adev); 4312 4313 amdgpu_ualink_npa_mm_fini(adev); 4314 4315 /* All three xarrays are empty by now. */ 4316 xa_destroy(&adev->ualink.exp_xa); 4317 xa_destroy(&adev->ualink.imp_xa); 4318 xa_destroy(&adev->ualink.handle_invalid_xa); 4319 4320 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) 4321 mutex_destroy(&adev->ualink.conn_state[i].lock); 4322 4323 destroy_workqueue(adev->ualink.npa_wq); 4324 amdgpu_vm_fini(adev, &adev->ualink.npa_vm); 4325 } 4326 4327 /* 4328 * UALink remote interrupt and shootdown 4329 */ 4330 4331 /* UALINK ring buffer size, same for both remote shootdown and interrupt ring */ 4332 #define AMDGPU_UALINK_RB_SIZE 4096 4333 4334 #define AMDGPU_UALINK_METADATA_HEADER 0x4E485446 4335 4336 /* 2MB NPA start address for 2MB page mapping */ 4337 #define AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET SZ_2M 4338 4339 struct amdgpu_ualink_metadata { 4340 u32 header; 4341 4342 /* 4343 * "ring entries" as unit. So the ring-size-in-bytes could be calculated 4344 * as entry-size * 2^RBsize. This would make the minimum size a single 4345 * entry and the maximum size 32786 entries. 4346 * 4347 * u32 rb_size:4; 4348 * u32 reserved0:4; 4349 * u32 vmid:4; 4350 * u32 reserved1:20; 4351 */ 4352 u32 rb_size; 4353 4354 /* ring buffer base address for remote interrupt */ 4355 u64 ri_rb; 4356 4357 /* tlb invalidate ring buffer base address for remote shootdown */ 4358 u64 tlb_inv_rb; 4359 4360 /* remote interrupt Tail pointer, write pointer address */ 4361 u64 tailptr_ri; 4362 4363 /* TLB invalidate ring buffer's Tail pointer, write pointer address */ 4364 u64 tailptr_tlb_inv; 4365 }; 4366 4367 struct amdgpu_ualink_wb { 4368 /* last finished command seq number */ 4369 u32 data; 4370 4371 /* command complete error code */ 4372 u32 status; 4373 4374 /* ring rptr updated by FW */ 4375 u64 rptr; 4376 }; 4377 4378 /* 4379 * For remote interrupt and shootdown 4380 */ 4381 struct amdgpu_ualink_ring { 4382 u32 rb_size; 4383 4384 /* writeback data */ 4385 u32 seq; 4386 4387 /* local copy */ 4388 u64 wptr, rptr; 4389 4390 /* NPA gart mapping for SDMA */ 4391 u64 rb_npa_gart; 4392 u64 wptr_npa_gart; 4393 u64 doorbell_npa_gart; 4394 4395 /* gart mapping node */ 4396 struct drm_mm_node mm_node_rb; 4397 struct drm_mm_node mm_node_wptr; 4398 struct drm_mm_node mm_node_doorbell; 4399 4400 /* seq, complete status write back cpu address */ 4401 struct amdgpu_ualink_wb *wb_cpu; 4402 4403 /* true if fw write back address updated successfully */ 4404 bool ready; 4405 }; 4406 4407 struct amdgpu_ualink_peer { 4408 /* remote interrupt and shootdown uses same SDMA entity */ 4409 struct mutex lock; 4410 4411 /* SDMA engine to send remote command via NPA */ 4412 struct drm_sched_entity entity; 4413 4414 /* to select different DXS ports, cycles through different values */ 4415 u32 dxs_port; 4416 4417 struct amdgpu_ualink_ring interrupt; 4418 struct amdgpu_ualink_ring shootdown; 4419 }; 4420 4421 struct amdgpu_ualink_remote { 4422 /* ualink metadata passed to MPNHT FW */ 4423 struct amdgpu_bo *metadata_bo; 4424 u64 metadata_gpu_addr; 4425 void *metadata_cpu_addr; 4426 4427 /* ualink ring, tlb ring buffer, wptr */ 4428 struct amdgpu_bo *ring_bo; 4429 u64 rb_gpu_addr; 4430 void *rb_cpu_addr; 4431 4432 /* ualink rptr, wb data, statuss for address alias mode */ 4433 struct amdgpu_bo *rptr_bo; 4434 u64 rptr_gpu_addr; 4435 void *rptr_cpu_addr; 4436 u64 rptr_npa; 4437 4438 /* address alias mode alloc npa address for shared wb */ 4439 struct drm_mm_node rptr_mm_node; 4440 4441 /* 4442 * Owned snapshot of the vPod's active accelerator bitmap, taken at 4443 * sw_init time. Must NOT alias info->vpod.active_accel_bits, which 4444 * psp_ual_query_info() refreshes on every commit: aliasing it would let 4445 * teardown unmap a different set than setup mapped, leaking drm_mm nodes 4446 * in the shared GTT manager. 4447 */ 4448 DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); 4449 u32 num_accel; 4450 4451 /* remote GPUs ring buffer, read, write pointer local copy and gart mapping */ 4452 struct amdgpu_ualink_peer peer[AMDGPU_UALINK_ACCEL_MAX]; 4453 4454 /* use lsdma write to NPA address for remote interrupt */ 4455 bool use_lsdma; 4456 }; 4457 4458 static inline struct amdgpu_ualink_remote *to_remote(struct amdgpu_device *adev) 4459 { 4460 return adev->ualink.remote; 4461 } 4462 4463 /* 4464 * Returns true if the vPod membership most recently reported by firmware 4465 * (info->vpod.active_accel_bits, read back via psp_ual_query_info() on the 4466 * committing GPU) differs from the owned snapshot captured when this 4467 * accelerator was last activated (remote->active_accel_bits). Used to 4468 * decide whether an accelerator must be (re)built to match the new member 4469 * set. A NULL remote means the accelerator has never been activated (no 4470 * snapshot yet), which also counts as "changed" so it gets brought up. 4471 */ 4472 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev) 4473 { 4474 struct amdgpu_ualink_remote *remote = to_remote(adev); 4475 4476 if (!remote) 4477 return true; 4478 4479 return !bitmap_equal(remote->active_accel_bits, 4480 adev->ualink.info->vpod.active_accel_bits, 4481 AMDGPU_UALINK_ACCEL_MAX); 4482 } 4483 4484 static inline u32 ualink_accel_id(struct amdgpu_device *adev) 4485 { 4486 return adev->ualink.info->ppod.accel_id; 4487 } 4488 4489 static inline enum amdgpu_ualink_addr_mode ualink_addr_mode(struct amdgpu_device *adev) 4490 { 4491 return adev->ualink.info->vpod.addr_mode; 4492 } 4493 4494 /* 4495 * address mode and NPA address for ring wptr, rptr, wb status, wb data 4496 * 4497 * source identification address mode 4498 * 4499 * we need to export different tail pointers for different remote GPU accel_id, 4500 * hence we need to allocate a whole page for each remote GPU separately. 4501 * 4502 * rptr, wb data, status share the whole page with wptr. 4503 * 4504 * 1 page reserved NPA address for 1 active remote GPU, max 255 pages 4505 * starting address: wptr NPA address + accel_id * 4K 4506 * offset: 4507 * remote interrupt ring wptr 0 4508 * remote shootdown ring wptr 8 4509 * remote interrupt wb status 16 4510 * remote interrupt wb data 20 4511 * remote interrupt ring rptr 24 4512 * remote shootdown wb status 32 4513 * remote shootdown wb data 36 4514 * remote shootdown ring rptr 40 4515 * 4516 * source aliasing address mode 4517 * 4518 * wptr for different remote GPU can share same reserved 1 page NPA address page. 4519 * 4520 * to save reserved NPA address space, rptr, wb data, status for different 4521 * remote GPU share same allocated 2 pages NPA address. 4522 * 4523 * 1 page reserved NPA address for wptr of all active remote GPUs 4524 * offset: 4525 * remote interrupt ring wptr accel_id * 16 4526 * remote shootdown ring wptr accel_id * 16 + 8 4527 * 4528 * 2 page allocated NPA address for rptr, writeback of all active remote GPUs 4529 * starting address: at accel_id * 32 for each remote GPU 4530 * offset: 4531 * remote interrupt wb status 0 4532 * remote interrupt wb data 4 4533 * remote interrupt ring rptr 8 4534 * remote shootdown wb status 16 4535 * remote shootdown wb data 20 4536 * remote shootdown ring rptr 24 4537 */ 4538 static inline u32 ualink_wptr_size(struct amdgpu_device *adev) 4539 { 4540 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4541 return AMDGPU_GPU_PAGE_SIZE; 4542 else 4543 return 0; /* wptr uses the ring buffer for itself */ 4544 } 4545 4546 static inline u32 ualink_wb_size(struct amdgpu_device *adev) 4547 { 4548 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4549 return 0; /* share same page with wptr */ 4550 else 4551 return 2 * sizeof(struct amdgpu_ualink_wb); 4552 } 4553 4554 static inline u32 ualink_wptr_offset(struct amdgpu_device *adev, u32 accel_id) 4555 { 4556 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4557 return 0; /* one separate page per GPU */ 4558 else 4559 return accel_id * 2 * sizeof(u64); /* two pointers per GPU */ 4560 } 4561 4562 static inline u32 ualink_tlb_wptr_offset(struct amdgpu_device *adev, u32 accel_id) 4563 { 4564 return ualink_wptr_offset(adev, accel_id) + sizeof(u64); 4565 } 4566 4567 static inline u32 ualink_wb_offset(struct amdgpu_device *adev, u32 accel_id) 4568 { 4569 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4570 return 2 * sizeof(u64); /* after two wptr pointers */ 4571 else 4572 return accel_id * ualink_wb_size(adev); 4573 } 4574 4575 static inline u32 ualink_tlb_wb_offset(struct amdgpu_device *adev, u32 accel_id) 4576 { 4577 return ualink_wb_offset(adev, accel_id) + sizeof(struct amdgpu_ualink_wb); 4578 } 4579 4580 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, u32 flush_type) 4581 { 4582 u32 bit; 4583 4584 bit = AMDGPU_MMHUB0_START; 4585 4586 for_each_set_bit_from(bit, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) 4587 amdgpu_gmc_flush_gpu_tlb(adev, adev->vm_manager.npa_vmid, 4588 bit, flush_type); 4589 } 4590 4591 /** 4592 * amdgpu_ualink_npa_vm_map_range - Map a range in the NPA VM 4593 * @adev: amdgpu device pointer 4594 * @bo: buffer object backing the mapping 4595 * @pte_flags: page table entry flags 4596 * @offset: offset into the buffer object in bytes 4597 * @size_in_pages: size of the range to map in pages 4598 * @npa_in_pages: NPA target address in pages 4599 * 4600 * Maps a buffer object range into the NPA VM page table at the specified 4601 * NPA address. 4602 * 4603 * Return: 0 on success, negative error code on failure 4604 */ 4605 static int amdgpu_ualink_npa_vm_map_range(struct amdgpu_device *adev, struct amdgpu_bo *bo, 4606 u64 pte_flags, u64 offset, u64 size_in_pages, 4607 u64 npa_in_pages) 4608 { 4609 struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm; 4610 int r; 4611 4612 dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n", 4613 offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags, 4614 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4615 4616 r = amdgpu_vm_update_range(adev, npa_vm, false, false, true, 4617 false, NULL, npa_in_pages, 4618 npa_in_pages + size_in_pages - 1, 4619 pte_flags, offset, adev->vm_manager.vram_base_offset, 4620 bo->tbo.resource, NULL, &npa_vm->last_update); 4621 if (r) 4622 dev_dbg(adev->dev, "failed %d to map npa 0x%llx to NPA VM\n", r, 4623 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4624 return r; 4625 } 4626 4627 /** 4628 * amdgpu_npa_vm_unmap_range - Unmap a range from the NPA VM 4629 * @adev: amdgpu device pointer 4630 * @bo: buffer object backing the mapping 4631 * @pte_flags: page table entry flags 4632 * @offset: offset into the buffer object in bytes 4633 * @size_in_pages: size of the range to unmap in pages 4634 * @npa_in_pages: NPA target address in pages 4635 * 4636 * Removes a previously established mapping from the NPA VM page table. 4637 * 4638 * Return: 0 on success, negative error code on failure 4639 */ 4640 static int amdgpu_ualink_npa_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_bo *bo, 4641 u64 pte_flags, u64 offset, u64 size_in_pages, 4642 u64 npa_in_pages) 4643 { 4644 struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm; 4645 int r; 4646 4647 dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n", 4648 offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags, 4649 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4650 4651 r = amdgpu_vm_update_range(adev, npa_vm, false, false, true, 4652 false, NULL, npa_in_pages, 4653 npa_in_pages + size_in_pages - 1, 4654 pte_flags, offset, 0, bo->tbo.resource, NULL, 4655 &npa_vm->last_update); 4656 if (r) 4657 dev_dbg(adev->dev, "failed %d to unmap npa 0x%llx from NPA VM\n", r, 4658 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4659 return r; 4660 } 4661 4662 /* 4663 * Reserved NPA space for remote shootdown and interrupt ring buffer, 4664 * write pointers, read pointers and writeback buffers 4665 * 4666 * NPA ring type: 0 = shootdown, 1 = interrupt, 2 = tailptr (wptr) 4667 */ 4668 enum ring_buffer_type { 4669 RB_TYPE_TLB_INV = 0, 4670 RB_TYPE_REMOTE_INTERRUPT = 1, 4671 RB_TYPE_TAILPTR = 2 4672 }; 4673 4674 /** 4675 * amdgpu_ualink_npa_addr - Get reserved NPA address for ring buffer 4676 * @adev: amdgpu device pointer 4677 * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR) 4678 * @src_accel_id: Source accelerator ID 4679 * @dst_accel_id: Destination accelerator ID 4680 * 4681 * Return: NPA address for the specified ring buffer type and GPUs 4682 */ 4683 static u64 amdgpu_ualink_npa_addr(struct amdgpu_device *adev, u32 type, 4684 u32 src_accel_id, u32 dst_accel_id) 4685 { 4686 u32 addr_mode = ualink_addr_mode(adev); 4687 u64 npa; 4688 4689 WARN_ON_ONCE(src_accel_id >= AMDGPU_UALINK_ACCEL_MAX || 4690 dst_accel_id >= AMDGPU_UALINK_ACCEL_MAX || 4691 type > RB_TYPE_TAILPTR || 4692 (addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT && 4693 addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)); 4694 4695 switch (addr_mode) { 4696 case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT: 4697 npa = (u64)src_accel_id << 41 | type << 12; 4698 break; 4699 4700 case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS: 4701 default: 4702 if (type == RB_TYPE_TAILPTR) 4703 npa = (u64)dst_accel_id << 13; 4704 else 4705 npa = (u64)src_accel_id << 13 | type << 12; 4706 4707 /* 4708 * In order to avoid address conflicts between source-identification 4709 * and source-aliasing mode, add 2MB to the buffer addresses in 4710 * source-aliasing mode. This way in a misconfigured cluster, a GPU 4711 * configured in the wrong address mode will access unmapped NPA addresses 4712 * rather than the wrong buffer mappings. 4713 */ 4714 npa += AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET; 4715 break; 4716 } 4717 4718 dev_dbg(adev->dev, "addr mode %d from accel %d to accel %d type %d NPA 0x%llx\n", 4719 addr_mode, src_accel_id, dst_accel_id, type, npa); 4720 4721 return npa; 4722 } 4723 4724 /** 4725 * amdgpu_ualink_gart_npa_addr - Get GART-mapped NPA address for ring buffer 4726 * @adev: amdgpu device pointer 4727 * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR) 4728 * @src_accel_id: Source accelerator ID 4729 * @dst_accel_id: Destination accelerator ID 4730 * 4731 * Computes the NPA address suitable for GART mapping by adding dst_accid_id to 4732 * NPA address to access remote by SDMA. 4733 4734 * clears the source GPU ID bits from the NPA address which is set for 4735 * address-identification mode. 4736 * 4737 * Return: NPA address adjusted for GART mapping 4738 */ 4739 4740 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT 41 4741 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK GENMASK_ULL(50, 41) 4742 4743 static u64 amdgpu_ualink_gart_npa_addr(struct amdgpu_device *adev, u32 type, 4744 u32 src_accel_id, u32 dst_accel_id) 4745 { 4746 u64 npa; 4747 4748 npa = amdgpu_ualink_npa_addr(adev, type, src_accel_id, dst_accel_id); 4749 4750 /* wiping out source accelerator id */ 4751 npa &= ~AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK; 4752 4753 return npa | ((u64)dst_accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT); 4754 } 4755 4756 /** 4757 * amdgpu_ualink_reserve_npa_vm_and_bos - Reserve the NPA VM page directory and BOs 4758 * @adev: amdgpu device pointer 4759 * @bos: array of buffer objects to reserve 4760 * @n_bos: number of entries in @bos 4761 * @exec: drm_exec context to initialize and use for locking 4762 * @interruptible: true to use the interruptible dma_resv_lock 4763 * 4764 * Initializes @exec and uses it to lock all BOs in @bos together with the 4765 * NPA VM page directory, retrying on contention. On failure, @exec is 4766 * finalized and the error code is returned. 4767 * 4768 * Returns: 0 on success, negative error code on failure. 4769 */ 4770 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev, 4771 struct amdgpu_bo *bos[], u32 n_bos, 4772 struct drm_exec *exec, 4773 bool interruptible) 4774 { 4775 u32 flags = DRM_EXEC_IGNORE_DUPLICATES; 4776 int i, r = 0; 4777 4778 if (interruptible) 4779 flags |= DRM_EXEC_INTERRUPTIBLE_WAIT; 4780 4781 dev_dbg(adev->dev, "reserve NPA vm and %d bos\n", n_bos); 4782 4783 drm_exec_init(exec, flags, 0); 4784 4785 drm_exec_until_all_locked(exec) { 4786 for (i = 0; i < n_bos; i++) { 4787 r = drm_exec_lock_obj(exec, &bos[i]->tbo.base); 4788 drm_exec_retry_on_contention(exec); 4789 if (unlikely(r)) 4790 goto out; 4791 } 4792 4793 r = amdgpu_vm_lock_pd(&adev->ualink.npa_vm, exec, 0); 4794 drm_exec_retry_on_contention(exec); 4795 if (unlikely(r)) 4796 goto out; 4797 } 4798 4799 out: 4800 if (r) 4801 drm_exec_fini(exec); 4802 return r; 4803 4804 } 4805 4806 /** 4807 * amdgpu_ualink_unreserve_npa_vm_and_bos - Release the NPA VM page directory and BOs 4808 * @adev: amdgpu device pointer 4809 * @exec: drm_exec context previously initialized by 4810 * amdgpu_ualink_reserve_npa_vm_and_bos() 4811 * 4812 * Finalizes @exec, releasing all locks on the NPA VM page directory and 4813 * the associated BOs acquired during reservation. 4814 */ 4815 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev, 4816 struct drm_exec *exec) 4817 { 4818 dev_dbg(adev->dev, "unreserve NPA vm and bos\n"); 4819 drm_exec_fini(exec); 4820 } 4821 4822 /** 4823 * amdgpu_ualink_metadata_npa_unmapping - Tear down NPA address mappings 4824 * @adev: amdgpu device pointer 4825 * 4826 * Unmaps all NPA address mappings from the NPA VM for ring buffers, 4827 * write pointers, and read pointers. Waits for outstanding DMA fences 4828 * and flushes the TLB. 4829 */ 4830 static void amdgpu_ualink_metadata_npa_unmapping(struct amdgpu_device *adev) 4831 { 4832 struct amdgpu_ualink_remote *remote = to_remote(adev); 4833 u64 timeout = msecs_to_jiffies(2000); 4834 u32 dst_accel_id = ualink_accel_id(adev); 4835 struct amdgpu_bo *bos[2]; 4836 u32 n_bos; 4837 struct dma_fence *fence; 4838 struct drm_exec exec; 4839 u32 rptr_size, rptr_size_in_pages; 4840 u32 rb_size, rb_size_in_pages; 4841 u64 pte_flags = adev->gmc.noretry_flags; 4842 u64 npa; 4843 int r; 4844 4845 if (!remote->ring_bo) 4846 return; 4847 4848 rb_size = AMDGPU_UALINK_RB_SIZE; 4849 rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT; 4850 4851 bos[0] = remote->ring_bo; 4852 n_bos = 1; 4853 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4854 bos[1] = remote->rptr_bo; 4855 n_bos = 2; 4856 } 4857 4858 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false); 4859 if (unlikely(r)) 4860 return; 4861 4862 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4863 /* rptr npa mapping, up to allocated 2 pages npa address */ 4864 rptr_size = ualink_wb_size(adev); 4865 rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel); 4866 rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT; 4867 4868 npa = remote->rptr_npa; 4869 4870 amdgpu_ualink_npa_vm_unmap_range(adev, remote->rptr_bo, 4871 pte_flags, 0, rptr_size_in_pages, 4872 npa >> AMDGPU_GPU_PAGE_SHIFT); 4873 amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node); 4874 4875 /* 4876 * unmap wptr, remote interrupt, shootdown ring. 4877 * wptr page is part of the single 2MB mapping for remote GPUs 4878 * interrupt and shootdown ring buffer 4879 */ 4880 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, 4881 0, dst_accel_id); 4882 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, pte_flags, 4883 0, 4884 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX, 4885 npa >> AMDGPU_GPU_PAGE_SHIFT); 4886 } else { 4887 u32 wptr_offset = 2 * rb_size * remote->num_accel; 4888 u32 idx = 0; 4889 u32 accel_id; 4890 4891 /* source identification mode */ 4892 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 4893 if (accel_id == dst_accel_id) 4894 continue; 4895 4896 /* remote interrupt ring npa mapping */ 4897 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 4898 accel_id, dst_accel_id); 4899 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4900 pte_flags, idx * 2 * rb_size, 4901 rb_size_in_pages, 4902 npa >> AMDGPU_GPU_PAGE_SHIFT); 4903 4904 /* remote shootdown ring npa mapping */ 4905 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id, 4906 dst_accel_id); 4907 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4908 pte_flags, (idx * 2 + 1) * rb_size, 4909 rb_size_in_pages, 4910 npa >> AMDGPU_GPU_PAGE_SHIFT); 4911 4912 /* wptr, rptr npa mapping */ 4913 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id, 4914 dst_accel_id); 4915 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4916 pte_flags, wptr_offset + idx * PAGE_SIZE, 4917 1, npa >> AMDGPU_GPU_PAGE_SHIFT); 4918 idx++; 4919 } 4920 } 4921 4922 r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false); 4923 if (r) { 4924 dev_dbg(adev->dev, "failed %d to update directories\n", r); 4925 goto out_unreserve; 4926 } 4927 4928 fence = dma_fence_get(adev->ualink.npa_vm.last_update); 4929 if (fence) { 4930 r = dma_fence_wait_timeout(fence, true, timeout); 4931 dma_fence_put(fence); 4932 if (r <= 0) 4933 dev_dbg(adev->dev, "failed %d to dma fence wait\n", r); 4934 } 4935 4936 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 4937 out_unreserve: 4938 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 4939 } 4940 4941 /** 4942 * amdgpu_ualink_metadata_npa_mapping - Setup NPA address mapping in VM 4943 * @adev: amdgpu device pointer 4944 * 4945 * Maps NPA addresses to GPA for metadata ring buffers, write pointers, 4946 * on NPA VMID 15. 4947 * 4948 * Return: 0 on success, negative error code on failure 4949 */ 4950 static int amdgpu_ualink_metadata_npa_mapping(struct amdgpu_device *adev) 4951 { 4952 struct amdgpu_ualink_remote *remote = to_remote(adev); 4953 u32 dst_accel_id = ualink_accel_id(adev); 4954 u64 timeout = msecs_to_jiffies(2000); 4955 struct amdgpu_bo *bos[2]; 4956 u32 n_bos; 4957 struct dma_fence *fence; 4958 struct drm_exec exec; 4959 u32 rb_size, rb_size_in_pages; 4960 u64 npa, npa_in_pages, pte_flags; 4961 int r; 4962 4963 rb_size = AMDGPU_UALINK_RB_SIZE; 4964 rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT; 4965 4966 bos[0] = remote->ring_bo; 4967 n_bos = 1; 4968 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4969 bos[1] = remote->rptr_bo; 4970 n_bos = 2; 4971 } 4972 4973 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false); 4974 if (unlikely(r)) 4975 return r; 4976 4977 pte_flags = amdgpu_ttm_tt_pte_flags(adev, remote->ring_bo->tbo.ttm, 4978 remote->ring_bo->tbo.resource); 4979 dev_dbg(adev->dev, "init pte_flags 0x%llx\n", pte_flags); 4980 4981 amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, remote->ring_bo, 4982 AMDGPU_VM_MTYPE_DEFAULT, &pte_flags); 4983 dev_dbg(adev->dev, "after get coherent pte_flags 0x%llx\n", pte_flags); 4984 4985 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4986 u32 rptr_size, rptr_size_in_pages; 4987 4988 /* rptr npa mapping, up to allocated 2 pages npa address */ 4989 rptr_size = ualink_wb_size(adev); 4990 rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel); 4991 rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT; 4992 4993 r = amdgpu_ualink_npa_alloc_va(adev, &remote->rptr_mm_node, 4994 0, 0, 0, 4995 rptr_size_in_pages); 4996 if (r) 4997 goto out; 4998 4999 npa_in_pages = remote->rptr_mm_node.start; 5000 5001 dev_dbg(adev->dev, "source aliasing rptr alloc 0x%llx and map to npa vm\n", 5002 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 5003 5004 r = amdgpu_ualink_npa_vm_map_range(adev, remote->rptr_bo, pte_flags, 0, 5005 rptr_size_in_pages, 5006 npa_in_pages); 5007 if (r) 5008 goto error_npa_mapping; 5009 5010 remote->rptr_npa = npa_in_pages << AMDGPU_GPU_PAGE_SHIFT; 5011 5012 /* wptr, remote interrupt, shootdown ring, single big 2MB mapping */ 5013 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, 5014 0, dst_accel_id); 5015 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5016 0, 5017 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX, 5018 npa >> AMDGPU_GPU_PAGE_SHIFT); 5019 if (r) 5020 goto error_npa_mapping; 5021 5022 } else { 5023 u32 wptr_offset = 2 * rb_size * remote->num_accel; 5024 u32 accel_id, idx = 0; 5025 5026 /* Source identification mode */ 5027 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5028 if (accel_id == dst_accel_id) 5029 continue; 5030 5031 /* remote interrupt ring npa mapping */ 5032 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 5033 accel_id, dst_accel_id); 5034 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5035 idx * 2 * rb_size, 5036 rb_size_in_pages, 5037 npa >> AMDGPU_GPU_PAGE_SHIFT); 5038 if (r) 5039 goto error_npa_mapping; 5040 5041 /* remote shootdown ring npa mapping */ 5042 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id, 5043 dst_accel_id); 5044 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5045 (idx * 2 + 1) * rb_size, 5046 rb_size_in_pages, 5047 npa >> AMDGPU_GPU_PAGE_SHIFT); 5048 if (r) 5049 goto error_npa_mapping; 5050 5051 /* wptr, rptr npa mapping */ 5052 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id, 5053 dst_accel_id); 5054 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, 5055 pte_flags, 5056 wptr_offset + idx * PAGE_SIZE, 5057 1, npa >> AMDGPU_GPU_PAGE_SHIFT); 5058 if (r) 5059 goto error_npa_mapping; 5060 5061 idx++; 5062 } 5063 } 5064 5065 r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false); 5066 if (r) { 5067 dev_dbg(adev->dev, "failed %d to update directories\n", r); 5068 goto error_npa_mapping; 5069 } 5070 5071 /* TODO: only wait the last fence, then flush TLB */ 5072 fence = dma_fence_get(adev->ualink.npa_vm.last_update); 5073 if (fence) { 5074 r = dma_fence_wait_timeout(fence, true, timeout); 5075 dma_fence_put(fence); 5076 if (r <= 0) 5077 dev_dbg(adev->dev, "failed %d to dma fence wait\n", r); 5078 } 5079 5080 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 5081 5082 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 5083 return 0; 5084 5085 error_npa_mapping: 5086 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 5087 amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node); 5088 if (r) 5089 dev_dbg(adev->dev, "failed %d to map NPA vm\n", r); 5090 5091 out: 5092 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 5093 5094 return r; 5095 } 5096 5097 /** 5098 * amdgpu_ualink_sdma_entities_init - Initialize SDMA job scheduler entities 5099 * @adev: amdgpu device pointer 5100 * 5101 * Sets up SDMA job scheduler entities for remote interrupt and shootdown. 5102 * Each entity uses one SDMA scheduler/ring. 5103 * 5104 * Return: 0 on success, negative error code on failure 5105 */ 5106 static int amdgpu_ualink_sdma_entities_init(struct amdgpu_device *adev) 5107 { 5108 struct amdgpu_ualink_remote *remote = to_remote(adev); 5109 struct drm_gpu_scheduler *sched; 5110 u32 accel_id; 5111 int i = 0, r; 5112 5113 dev_dbg(adev->dev, "enter\n"); 5114 5115 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5116 sched = &adev->sdma.instance[accel_id % adev->sdma.num_instances].ring.sched; 5117 5118 r = drm_sched_entity_init(&remote->peer[accel_id].entity, DRM_SCHED_PRIORITY_HIGH, 5119 &sched, 1, NULL); 5120 if (r) 5121 goto out_free; 5122 i++; 5123 } 5124 5125 remote->use_lsdma = true; 5126 5127 dev_dbg(adev->dev, "exit\n"); 5128 return 0; 5129 5130 out_free: 5131 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5132 if (i--) 5133 drm_sched_entity_destroy(&remote->peer[accel_id].entity); 5134 else 5135 break; 5136 } 5137 5138 return r; 5139 } 5140 5141 static void amdgpu_ualink_sdma_entities_fini(struct amdgpu_device *adev) 5142 { 5143 struct amdgpu_ualink_remote *remote = to_remote(adev); 5144 u32 accel_id; 5145 5146 dev_dbg(adev->dev, "enter\n"); 5147 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) 5148 drm_sched_entity_destroy(&remote->peer[accel_id].entity); 5149 dev_dbg(adev->dev, "exit\n"); 5150 } 5151 5152 /** 5153 * amdgpu_ualink_gart_map - Allocate GART entry and map NPA address 5154 * @adev: amdgpu device pointer 5155 * @npages: Number of pages to map 5156 * @npa: NPA address to map 5157 * @mm_node: DRM memory manager node for GART allocation 5158 * @pte_flags: the GART mapping flags 5159 * 5160 * Allocates GART entries and sets up NPA address mapping without TTM BO. 5161 * 5162 * Return: 0 on success, negative error code on failure 5163 */ 5164 static int amdgpu_ualink_gart_map(struct amdgpu_device *adev, u64 npages, 5165 u64 npa, struct drm_mm_node *mm_node, 5166 u64 pte_flags) 5167 { 5168 struct ttm_resource_manager *man = 5169 ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 5170 struct amdgpu_gtt_mgr *mgr = 5171 container_of(man, struct amdgpu_gtt_mgr, manager); 5172 dma_addr_t *dma_addr; 5173 int i, r; 5174 5175 dev_dbg(adev->dev, "npa 0x%llx npages 0x%llx\n", npa, npages); 5176 5177 if (npages > 1) { 5178 dma_addr = kmalloc_array(npages, sizeof(*dma_addr), GFP_KERNEL); 5179 if (!dma_addr) 5180 return -ENOMEM; 5181 5182 for (i = 0; i < npages; i++) 5183 dma_addr[i] = npa + i * AMDGPU_GPU_PAGE_SIZE; 5184 } else { 5185 dma_addr = (dma_addr_t *)&npa; 5186 } 5187 5188 r = amdgpu_gtt_mgr_alloc_entries(mgr, mm_node, npages, DRM_MM_INSERT_BEST); 5189 if (r) 5190 goto out; 5191 amdgpu_gart_bind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, dma_addr, 5192 pte_flags); 5193 5194 out: 5195 dev_dbg(adev->dev, "npa 0x%llx mapped to 0x%llx npages 0x%llx r=%d\n", 5196 npa, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, r); 5197 5198 if (npages > 1) 5199 kfree(dma_addr); 5200 return r; 5201 } 5202 5203 /** 5204 * amdgpu_ualink_gart_unmap - Unmap and free GART entry 5205 * @adev: amdgpu device pointer 5206 * @npages: Number of pages to unmap 5207 * @mm_node: DRM memory manager node for GART allocation 5208 * 5209 * Unbinds GART mapping and frees allocated entries. 5210 */ 5211 static void amdgpu_ualink_gart_unmap(struct amdgpu_device *adev, u64 npages, 5212 struct drm_mm_node *mm_node) 5213 { 5214 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 5215 struct amdgpu_gtt_mgr *mgr = 5216 container_of(man, struct amdgpu_gtt_mgr, manager); 5217 5218 dev_dbg(adev->dev, "0x%llx npages 0x%llx\n", mm_node->start << AMDGPU_GPU_PAGE_SHIFT, 5219 npages); 5220 5221 if (!drm_mm_node_allocated(mm_node)) 5222 return; 5223 amdgpu_gart_unbind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages); 5224 amdgpu_gtt_mgr_free_entries(mgr, mm_node); 5225 } 5226 5227 /** 5228 * amdgpu_ualink_metadata_fini - Clean up ualink metadata structures 5229 * @adev: amdgpu device pointer 5230 * 5231 * Unmaps NPA addresses and frees all allocated buffers for metadata, 5232 * ring buffers, and pointers. 5233 */ 5234 static void amdgpu_ualink_metadata_fini(struct amdgpu_device *adev) 5235 { 5236 struct amdgpu_ualink_remote *remote = to_remote(adev); 5237 5238 dev_dbg(adev->dev, "accel_id %u\n", ualink_accel_id(adev)); 5239 amdgpu_bo_free_kernel(&remote->rptr_bo, &remote->rptr_gpu_addr, 5240 &remote->rptr_cpu_addr); 5241 amdgpu_bo_free_kernel(&remote->ring_bo, &remote->rb_gpu_addr, 5242 &remote->rb_cpu_addr); 5243 amdgpu_bo_free_kernel(&remote->metadata_bo, &remote->metadata_gpu_addr, 5244 &remote->metadata_cpu_addr); 5245 } 5246 5247 /** 5248 * amdgpu_ualink_metadata_init - Initialize ualink metadata structures 5249 * @adev: amdgpu device pointer 5250 * 5251 * Allocates and initializes metadata structures, ring buffers, and write/read 5252 * pointers for multi-GPU ualink. Communicates with firmware to load metadata. 5253 * 5254 * Return: 0 on success, negative error code on failure 5255 */ 5256 static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev) 5257 { 5258 struct amdgpu_ualink_remote *remote = to_remote(adev); 5259 struct amdgpu_ualink_metadata *metadata; 5260 u32 size, rb_size, wptr_size, rptr_size, metadata_size; 5261 u64 rb_gpu_addr, wptr_gpu_addr; 5262 u32 status, accel_id; 5263 int r; 5264 5265 bitmap_copy(remote->active_accel_bits, 5266 adev->ualink.info->vpod.active_accel_bits, 5267 AMDGPU_UALINK_ACCEL_MAX); 5268 dev_dbg(adev->dev, "%d active accelerators config in vpod\n", 5269 bitmap_weight(remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX)); 5270 5271 /* 5272 * allocate metadata entries and ring buffer for all remote GPUs, 5273 * to get 2MB page ring buffer NPA mapping for remote access. 5274 */ 5275 remote->num_accel = AMDGPU_UALINK_ACCEL_MAX; 5276 5277 status = adev->ualink.msg_ctl->check_status(adev); 5278 if (status != AMDGPU_NHT_FW_ST_PREINIT && 5279 status != AMDGPU_NHT_FW_ST_HALT) { 5280 dev_dbg(adev->dev, "fw status 0x%x not preinit or halt\n", status); 5281 return -ENODEV; 5282 } 5283 5284 dev_dbg(adev->dev, "accel_id %u addr_mode %d fw status 0x%x\n", 5285 ualink_accel_id(adev), ualink_addr_mode(adev), status); 5286 5287 /* Alloc metadata structure for all GPUs */ 5288 metadata_size = sizeof(struct amdgpu_ualink_metadata) * AMDGPU_UALINK_ACCEL_MAX; 5289 metadata_size = AMDGPU_GPU_PAGE_ALIGN(metadata_size); 5290 5291 dev_dbg(adev->dev, "metadata size 0x%x\n", metadata_size); 5292 5293 /* pinned system memory */ 5294 r = amdgpu_bo_create_kernel(adev, metadata_size, PAGE_SIZE, 5295 AMDGPU_GEM_DOMAIN_GTT, 5296 &remote->metadata_bo, &remote->metadata_gpu_addr, 5297 &remote->metadata_cpu_addr); 5298 if (r) 5299 goto out; 5300 5301 memset(remote->metadata_cpu_addr, 0, metadata_size); 5302 metadata = remote->metadata_cpu_addr; 5303 5304 dev_dbg(adev->dev, "metadata gpu address 0x%llx\n", remote->metadata_gpu_addr); 5305 5306 /* Allocate ring buffers, wptr for remote interrupt and shootdown */ 5307 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE); 5308 wptr_size = ualink_wptr_size(adev); 5309 size = (rb_size + wptr_size) * remote->num_accel; 5310 size = AMDGPU_GPU_PAGE_ALIGN(size); 5311 5312 dev_dbg(adev->dev, "rb_size 0x%x wptr_size 0x%x total alloc size 0x%x\n", 5313 rb_size, wptr_size, size); 5314 5315 /* pinned VRAM */ 5316 r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 5317 AMDGPU_GEM_DOMAIN_VRAM, 5318 &remote->ring_bo, 5319 &remote->rb_gpu_addr, 5320 &remote->rb_cpu_addr); 5321 if (r) 5322 goto out; 5323 5324 memset(remote->rb_cpu_addr, 0, size); 5325 5326 dev_dbg(adev->dev, "rb gpu addr 0x%llx cpu addr 0x%p vram_start 0x%llx vram_base 0x%llx\n", 5327 remote->rb_gpu_addr, remote->rb_cpu_addr, adev->gmc.vram_start, 5328 adev->vm_manager.vram_base_offset); 5329 5330 /* 5331 * address aliasing mode, shared wptr pagee left is not enough for wb data, 5332 * wb status and rptr, alloc another BO, and then alloc npa address and map 5333 * to npa vm, for remote to access. 5334 * 5335 * No gart mapping required for wb data, status and rptr because this is 5336 * updated by firmware. 5337 */ 5338 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 5339 rptr_size = AMDGPU_GPU_PAGE_ALIGN(ualink_wb_size(adev) * remote->num_accel); 5340 dev_dbg(adev->dev, "source aliasing mode rptr_size 0x%x\n", rptr_size); 5341 5342 /* pinned VRAM */ 5343 r = amdgpu_bo_create_kernel(adev, rptr_size, PAGE_SIZE, 5344 AMDGPU_GEM_DOMAIN_VRAM, 5345 &remote->rptr_bo, 5346 &remote->rptr_gpu_addr, 5347 &remote->rptr_cpu_addr); 5348 if (r) 5349 goto out; 5350 5351 memset(remote->rptr_cpu_addr, 0, rptr_size); 5352 5353 dev_dbg(adev->dev, "source aliasing rptr gpu addr 0x%llx cpu addr 0x%p\n", 5354 remote->rptr_gpu_addr, remote->rptr_cpu_addr); 5355 } 5356 5357 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 5358 wptr_gpu_addr = remote->rb_gpu_addr + rb_size * ualink_accel_id(adev); 5359 5360 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5361 dev_dbg(adev->dev, "init for accel_id %u\n", accel_id); 5362 5363 if (accel_id == ualink_accel_id(adev)) 5364 continue; 5365 5366 rb_gpu_addr = remote->rb_gpu_addr + rb_size * accel_id; 5367 5368 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 5369 wptr_gpu_addr = remote->rb_gpu_addr + rb_size * remote->num_accel; 5370 wptr_gpu_addr += wptr_size * accel_id; 5371 } 5372 5373 metadata[accel_id].header = AMDGPU_UALINK_METADATA_HEADER; 5374 metadata[accel_id].rb_size = fls(AMDGPU_UALINK_RB_SIZE / 64) - 1; 5375 5376 /* 5377 * remote shootdown type is 0, remote command type is 1 5378 * with 2MB ring buffer mapping, remote shootdown ring NPA is before interrupt 5379 */ 5380 metadata[accel_id].ri_rb = rb_gpu_addr + AMDGPU_UALINK_RB_SIZE; 5381 metadata[accel_id].tlb_inv_rb = rb_gpu_addr; 5382 5383 metadata[accel_id].tailptr_ri = wptr_gpu_addr + 5384 ualink_wptr_offset(adev, accel_id); 5385 metadata[accel_id].tailptr_tlb_inv = wptr_gpu_addr + 5386 ualink_tlb_wptr_offset(adev, accel_id); 5387 5388 dev_dbg(adev->dev, "init from accel_id %u to accel_id %u, rb_size 0x%x\n", 5389 accel_id, ualink_accel_id(adev), metadata[accel_id].rb_size); 5390 dev_dbg(adev->dev, "rb 0x%llx tlb rb 0x%llx\n", 5391 metadata[accel_id].ri_rb, metadata[accel_id].tlb_inv_rb); 5392 dev_dbg(adev->dev, "rb wptr at 0x%llx tlb wptr at 0x%llx\n", 5393 metadata[accel_id].tailptr_ri, metadata[accel_id].tailptr_tlb_inv); 5394 } 5395 5396 r = adev->ualink.msg_ctl->send_metadata(adev, 5397 remote->metadata_gpu_addr, 5398 AMDGPU_UALINK_ACCEL_MAX << 8); 5399 5400 out: 5401 if (r) 5402 amdgpu_ualink_metadata_fini(adev); 5403 5404 dev_dbg(adev->dev, "ret 0x%x\n", r); 5405 return r; 5406 } 5407 5408 /* 5409 * nHT Firmware Error code 5410 * 5411 * Success 0x0 5412 * FIFO overflow 0x1 5413 * Invalid RB command 0x2 5414 * Timeout 0x3 5415 * Invalid metadata entry 0x4 5416 * RB overflow 0x5 5417 * Invalid wptr address 0x6 5418 * Interrupt cookie send failure 0x7 5419 * Invalid RB address 0x8 5420 * Invalid writeback address 0x9 5421 */ 5422 static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status) 5423 { 5424 const char *fw_err_code_msg[] = { 5425 "Unknown FW error status", 5426 "FIFO overflow", /* error code 1 */ 5427 "Invalid RB command", 5428 "Timeout", 5429 "Invalid metadata entry", 5430 "RB overflow", 5431 "Invalid wptr address", 5432 "Interrupt cookie send failure", 5433 "Invalid RB address", 5434 "Invalid writeback address" /* error code 9 */ 5435 }; 5436 5437 if (status >= ARRAY_SIZE(fw_err_code_msg)) 5438 status = 0; 5439 5440 dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]); 5441 return; 5442 } 5443 5444 /** 5445 * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion 5446 * @remote_accel_id: remote accelator id to wait for reply 5447 * @adev: amdgpu device pointer 5448 * @wb_cpu: CPU virtual address of writeback buffer 5449 * @seq: Sequence number to wait for 5450 * 5451 * Polls the writeback buffer waiting for the sequence number to reach or 5452 * exceed the expected value. The writeback buffer contains status and data 5453 * fields updated by firmware to indicate completion and error conditions. 5454 * 5455 * Return: 0 on success 5456 * -ETIME on timeout 5457 * -ECOMM on firmware return error status 5458 */ 5459 static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev, 5460 u32 remote_accel_id, 5461 struct amdgpu_ualink_wb *wb_cpu, 5462 u32 seq) 5463 { 5464 /* 2 seconds timeout, long enough for FW to reply */ 5465 ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC); 5466 u32 status, data; 5467 u64 rptr; 5468 5469 while (true) { 5470 data = READ_ONCE(wb_cpu->data); 5471 if (data >= seq) { 5472 status = READ_ONCE(wb_cpu->status); 5473 rptr = READ_ONCE(wb_cpu->rptr); 5474 break; 5475 } 5476 5477 if (ktime_after(ktime_get(), timeout)) { 5478 dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id); 5479 return -ETIME; 5480 } 5481 5482 usleep_range(10, 50); 5483 } 5484 5485 if (status) { 5486 dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n", 5487 remote_accel_id, status, data, seq, rptr); 5488 5489 amdgpu_ualink_remote_error(adev, status); 5490 return -ECOMM; 5491 } 5492 5493 dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n", 5494 remote_accel_id, seq, data, rptr); 5495 return 0; 5496 } 5497 5498 #define AMDGPU_UALINK_REMOTE_OP_TLB_INV 0x1 5499 #define AMDGPU_UALINK_REMOTE_OP_INT 0x2 5500 #define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB 0x3 5501 5502 static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data, 5503 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5504 { 5505 u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT; 5506 u32 *cpu_addr = *cpu_addr_p; 5507 u32 flush_type = dw0; 5508 u32 size_in_pages = dw3; 5509 u32 cmd; 5510 5511 cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV; 5512 cmd |= 1 << 28; /* headptr update */ 5513 cmd |= 1 << 29; /* wb enable */ 5514 5515 *cpu_addr++ = cmd; 5516 *cpu_addr++ = wb_data; 5517 *cpu_addr++ = 0; 5518 *cpu_addr++ = 0; 5519 *cpu_addr++ = 0; 5520 *cpu_addr++ = 0; 5521 *cpu_addr++ = 0; /* metadata */ 5522 *cpu_addr++ = 0; /* metadata */ 5523 5524 pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages); 5525 5526 /* Always shootdown everything with full address size s-field coding */ 5527 addr = GENMASK_ULL(51, 11); 5528 5529 *cpu_addr++ = lower_32_bits(addr) | flush_type; 5530 *cpu_addr++ = upper_32_bits(addr); 5531 5532 *cpu_addr++ = 0; 5533 *cpu_addr++ = 0; 5534 *cpu_addr++ = 0; 5535 *cpu_addr++ = 0; 5536 *cpu_addr++ = 0; 5537 *cpu_addr++ = 0; 5538 *cpu_addr_p = cpu_addr; 5539 } 5540 5541 static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data, 5542 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5543 { 5544 u32 *cpu_addr = *cpu_addr_p; 5545 u32 cmd; 5546 5547 cmd = AMDGPU_UALINK_REMOTE_OP_INT; 5548 cmd |= 1 << 28; /* headptr_update */ 5549 cmd |= 1 << 29; /* wb enable */ 5550 5551 *cpu_addr++ = cmd; 5552 *cpu_addr++ = wb_data; 5553 *cpu_addr++ = 0; 5554 *cpu_addr++ = 0; 5555 *cpu_addr++ = 0; 5556 *cpu_addr++ = 0; 5557 *cpu_addr++ = 0; /* metadata */ 5558 *cpu_addr++ = 0; /* metadata */ 5559 /* updated by f/w, overwrite PASID with GPU ID */ 5560 *cpu_addr++ = 0; 5561 *cpu_addr++ = 0; 5562 *cpu_addr++ = 0; 5563 *cpu_addr++ = 0; 5564 /* IH ContextID 4 dwords */ 5565 *cpu_addr++ = dw0; 5566 *cpu_addr++ = dw1; 5567 *cpu_addr++ = dw2; 5568 *cpu_addr++ = dw3; 5569 *cpu_addr_p = cpu_addr; 5570 } 5571 5572 static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data, 5573 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5574 { 5575 u32 *cpu_addr = *cpu_addr_p; 5576 u32 cmd; 5577 5578 cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB; 5579 cmd |= 1 << 28; /* headptr update */ 5580 cmd |= 1 << 29; /* wb enable */ 5581 5582 *cpu_addr++ = cmd; 5583 *cpu_addr++ = wb_data; 5584 *cpu_addr++ = dw1 & 0xFFFFFFFC; /* lower32 wb_npa */ 5585 *cpu_addr++ = dw0 & 0xFFFFF; /* upper32 wb_npa */ 5586 *cpu_addr++ = 0; 5587 *cpu_addr++ = 0; 5588 *cpu_addr++ = 0; /* metadata */ 5589 *cpu_addr++ = 0; /* metadata */ 5590 *cpu_addr++ = 0; 5591 *cpu_addr++ = 0; 5592 *cpu_addr++ = 0; 5593 *cpu_addr++ = 0; 5594 *cpu_addr++ = 0; 5595 *cpu_addr++ = 0; 5596 *cpu_addr++ = 0; 5597 *cpu_addr++ = 0; 5598 *cpu_addr_p = cpu_addr; 5599 } 5600 5601 /* 5602 * Function proto to generate packet using above different emit_ functions 5603 */ 5604 typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0, 5605 u32 dw1, u32 dw2, u32 dw3); 5606 5607 /* 5608 * Refresh rptr from the writeback and check the ring has room for one more 5609 * command. The caller must hold the peer lock. 5610 * 5611 * Return 0 on success or a negative error code. 5612 */ 5613 static int amdgpu_ualink_ring_check_space(struct amdgpu_device *adev, 5614 u32 remote_accel_id, 5615 struct amdgpu_ualink_ring *ring, 5616 struct amdgpu_ualink_wb *wb_cpu) 5617 { 5618 ring->rptr = READ_ONCE(wb_cpu->rptr); 5619 5620 if (WARN_ON_ONCE(ring->rptr > ring->wptr)) { 5621 dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n", 5622 remote_accel_id, ring->wptr, ring->rptr); 5623 return -EFAULT; 5624 } 5625 5626 if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) { 5627 dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n", 5628 remote_accel_id, ring->wptr, ring->rptr); 5629 return -ENOSPC; 5630 } 5631 5632 return 0; 5633 } 5634 5635 static int amdgpu_ualink_send_command(struct amdgpu_device *adev, 5636 u32 remote_accel_id, 5637 struct amdgpu_ualink_ring *ring, 5638 struct amdgpu_ualink_wb *wb_cpu, 5639 ualink_emit_packet emit_func, 5640 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5641 { 5642 struct amdgpu_ualink_remote *remote = to_remote(adev); 5643 struct amdgpu_ualink_peer *peer; 5644 struct amdgpu_ring *sdma_ring; 5645 struct dma_fence *fence; 5646 struct amdgpu_job *job; 5647 struct amdgpu_ib *ib; 5648 u32 seq, ndw, ndw_copy_cmd, rem; 5649 u64 doorbell_npa_gart; 5650 u32 *src_cpu; 5651 u64 src; 5652 int r; 5653 5654 /* Refuse sends unless active */ 5655 if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 5656 return -ESHUTDOWN; 5657 5658 peer = &remote->peer[remote_accel_id]; 5659 5660 r = amdgpu_ualink_ring_check_space(adev, remote_accel_id, ring, wb_cpu); 5661 if (r) 5662 goto out; 5663 5664 /* 5665 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell 5666 * 5667 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to 5668 * ensure the ordering of 3 sdma copy commands execution. 5669 */ 5670 5671 /* buffer after sdma commands, starting at 8 dwords boundary */ 5672 ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8); 5673 5674 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */ 5675 ndw = ndw_copy_cmd + 16 + 2 + 1; 5676 r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM, 5677 ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, 5678 AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job); 5679 if (r) 5680 goto out; 5681 5682 ib = &job->ibs[0]; 5683 src = ib->gpu_addr + ndw_copy_cmd * 4; 5684 src_cpu = ib->ptr + ndw_copy_cmd; 5685 5686 seq = ++ring->seq; 5687 5688 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5689 dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n", 5690 src, ring->rb_npa_gart + rem * 64, 5691 ring->wptr_npa_gart, ring->doorbell_npa_gart, seq); 5692 5693 dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr); 5694 5695 /* remote command packet */ 5696 emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3); 5697 5698 /* wptr value */ 5699 *src_cpu++ = lower_32_bits(ring->wptr + 1); 5700 *src_cpu++ = upper_32_bits(ring->wptr + 1); 5701 5702 /* doorbell value */ 5703 *src_cpu++ = lower_32_bits(ring->wptr + 1); 5704 5705 /* 5706 * Add an offset to the doorbell address that cycles through different 5707 * values of bit 11:8 to use different DXS ports. 5708 */ 5709 peer->dxs_port = (peer->dxs_port + 1) & 0xF; 5710 doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8); 5711 dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n", 5712 peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart); 5713 5714 dev_dbg(adev->dev, "use %s\n", remote->use_lsdma ? "lsdma" : "sdma"); 5715 5716 if (remote->use_lsdma) { 5717 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5718 r = amdgpu_lsdma_copy_mem(adev, src, 5719 ring->rb_npa_gart + rem * 64, 5720 64); 5721 if (!r) 5722 r = amdgpu_lsdma_copy_mem(adev, src + 64, 5723 ring->wptr_npa_gart, 8); 5724 if (!r) 5725 r = amdgpu_lsdma_copy_mem(adev, src + 72, 5726 doorbell_npa_gart, 4); 5727 5728 amdgpu_job_free(job); 5729 5730 /* Copy failed: command never reached the remote, so no 5731 * writeback will arrive. Return now instead of polling the 5732 * completion for the full remote timeout. 5733 */ 5734 if (r) { 5735 dev_dbg(adev->dev, 5736 "remote %u lsdma copy failed (r %d), skip completion wait\n", 5737 remote_accel_id, r); 5738 return r; 5739 } 5740 5741 goto out_wait_complete; 5742 } 5743 5744 sdma_ring = &adev->sdma.instance[0].ring; 5745 5746 /* 5747 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands 5748 * execution doesn't overlap, doorbell update is after data and wptr update 5749 * finished. 5750 */ 5751 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5752 amdgpu_emit_copy_buffer(adev, ib, src, 5753 ring->rb_npa_gart + rem * 64, 5754 64, 0); 5755 5756 ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop; 5757 5758 amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0); 5759 5760 ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop; 5761 5762 amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0); 5763 5764 amdgpu_ring_pad_ib(sdma_ring, ib); 5765 WARN_ON(ib->length_dw > ndw_copy_cmd); 5766 5767 fence = amdgpu_job_submit(job); 5768 r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT); 5769 dma_fence_put(fence); 5770 if (r <= 0) { 5771 dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n", 5772 remote_accel_id, r); 5773 if (r == 0) 5774 r = -ETIME; 5775 goto out; 5776 } 5777 5778 out_wait_complete: 5779 /* 5780 * poll remote completion writeback data 5781 */ 5782 r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq); 5783 5784 /* 5785 * Advance wptr unless the remote timed out, even if the FW returned an 5786 * error status. On timeout the caller resets the connection, which 5787 * resyncs wptr from the remote in amdgpu_ualink_setup_connection. 5788 */ 5789 if (r != -ETIME) 5790 ring->wptr++; 5791 5792 out: 5793 dev_dbg(adev->dev, "ret r = %d\n", r); 5794 return r; 5795 } 5796 5797 static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev, 5798 u32 remote_accel_id, 5799 struct amdgpu_ualink_wb **wb_cpu, 5800 u64 *wb_npa, u32 type) 5801 { 5802 struct amdgpu_ualink_remote *remote = to_remote(adev); 5803 u32 accel_id = ualink_accel_id(adev); 5804 u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE); 5805 uintptr_t wb; 5806 u64 npa; 5807 u32 offset = 0; 5808 5809 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 5810 wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel; 5811 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, 5812 remote_accel_id, accel_id); 5813 } else { 5814 wb = (uintptr_t)remote->rptr_cpu_addr; 5815 npa = remote->rptr_npa; 5816 npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT; 5817 } 5818 5819 if (type == RB_TYPE_TLB_INV) 5820 offset = ualink_tlb_wb_offset(adev, remote_accel_id); 5821 else if (type == RB_TYPE_REMOTE_INTERRUPT) 5822 offset = ualink_wb_offset(adev, remote_accel_id); 5823 5824 *wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset); 5825 if (wb_npa) 5826 *wb_npa = npa + offset; 5827 5828 dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id, 5829 remote_accel_id, npa + offset); 5830 } 5831 5832 struct amdgpu_ualink_wptr_fence_cb { 5833 struct dma_fence_cb base; 5834 u64 *wptr_cpu; 5835 u64 result; 5836 }; 5837 5838 static void amdgpu_ualink_read_wptr_fence_cb(struct dma_fence *fence, 5839 struct dma_fence_cb *cb) 5840 { 5841 struct amdgpu_ualink_wptr_fence_cb *wptr_cb; 5842 5843 wptr_cb = container_of(cb, typeof(*wptr_cb), base); 5844 wptr_cb->result = *wptr_cb->wptr_cpu; 5845 } 5846 5847 static int amdgpu_ualink_read_remote_wptr(struct amdgpu_device *adev, 5848 u32 remote_accel_id, 5849 struct amdgpu_ualink_ring *ring, 5850 u64 *wptr) 5851 { 5852 struct amdgpu_ualink_remote *remote = to_remote(adev); 5853 struct amdgpu_ualink_peer *peer = &remote->peer[remote_accel_id]; 5854 struct amdgpu_ualink_wptr_fence_cb wptr_cb = { }; 5855 struct amdgpu_ring *sdma_ring; 5856 struct dma_fence *fence; 5857 struct amdgpu_job *job; 5858 struct amdgpu_ib *ib; 5859 u32 ndw, ndw_copy_cmd; 5860 u64 wptr_gpu, *wptr_cpu; 5861 int r; 5862 5863 *wptr = 0; 5864 5865 /* Refuse remote access unless active */ 5866 if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 5867 return -ESHUTDOWN; 5868 5869 /* 1 sdma copy command to read wptr */ 5870 ndw_copy_cmd = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); 5871 5872 /* wptr 2 dwords */ 5873 ndw = ndw_copy_cmd + 2; 5874 r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM, 5875 ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, 5876 AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job); 5877 if (r) 5878 return r; 5879 5880 ib = &job->ibs[0]; 5881 wptr_gpu = ib->gpu_addr + ndw_copy_cmd * 4; 5882 wptr_cpu = (u64 *)(ib->ptr + ndw_copy_cmd); 5883 5884 dev_dbg(adev->dev, "read wptr from remote %u npa gart wptr 0x%llx use %s\n", 5885 remote_accel_id, ring->wptr_npa_gart, 5886 remote->use_lsdma ? "lsdma" : "sdma"); 5887 5888 if (remote->use_lsdma) { 5889 r = amdgpu_lsdma_copy_mem(adev, ring->wptr_npa_gart, wptr_gpu, 8); 5890 if (!r) 5891 *wptr = *wptr_cpu; 5892 amdgpu_job_free(job); 5893 goto out; 5894 } 5895 5896 wptr_cb.wptr_cpu = wptr_cpu; 5897 5898 amdgpu_emit_copy_buffer(adev, ib, ring->wptr_npa_gart, wptr_gpu, 8, 0); 5899 5900 sdma_ring = &adev->sdma.instance[0].ring; 5901 amdgpu_ring_pad_ib(sdma_ring, ib); 5902 WARN_ON(ib->length_dw > ndw_copy_cmd); 5903 5904 fence = amdgpu_job_submit(job); 5905 5906 r = dma_fence_add_callback(fence, &wptr_cb.base, 5907 amdgpu_ualink_read_wptr_fence_cb); 5908 if (r == -ENOENT) { 5909 /* Fence already signaled; the callback won't run, do it inline. */ 5910 amdgpu_ualink_read_wptr_fence_cb(fence, &wptr_cb.base); 5911 r = 0; 5912 goto out_put_fence; 5913 } else if (r) { 5914 /* -EINVAL: NULL fence or func, not expected here. */ 5915 goto out_put_fence; 5916 } 5917 5918 r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT); 5919 if (r > 0) { 5920 r = 0; /* read wptr successfully */ 5921 goto out_put_fence; 5922 } 5923 5924 dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n", 5925 remote_accel_id, r); 5926 5927 if (r == 0) 5928 r = -ETIME; 5929 5930 /* 5931 * Timed out: the callback is still armed and wptr_cb is on our stack. 5932 * Cancel it so it can't fire after we return and write into the freed 5933 * stack frame. 5934 */ 5935 dma_fence_remove_callback(fence, &wptr_cb.base); 5936 5937 out_put_fence: 5938 if (!r) 5939 *wptr = wptr_cb.result; 5940 dma_fence_put(fence); 5941 5942 out: 5943 dev_dbg(adev->dev, "remote %u wptr 0x%llx return %d\n", 5944 remote_accel_id, *wptr, r); 5945 return r; 5946 } 5947 5948 /** 5949 * amdgpu_ualink_ring_resync_ptrs - sync local ring pointers from remote 5950 * @adev: amdgpu device pointer 5951 * @remote_accel_id: remote accelerator ID 5952 * 5953 * After a reboot on either side the two ends disagree on wptr. Read the remote 5954 * wptr of both peer rings and adopt it as the local wptr and rptr. 5955 * 5956 * Return 0 if both rings already match, 1 if they were resynced, or a negative 5957 * error code if a remote read failed, leaving the rings untouched. 5958 */ 5959 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev, 5960 u32 remote_accel_id) 5961 { 5962 struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id]; 5963 struct amdgpu_ualink_wb *int_wb, *tlb_wb; 5964 u64 int_wptr, tlb_wptr; 5965 int r; 5966 5967 /* ring->wptr and ring->rptr are owned by the send path. */ 5968 guard(mutex)(&peer->lock); 5969 5970 /* Read both before updating either, so a failure changes nothing. */ 5971 r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id, 5972 &peer->interrupt, &int_wptr); 5973 if (!r) 5974 r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id, 5975 &peer->shootdown, &tlb_wptr); 5976 if (r) { 5977 dev_dbg(adev->dev, "accel_id %u read remote %u wptr failed %d\n", 5978 ualink_accel_id(adev), remote_accel_id, r); 5979 return r; 5980 } 5981 5982 if (peer->interrupt.wptr == int_wptr && peer->shootdown.wptr == tlb_wptr) { 5983 dev_dbg(adev->dev, "accel_id %u wptr 0x%llx 0x%llx is same as remote %u\n", 5984 ualink_accel_id(adev), int_wptr, tlb_wptr, remote_accel_id); 5985 return 0; 5986 } 5987 5988 dev_dbg(adev->dev, 5989 "accel_id %u wptr 0x%llx 0x%llx sync to remote %u wptr 0x%llx 0x%llx\n", 5990 ualink_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr, 5991 remote_accel_id, int_wptr, tlb_wptr); 5992 5993 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL, 5994 RB_TYPE_REMOTE_INTERRUPT); 5995 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL, 5996 RB_TYPE_TLB_INV); 5997 5998 peer->interrupt.wptr = int_wptr; 5999 peer->interrupt.rptr = int_wptr; 6000 peer->shootdown.wptr = tlb_wptr; 6001 peer->shootdown.rptr = tlb_wptr; 6002 6003 /* Drop the stale writeback rptr, it disagrees with the resynced wptr. */ 6004 WRITE_ONCE(int_wb->rptr, int_wptr); 6005 WRITE_ONCE(tlb_wb->rptr, tlb_wptr); 6006 return 1; 6007 } 6008 6009 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev, 6010 u32 remote_accel_id) 6011 { 6012 struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id]; 6013 6014 dev_dbg(adev->dev, "remote accel_id %u\n", remote_accel_id); 6015 6016 guard(mutex)(&peer->lock); 6017 6018 peer->interrupt.ready = false; 6019 peer->shootdown.ready = false; 6020 } 6021 6022 static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev, 6023 u32 remote_accel_id, u32 ring_type) 6024 { 6025 struct amdgpu_ualink_remote *remote = to_remote(adev); 6026 struct amdgpu_ualink_peer *peer; 6027 struct amdgpu_ualink_ring *ring; 6028 struct amdgpu_ualink_wb *wb_cpu; 6029 u64 wb_npa; 6030 int r; 6031 6032 peer = &remote->peer[remote_accel_id]; 6033 if (ring_type == RB_TYPE_REMOTE_INTERRUPT) 6034 ring = &peer->interrupt; 6035 else if (ring_type == RB_TYPE_TLB_INV) 6036 ring = &peer->shootdown; 6037 else 6038 return -EINVAL; 6039 6040 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa, 6041 ring_type); 6042 6043 r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu, 6044 amdgpu_ualink_emit_update_wb_addr, 6045 upper_32_bits(wb_npa), 6046 lower_32_bits(wb_npa), 6047 0, 0); 6048 if (!r) { 6049 ring->wb_cpu = wb_cpu; 6050 ring->ready = true; 6051 } 6052 return r; 6053 } 6054 6055 /** 6056 * amdgpu_ualink_remote_shootdown - Send a remote TLB invalidation request 6057 * @adev: amdgpu device pointer 6058 * @remote_accel_id: accelerator ID of the remote GPU to shootdown 6059 * @addr: page-aligned address to invalidate 6060 * @size_in_pages: number of pages to invalidate, 0 to invalidate entire TLB 6061 * @flush_type: type of TLB flush to perform 6062 * 6063 * Sends a remote TLB shootdown command to the specified peer GPU via the 6064 * UALink ring buffer. The address and size are translated using the S-field 6065 * encoding before being written to the ring. 6066 * 6067 * Return: 0 on success, negative error code on failure 6068 */ 6069 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev, 6070 u32 remote_accel_id, u64 addr, 6071 u32 size_in_pages, u32 flush_type) 6072 { 6073 struct amdgpu_ualink_remote *remote = to_remote(adev); 6074 struct amdgpu_ualink_peer *peer; 6075 struct amdgpu_ualink_ring *ring; 6076 int r; 6077 6078 dev_dbg(adev->dev, "skip remote shootdown to remote_accel_id %u\n", 6079 remote_accel_id); 6080 return 0; 6081 6082 peer = &remote->peer[remote_accel_id]; 6083 ring = &peer->shootdown; 6084 6085 guard(mutex)(&peer->lock); 6086 6087 if (!ring->ready) { 6088 dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id); 6089 r = amdgpu_ualink_update_wb_address(adev, remote_accel_id, 6090 RB_TYPE_TLB_INV); 6091 if (r) 6092 return r; 6093 } 6094 6095 return amdgpu_ualink_send_command(adev, remote_accel_id, ring, 6096 ring->wb_cpu, 6097 amdgpu_ualink_emit_shootdown, 6098 flush_type, upper_32_bits(addr), 6099 lower_32_bits(addr), size_in_pages); 6100 } 6101 6102 /** 6103 * amdgpu_ualink_remote_interrupt - Send a remote interrupt to a peer GPU 6104 * @adev: amdgpu device pointer 6105 * @remote_accel_id: accelerator ID of the remote GPU to interrupt 6106 * @dw0: IH context ID dword 0 6107 * @dw1: IH context ID dword 1 6108 * @dw2: IH context ID dword 2 6109 * @dw3: IH context ID dword 3 6110 * 6111 * Sends a remote interrupt command to the specified peer GPU via the 6112 * UALink ring buffer. The four context ID dwords are delivered to the 6113 * remote GPU's interrupt handler. 6114 * 6115 * Return: 0 on success, negative error code on failure 6116 */ 6117 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev, 6118 u32 remote_accel_id, u32 dw0, u32 dw1, 6119 u32 dw2, u32 dw3) 6120 { 6121 struct amdgpu_ualink_remote *remote = to_remote(adev); 6122 struct amdgpu_ualink_peer *peer; 6123 struct amdgpu_ualink_ring *ring; 6124 int r; 6125 6126 peer = &remote->peer[remote_accel_id]; 6127 ring = &peer->interrupt; 6128 6129 guard(mutex)(&peer->lock); 6130 6131 if (!ring->ready) { 6132 dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id); 6133 r = amdgpu_ualink_update_wb_address(adev, remote_accel_id, 6134 RB_TYPE_REMOTE_INTERRUPT); 6135 if (r) 6136 return r; 6137 } 6138 6139 return amdgpu_ualink_send_command(adev, remote_accel_id, ring, 6140 ring->wb_cpu, 6141 amdgpu_ualink_emit_interrupt, 6142 dw0, dw1, dw2, dw3); 6143 } 6144 6145 /** 6146 * amdgpu_ualink_peer_remote_init - Initialize remote peer GPU connections 6147 * @adev: amdgpu device pointer 6148 * 6149 * Sets up ring buffers, GART mappings, and control structures for 6150 * communicating with remote GPUs in the fabric. 6151 * 6152 * Return: 0 on success, negative error code on failure 6153 */ 6154 static int amdgpu_ualink_peer_remote_init(struct amdgpu_device *adev) 6155 { 6156 u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT; 6157 struct amdgpu_ualink_remote *remote = to_remote(adev); 6158 struct amdgpu_bo *bo = remote->ring_bo; 6159 struct amdgpu_ualink_peer *peer; 6160 struct amdgpu_ualink_ring *interrupt; 6161 struct amdgpu_ualink_ring *shootdown; 6162 u32 src_accel_id = ualink_accel_id(adev); 6163 u32 dst_accel_id; 6164 u64 npa, flags; 6165 int r = 0; 6166 6167 /* NPA mapping PTE flags VSCT = 0011 */ 6168 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, bo->tbo.resource); 6169 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | AMDGPU_PTE_BUS_ATOMICS; 6170 flags &= ~AMDGPU_PTE_VALID; 6171 flags &= ~AMDGPU_PTE_SYSTEM; 6172 /* PTE.X=0 turn off RPC checks for RBs, wptr and doorbell NPA address */ 6173 flags &= ~AMDGPU_PTE_EXECUTABLE; 6174 6175 dev_dbg(adev->dev, "src_accel_id %u gart mapping flags 0x%llx\n", 6176 src_accel_id, flags); 6177 6178 for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 6179 if (dst_accel_id == src_accel_id) 6180 continue; 6181 6182 dev_dbg(adev->dev, "peer_remote to dst_accel_id %u\n", dst_accel_id); 6183 6184 peer = &remote->peer[dst_accel_id]; 6185 mutex_init(&peer->lock); 6186 6187 interrupt = &peer->interrupt; 6188 shootdown = &peer->shootdown; 6189 6190 /* ring buffer npa gart mapping */ 6191 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 6192 src_accel_id, dst_accel_id); 6193 r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &interrupt->mm_node_rb, flags); 6194 if (r) 6195 break; 6196 interrupt->rb_npa_gart = adev->gmc.gart_start + 6197 (interrupt->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT); 6198 6199 dev_dbg(adev->dev, "rb npa 0x%llx mapped on gart 0x%llx\n", 6200 npa, interrupt->rb_npa_gart); 6201 6202 /* tlb invalidate ring buffer npa gart mapping */ 6203 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TLB_INV, src_accel_id, 6204 dst_accel_id); 6205 r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &shootdown->mm_node_rb, flags); 6206 if (r) 6207 break; 6208 shootdown->rb_npa_gart = adev->gmc.gart_start + 6209 (shootdown->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT); 6210 6211 dev_dbg(adev->dev, "tlb rb npa 0x%llx mapped on gart 0x%llx\n", 6212 npa, shootdown->rb_npa_gart); 6213 6214 /* wptr npa gart mapping */ 6215 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TAILPTR, src_accel_id, 6216 dst_accel_id); 6217 r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_wptr, flags); 6218 if (r) 6219 break; 6220 6221 interrupt->wptr_npa_gart = adev->gmc.gart_start + 6222 (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT) 6223 + ualink_wptr_offset(adev, src_accel_id); 6224 shootdown->wptr_npa_gart = adev->gmc.gart_start + 6225 (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT) 6226 + ualink_tlb_wptr_offset(adev, src_accel_id); 6227 6228 dev_dbg(adev->dev, "rb wptr npa 0x%llx on gart 0x%llx, tlb wptr on gart 0x%llx\n", 6229 npa, interrupt->wptr_npa_gart, shootdown->wptr_npa_gart); 6230 6231 /* 6232 * doorbell npa gart mapping, doorbell NPA address coding 6233 * 6234 * ReqAddr[51:40] = 0xFFF 6235 * ReqAddr[39:30] = GPU accel_id 6236 * ReqAddr[29:12] = 18'h00000 6237 * ReqAddr[11:8] = Random value 6238 * ReqAddr[7:6] = PMI Index 6239 * 0 - PMI0 Remote Interrupt 6240 * 1 - PMI1 Remote TLB Shootdown 6241 * ReqAddr[5:0] = 6'h0 6242 */ 6243 npa = 0xFFFULL << 40 | (u64)dst_accel_id << 30; 6244 r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_doorbell, flags); 6245 if (r) 6246 break; 6247 6248 interrupt->doorbell_npa_gart = adev->gmc.gart_start + 6249 (interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT); 6250 6251 dev_dbg(adev->dev, "rb doorbell npa 0x%llx mapped on gart 0x%llx\n", 6252 npa, interrupt->doorbell_npa_gart); 6253 6254 shootdown->doorbell_npa_gart = adev->gmc.gart_start+ 6255 (interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT) + (1 << 6); 6256 6257 dev_dbg(adev->dev, "tlb doorbell npa 0x%llx mapped on gart 0x%llx\n", 6258 npa, shootdown->doorbell_npa_gart); 6259 6260 interrupt->rb_size = AMDGPU_UALINK_RB_SIZE / 64; 6261 interrupt->wptr = 0; 6262 interrupt->rptr = 0; 6263 interrupt->seq = 0; 6264 6265 shootdown->rb_size = AMDGPU_UALINK_RB_SIZE / 64; 6266 shootdown->wptr = 0; 6267 shootdown->rptr = 0; 6268 shootdown->seq = 0; 6269 } 6270 6271 dev_dbg(adev->dev, "peer remote init done r=%d\n", r); 6272 return r; 6273 } 6274 6275 /** 6276 * amdgpu_ualink_peer_remote_fini - Clean up remote peer GPU connections 6277 * @adev: amdgpu device pointer 6278 * 6279 * Unmaps GART entries and frees resources for remote GPU communication. 6280 */ 6281 static void amdgpu_ualink_peer_remote_fini(struct amdgpu_device *adev) 6282 { 6283 u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT; 6284 struct amdgpu_ualink_remote *remote = to_remote(adev); 6285 u32 src_accel_id = ualink_accel_id(adev); 6286 u32 dst_accel_id; 6287 struct amdgpu_ualink_peer *peer; 6288 struct amdgpu_ualink_ring *ring; 6289 6290 dev_dbg(adev->dev, "src_accel_id %u\n", src_accel_id); 6291 6292 for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 6293 dev_dbg(adev->dev, "dst_accel_id %u\n", dst_accel_id); 6294 if (dst_accel_id == src_accel_id) 6295 continue; 6296 6297 peer = &remote->peer[dst_accel_id]; 6298 mutex_destroy(&peer->lock); 6299 6300 ring = &peer->interrupt; 6301 amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb); 6302 amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_wptr); 6303 amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_doorbell); 6304 6305 ring = &peer->shootdown; 6306 amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb); 6307 } 6308 } 6309 6310 /** 6311 * amdgpu_ualink_sw_init - Initialize UALink software resources 6312 * @adev: amdgpu device pointer 6313 * 6314 * Performs full UALink software initialization including metadata allocation, 6315 * NPA address mapping, SDMA scheduler entity setup, and remote peer GPU 6316 * connection initialization. On failure, all previously initialized resources 6317 * are cleaned up. 6318 * 6319 * Return: 0 on success, negative error code on failure 6320 */ 6321 int amdgpu_ualink_sw_init(struct amdgpu_device *adev) 6322 { 6323 struct amdgpu_ualink_remote *remote; 6324 int r; 6325 6326 remote = kzalloc(sizeof(*remote), GFP_KERNEL); 6327 if (!remote) 6328 return -ENOMEM; 6329 adev->ualink.remote = remote; 6330 6331 r = amdgpu_ualink_metadata_init(adev); 6332 if (r) 6333 goto out; 6334 6335 r = amdgpu_ualink_metadata_npa_mapping(adev); 6336 if (r) 6337 goto out_metadata_fini; 6338 6339 r = amdgpu_ualink_sdma_entities_init(adev); 6340 if (r) 6341 goto out_npa_unmap; 6342 6343 r = amdgpu_ualink_peer_remote_init(adev); 6344 if (r) 6345 goto out_sdma_entities_fini; 6346 6347 dev_dbg(adev->dev, "ualink sw init succeed\n"); 6348 return 0; 6349 6350 out_sdma_entities_fini: 6351 amdgpu_ualink_sdma_entities_fini(adev); 6352 out_npa_unmap: 6353 amdgpu_ualink_metadata_npa_unmapping(adev); 6354 out_metadata_fini: 6355 amdgpu_ualink_metadata_fini(adev); 6356 out: 6357 kfree(adev->ualink.remote); 6358 adev->ualink.remote = NULL; 6359 dev_dbg(adev->dev, "ualink sw init failed %d\n", r); 6360 return r; 6361 } 6362 6363 /** 6364 * amdgpu_ualink_sw_fini - Tear down UALink software resources 6365 * @adev: amdgpu device pointer 6366 * 6367 * Cleans up all UALink software resources in reverse order of initialization: 6368 * remote peer connections, SDMA entities, NPA mappings, and metadata. 6369 */ 6370 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev) 6371 { 6372 dev_dbg(adev->dev, "halt accel_id %u addr_mode %d\n", ualink_accel_id(adev), 6373 ualink_addr_mode(adev)); 6374 6375 adev->ualink.msg_ctl->send_halt(adev); 6376 6377 amdgpu_ualink_peer_remote_fini(adev); 6378 amdgpu_ualink_sdma_entities_fini(adev); 6379 amdgpu_ualink_metadata_npa_unmapping(adev); 6380 amdgpu_ualink_metadata_fini(adev); 6381 kfree(adev->ualink.remote); 6382 adev->ualink.remote = NULL; 6383 } 6384 6385 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id) 6386 { 6387 return amdgpu_ualink_send_hello_msg(adev, remote_accel_id); 6388 } 6389 6390 /* 6391 * The low bits of handle_lo carry the message-header field on the wire. 6392 * Strip those bits and splice in @acc_id to recover the 6393 * fully-qualified handle_lo value. 6394 */ 6395 static inline u64 amdgpu_ualink_reassemble_handle(u64 handle_lo, u32 acc_id) 6396 { 6397 return (handle_lo & ~AMDGPU_UALINK_MESSAGE_HEADER_MASK) | 6398 (acc_id & AMDGPU_UALINK_HANDLE_ACCID_MASK); 6399 } 6400 6401 static int amdgpu_ualink_process_irq(struct amdgpu_device *adev, 6402 struct amdgpu_irq_src *source, 6403 struct amdgpu_iv_entry *entry) 6404 { 6405 u32 sender_acc_id, receiver_acc_id, msg_type, src_acc_id; 6406 enum amdgpu_ualink_accel_state accel_state; 6407 struct amdgpu_ualink_handle handle; 6408 u32 size, npa_addr, fail_reason; 6409 u32 dw0, dw1, dw2, dw3; 6410 u32 local_acc_id; 6411 int handled = 1; 6412 6413 if (unlikely(adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)) { 6414 dev_dbg(adev->dev, 6415 "UALink manager not initialized, dropping irq\n"); 6416 return handled; 6417 } 6418 6419 /* Only handle interrupts while ACTIVE. Otherwise (e.g. during teardown) 6420 * consume stale ring entries but drop them, so they don't touch state 6421 * being freed. 6422 */ 6423 accel_state = adev->ualink.info->accel_state; 6424 if (accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) { 6425 dev_dbg(adev->dev, 6426 "Dropping stale UALink interrupt, accel_state %d\n", 6427 accel_state); 6428 return handled; 6429 } 6430 6431 dev_dbg(adev->dev, "%s client_id 0x%x src_id 0x%x ih\n", 6432 entry->ih == &adev->irq.ih ? "ring" : "ualink soft ring", 6433 entry->client_id, entry->src_id); 6434 6435 /* ContextID 4 dwords */ 6436 src_acc_id = entry->pasid; 6437 dw0 = entry->src_data[0]; 6438 dw1 = entry->src_data[1]; 6439 dw2 = entry->src_data[2]; 6440 dw3 = entry->src_data[3]; 6441 6442 /* Copy IH entry into ualink soft ring. */ 6443 if (entry->ih == &adev->irq.ih) { 6444 dev_dbg(adev->dev, "src accel_id %u context id 0x%x 0x%x 0x%x 0x%x\n", 6445 src_acc_id, dw0, dw1, dw2, dw3); 6446 dev_dbg(adev->dev, "delegate to ualink irq soft ring\n"); 6447 amdgpu_irq_ualink_delegate(adev, entry, 8); 6448 return handled; 6449 } 6450 6451 msg_type = dw0 & AMDGPU_UALINK_MESSAGE_HEADER_MASK; 6452 local_acc_id = adev->ualink.info->ppod.accel_id; 6453 dev_dbg(adev->dev, "Got MSG: remote acc_id %u msg_type %u\n", 6454 src_acc_id, msg_type); 6455 6456 /* Debug hook: if the bit corresponding to this msg_type is set in 6457 * drop_msg_bitmap, drop this single packet and clear the bit so that 6458 * any subsequent incoming messages are processed normally. Used to 6459 * exercise the connection reset/recovery paths via debugfs. 6460 */ 6461 if (msg_type < BITS_PER_LONG && 6462 test_and_clear_bit(msg_type, &adev->ualink.drop_msg_bitmap)) { 6463 dev_warn(adev->dev, 6464 "DROP MSG (debugfs): src acc_id %u msg_type %u dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 6465 src_acc_id, msg_type, dw0, dw1, dw2, dw3); 6466 return handled; 6467 } 6468 6469 switch (msg_type) { 6470 case AMDGPU_UALINK_HELLO_MSG: 6471 receiver_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT) & 6472 AMDGPU_UALINK_HELLO_MSG_ACCID_MASK; 6473 sender_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT) & 6474 AMDGPU_UALINK_HELLO_MSG_ACCID_MASK; 6475 dev_dbg(adev->dev, 6476 "Got HELLO MSG: src acc_id %u receiver_acc_id %u sender_acc_id %u\n", 6477 src_acc_id, receiver_acc_id, sender_acc_id); 6478 amdgpu_ualink_process_hello_msg(adev, receiver_acc_id, sender_acc_id, 6479 src_acc_id); 6480 break; 6481 case AMDGPU_UALINK_HELLO_ACK_MSG: 6482 dev_dbg(adev->dev, 6483 "Got HELLO-ACK MSG: remote acc_id %u\n", src_acc_id); 6484 amdgpu_ualink_process_hello_ack_msg(adev, src_acc_id); 6485 break; 6486 case AMDGPU_UALINK_NPA_REQ_MSG: 6487 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6488 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6489 local_acc_id); 6490 dev_dbg(adev->dev, 6491 "Got NPA-REQ MSG: remote acc_id %u handle %llx:%llx\n", 6492 src_acc_id, handle.handle_hi, handle.handle_lo); 6493 amdgpu_ualink_process_npa_req_msg(adev, src_acc_id, handle); 6494 break; 6495 case AMDGPU_UALINK_NPA_RSP_MSG: 6496 memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo)); 6497 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6498 src_acc_id); 6499 size = entry->src_data[2]; 6500 npa_addr = entry->src_data[3]; 6501 dev_dbg(adev->dev, 6502 "Got NPA-RSP MSG: remote acc_id %u handle_lo %llx size 0x%x npa_addr 0x%x\n", 6503 src_acc_id, handle.handle_lo, size, npa_addr); 6504 amdgpu_ualink_process_npa_rsp_msg(adev, src_acc_id, 6505 handle.handle_lo, 6506 npa_addr, size); 6507 break; 6508 case AMDGPU_UALINK_NPA_FAIL_MSG: 6509 memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo)); 6510 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6511 src_acc_id); 6512 fail_reason = entry->src_data[2] & 6513 AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK; 6514 dev_dbg(adev->dev, 6515 "Got NPA-FAIL MSG: remote acc_id %u handle_lo %llx fail_reason %u\n", 6516 src_acc_id, handle.handle_lo, fail_reason); 6517 amdgpu_ualink_process_npa_fail_msg(adev, src_acc_id, 6518 handle.handle_lo, 6519 fail_reason); 6520 break; 6521 case AMDGPU_UALINK_NPA_REVOKE_MSG: 6522 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6523 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6524 src_acc_id); 6525 dev_dbg(adev->dev, 6526 "Got NPA-REVOKE MSG: remote acc_id %u handle %llx:%llx\n", 6527 src_acc_id, handle.handle_hi, handle.handle_lo); 6528 amdgpu_ualink_process_npa_revoke_msg(adev, src_acc_id, handle); 6529 break; 6530 case AMDGPU_UALINK_NPA_RELEASE_MSG: 6531 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6532 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6533 local_acc_id); 6534 dev_dbg(adev->dev, 6535 "Got NPA-RELEASE MSG: remote acc_id %u handle %llx:%llx\n", 6536 src_acc_id, handle.handle_hi, handle.handle_lo); 6537 amdgpu_ualink_process_npa_release_msg(adev, src_acc_id, handle); 6538 break; 6539 default: 6540 dev_err(adev->dev, "Unknown message type (%u)\n", msg_type); 6541 break; 6542 } 6543 6544 return handled; 6545 } 6546 6547 static int amdgpu_ualink_set_irq_state(struct amdgpu_device *adev, 6548 struct amdgpu_irq_src *source, 6549 u32 type, 6550 enum amdgpu_interrupt_state state) 6551 { 6552 /* 6553 * Don't set register to enable/disable nHT controller interrupt. 6554 * 6555 * F/W running on MP2, which can always send cookie to IH block to 6556 * interrupt driver. 6557 */ 6558 dev_dbg(adev->dev, "ualink interrupt %s\n", 6559 state == AMDGPU_IRQ_STATE_ENABLE ? "enable" : "disable"); 6560 return 0; 6561 } 6562 6563 static const struct amdgpu_irq_src_funcs ualink_irq_funcs = { 6564 .set = amdgpu_ualink_set_irq_state, 6565 .process = amdgpu_ualink_process_irq, 6566 }; 6567 6568 /** 6569 * amdgpu_ualink_init_interrupt - register the UALink IRQ source 6570 * @adev: amdgpu device pointer 6571 * @client_id: IH client ID for the interrupt source 6572 * @src_id: source ID for the interrupt source 6573 * 6574 * Registers the UALink interrupt source with the IH subsystem. 6575 * 6576 * Return: 0 on success, negative error code on failure 6577 */ 6578 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev, 6579 unsigned int client_id, unsigned int src_id) 6580 { 6581 int r; 6582 6583 dev_dbg(adev->dev, "init ualink irq client_id 0x%x src_id 0x%x\n", 6584 client_id, src_id); 6585 6586 adev->ualink.irq.num_types = 1; 6587 adev->ualink.irq.funcs = &ualink_irq_funcs; 6588 6589 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->ualink.irq); 6590 return r; 6591 } 6592 6593