1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2026 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/xarray.h> 26 #include <drm/drm_mm.h> 27 #include <linux/random.h> 28 #include "amdgpu.h" 29 #include "amdgpu_sdma.h" 30 #include "amdgpu_ualink.h" 31 #include "amdgpu_xgmi.h" 32 #include "amdgpu_dma_buf.h" 33 #include "psp_gfx_if.h" 34 #include <linux/math.h> 35 #include <linux/sysfs.h> 36 #include <linux/string.h> 37 38 static void deactivate_accelerator(struct amdgpu_device *adev); 39 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev, 40 u32 remote_accel_id, u32 dw0, u32 dw1, 41 u32 dw2, u32 dw3); 42 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, 43 u32 flush_type); 44 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev, 45 struct amdgpu_bo *bos[], u32 n_bos, 46 struct drm_exec *exec, 47 bool interruptible); 48 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev, 49 struct drm_exec *exec); 50 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev, 51 u32 remote_accel_id, u32 state, 52 u32 generation_count); 53 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo); 54 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev, 55 u32 remote_accel_id, u64 addr, 56 u32 size_in_pages, u32 flush_type); 57 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev); 58 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev); 59 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev, 60 u32 remote_accel_id); 61 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev, 62 u32 remote_accel_id); 63 64 #define STRIP_NPA(addr) \ 65 (((u64)(addr) & ~AMDGPU_UALINK_NPA_ADDR_GPUID_MASK)) 66 67 #define GENERATE_NPA(addr, remote_acc_id) \ 68 ((u64)(((u64)(addr)) | \ 69 ((u64)(remote_acc_id) << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT))) 70 71 static const struct drm_client_funcs ualink_client_funcs = { 72 .unregister = drm_client_release, 73 }; 74 75 static int amdgpu_ualink_drm_client_create(struct amdgpu_device *adev) 76 { 77 int ret; 78 79 ret = drm_client_init(&adev->ddev, &adev->ualink.client, "ualink", 80 &ualink_client_funcs); 81 if (ret) { 82 dev_err(adev->dev, "Failed to init UALink DRM client: %d\n", 83 ret); 84 return ret; 85 } 86 87 drm_client_register(&adev->ualink.client); 88 89 return 0; 90 } 91 92 static void amdgpu_ualink_object_fini(struct amdgpu_device *adev) 93 { 94 if (!adev->ualink.info) 95 return; 96 97 kobject_put(&adev->ualink.stations->kobj); 98 kobject_put(&adev->ualink.config->kobj); 99 kobject_put(&adev->ualink.setup->kobj); 100 kobject_put(&adev->ualink.info->kobj); 101 adev->ualink.stations = NULL; 102 adev->ualink.config = NULL; 103 adev->ualink.setup = NULL; 104 adev->ualink.info = NULL; 105 } 106 107 static bool __check_ppod_info(struct amdgpu_device *adev, 108 const struct amdgpu_ualink_info *info) 109 { 110 const struct amdgpu_ualink_ppod_info *ppod = &info->ppod; 111 112 if (ppod->size <= 0 || ppod->size > AMDGPU_UALINK_ACCEL_MAX) { 113 dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n", 114 ppod->size, AMDGPU_UALINK_ACCEL_MAX); 115 return false; 116 } 117 if (ppod->accel_id >= ppod->size) { 118 dev_dbg(adev->dev, 119 "Accelerator ID %u greater or equal pPod size %u\n", 120 ppod->accel_id, ppod->size); 121 return false; 122 } 123 124 return true; 125 } 126 127 static bool __check_vpod_info(struct amdgpu_device *adev, 128 const struct amdgpu_ualink_info *info) 129 { 130 const struct amdgpu_ualink_ppod_info *ppod = &info->ppod; 131 const struct amdgpu_ualink_vpod_info *vpod = &info->vpod; 132 unsigned int weight; 133 134 if (vpod->size == 0 || vpod->size > ppod->size) { 135 dev_warn(adev->dev, 136 "UALINK: vPod size %u out of range [1..%u]\n", 137 vpod->size, ppod->size); 138 return false; 139 } 140 141 switch (vpod->addr_mode) { 142 case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS: 143 break; 144 case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT: 145 dev_warn(adev->dev, 146 "UALINK: source-identification addr mode is not supported\n"); 147 return false; 148 default: 149 dev_warn(adev->dev, "UALINK: invalid addr mode %u\n", 150 vpod->addr_mode); 151 return false; 152 } 153 154 weight = bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); 155 if (weight != vpod->size) { 156 dev_warn(adev->dev, 157 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n", 158 vpod->size, weight); 159 return false; 160 } 161 if (!test_bit(ppod->accel_id, vpod->active_accel_bits)) { 162 dev_warn(adev->dev, 163 "UALINK: accelerator ID %u not listed in vpod_active_accels\n", 164 ppod->accel_id); 165 return false; 166 } 167 168 return true; 169 } 170 171 static void 172 amdgpu_ualink_info_set_accel_state(struct amdgpu_device *adev, 173 struct amdgpu_ualink_info *info, 174 enum psp_gfx_ual_config_state cfg_state) 175 { 176 enum amdgpu_ualink_accel_state cur; 177 enum amdgpu_ualink_accel_state target; 178 bool ppod_validated; 179 bool vpod_validated; 180 181 if (!info) 182 return; 183 184 cur = info->accel_state; 185 186 ppod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED && 187 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 188 vpod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED && 189 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 190 191 switch (cfg_state) { 192 case UAL_CFG_IDLE: 193 return; 194 case UAL_CFG_PPOD: 195 target = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 196 break; 197 case UAL_CFG_VPOD: 198 case UAL_CFG_STATION: 199 target = AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED; 200 break; 201 case UAL_CFG_COMPLETE: 202 target = AMDGPU_UALINK_ACCEL_STATE_READY; 203 break; 204 default: 205 dev_dbg(adev->dev, "invalid configuration state %u", cfg_state); 206 return; 207 } 208 209 /* ppod stage: should be part of a ppod first */ 210 if (!ppod_validated && !__check_ppod_info(adev, info)) { 211 if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 212 info->accel_state = 213 AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 214 } else { 215 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR; 216 dev_err(adev->dev, 217 "ppod configuration is invalid, setting to error state"); 218 } 219 return; 220 } 221 if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 222 if (!ppod_validated) 223 info->accel_state = 224 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 225 return; 226 } 227 228 /* A vpod_id of 0 means the GPU is not part of any vPod */ 229 if (info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) { 230 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 231 return; 232 } 233 234 /* vpod stage: required to reach vpod_configured or ready */ 235 if (!vpod_validated && !__check_vpod_info(adev, info)) { 236 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR; 237 dev_err(adev->dev, 238 "vpod configuration is invalid, setting to error state"); 239 return; 240 } 241 if (target == AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) { 242 if (!vpod_validated) 243 info->accel_state = 244 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED; 245 return; 246 } 247 248 /* complete stage: advance to ready unless already ready/active */ 249 if (cur < AMDGPU_UALINK_ACCEL_STATE_READY || 250 cur > AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 251 info->accel_state = AMDGPU_UALINK_ACCEL_STATE_READY; 252 } 253 254 static int amdgpu_ualink_update_vpod_config(struct amdgpu_device *adev) 255 { 256 /* TBD: Do updates/cleanup based on updated vpod configuration */ 257 return 0; 258 } 259 260 static int amdgpu_ualink_update_accel_state( 261 struct amdgpu_device *adev, enum amdgpu_ualink_accel_state prev_state, 262 u32 prev_vpod_id, enum psp_gfx_ual_config_state cfg_state) 263 { 264 dev_info(adev->dev, 265 "UALINK: update_accel_state: prev_state=%d prev_vpod_id=%u new_vpod_id=%u cfg_state=%d\n", 266 prev_state, prev_vpod_id, adev->ualink.info->vpod.id, 267 cfg_state); 268 269 /* If the device is already active and its vpod_id is unchanged, the 270 * update does not affect vpod membership. Skip the local vpod 271 * integrity check and re-activation. 272 */ 273 if (prev_state == AMDGPU_UALINK_ACCEL_STATE_ACTIVE && 274 adev->ualink.info->vpod.id == prev_vpod_id) { 275 amdgpu_ualink_update_vpod_config(adev); 276 277 /* Same vpod_id, but the member set may have changed (vPod grown 278 * or shrunk while this GPU stayed ACTIVE). If so, run the local 279 * vpod activation path: once every local peer has committed the 280 * new config (integrity passes), it bounces the affected ACTIVE 281 * peers to rebuild links/GART for the new member set. 282 */ 283 if (amdgpu_ualink_vpod_membership_changed(adev)) { 284 dev_info(adev->dev, 285 "UALINK: update_accel_state: ACTIVE vpod_id=%u membership changed, reconfiguring\n", 286 adev->ualink.info->vpod.id); 287 scoped_guard(mutex, &mgpu_info.mutex) 288 __amdgpu_ualink_activate_vpod_locked(adev); 289 } else { 290 dev_info(adev->dev, 291 "UALINK: update_accel_state: already ACTIVE, vpod_id/membership unchanged\n"); 292 } 293 return 0; 294 } 295 296 /* A new vpod_id of 0 means this GPU was removed from the vPod. */ 297 if (adev->ualink.info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) { 298 dev_info(adev->dev, 299 "UALINK: update_accel_state: vpod_id=0, removing accelerator from vPod\n"); 300 amdgpu_ualink_update_vpod_config(adev); 301 scoped_guard(mutex, &mgpu_info.mutex) 302 deactivate_accelerator(adev); 303 return 0; 304 } 305 306 /* GPU joining a new vpod should be with invalid vpod id*/ 307 scoped_guard(mutex, &mgpu_info.mutex) { 308 if (prev_vpod_id == AMDGPU_UALINK_VPOD_ID_INVALID) { 309 dev_info(adev->dev, 310 "UALINK: update_accel_state: joining vpod_id=%u\n", 311 adev->ualink.info->vpod.id); 312 amdgpu_ualink_info_set_accel_state( 313 adev, adev->ualink.info, cfg_state); 314 __amdgpu_ualink_activate_vpod_locked(adev); 315 } else { 316 /* GPU should first get removal which will set invalid vpod_id 317 * and then join a new vpod 318 */ 319 dev_err(adev->dev, 320 "Invalid vpod transition from %u to %u\n", 321 prev_vpod_id, adev->ualink.info->vpod.id); 322 return -EINVAL; 323 } 324 } 325 326 return 0; 327 } 328 329 int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev) 330 { 331 enum amdgpu_ualink_accel_state prev_state; 332 enum psp_gfx_ual_config_state cfg_state; 333 u32 prev_vpod_id; 334 int r, qerr; 335 336 /* TBD: Stop ASP interrupts if driver faced an issue */ 337 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 338 u32 status; 339 340 dev_dbg(adev->dev, 341 "UALink not initialized, skipping config update\n"); 342 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 343 return psp_ual_send_completion( 344 &adev->psp, adev->ualink.psp_if_ver, 345 PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, status); 346 } 347 348 prev_state = adev->ualink.info->accel_state; 349 prev_vpod_id = adev->ualink.info->vpod.id; 350 351 qerr = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, 352 adev->ualink.info, &cfg_state); 353 354 /*TBD: find the right value of status to be sent to ASP*/ 355 r = psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 356 PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, 0); 357 if (r || qerr) 358 goto err; 359 360 r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id, 361 cfg_state); 362 if (r) 363 goto err; 364 365 return 0; 366 367 err: 368 scoped_guard(mutex, &mgpu_info.mutex) { 369 deactivate_accelerator(adev); 370 adev->ualink.info->accel_state = 371 AMDGPU_UALINK_ACCEL_STATE_ERROR; 372 } 373 dev_err(adev->dev, 374 "UALink config update failed, setting to error state"); 375 376 return r; 377 } 378 379 int amdgpu_ualink_pause_handler(struct amdgpu_device *adev) 380 { 381 u32 status = 0; 382 383 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 384 dev_dbg(adev->dev, 385 "UALink not initialized, skipping pause update\n"); 386 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 387 goto out; 388 } 389 390 dev_dbg(adev->dev, "UALink pause command is not handled\n"); 391 392 out: 393 return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 394 PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID, 395 status); 396 } 397 398 int amdgpu_ualink_resume_handler(struct amdgpu_device *adev) 399 { 400 u32 status = 0; 401 402 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) { 403 dev_dbg(adev->dev, 404 "UALink not initialized, skipping pause update\n"); 405 status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR); 406 goto out; 407 } 408 409 dev_dbg(adev->dev, "UALink resume command is not handled\n"); 410 411 out: 412 return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver, 413 PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID, 414 status); 415 } 416 417 int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block) 418 { 419 struct amdgpu_device *adev = ip_block->adev; 420 int r; 421 422 if (!adev->ualink.info) 423 return 0; 424 425 adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 426 r = psp_ual_get_interface_version(&adev->psp, &adev->ualink.psp_if_ver); 427 if (r) { 428 adev->ualink.psp_if_ver = 0xffffffff; 429 goto disable; 430 } 431 dev_info(adev->dev, "Found UALink interface version 0x%x\n", 432 adev->ualink.psp_if_ver); 433 434 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_HW; 435 436 return 0; 437 disable: 438 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR; 439 return 0; 440 } 441 442 int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block) 443 { 444 struct amdgpu_device *adev = ip_block->adev; 445 446 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) 447 return 0; 448 449 scoped_guard(mutex, &mgpu_info.mutex) 450 deactivate_accelerator(adev); 451 452 return 0; 453 } 454 455 int ualink_ip_late_init(struct amdgpu_ip_block *ip_block) 456 { 457 struct amdgpu_device *adev = ip_block->adev; 458 enum psp_gfx_ual_config_state cfg_state; 459 int r; 460 461 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_HW) 462 return 0; 463 464 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, 465 adev->ualink.info, &cfg_state); 466 if (r) 467 return r; 468 469 scoped_guard(mutex, &mgpu_info.mutex) { 470 amdgpu_ualink_info_set_accel_state(adev, adev->ualink.info, 471 cfg_state); 472 __amdgpu_ualink_activate_vpod_locked(adev); 473 } 474 475 r = amdgpu_ualink_drm_client_create(adev); 476 if (r) { 477 dev_err(adev->dev, "Failed to create UALink DRM client: %d\n", 478 r); 479 goto error; 480 } 481 482 /* Consider UALink initialized only at this stage */ 483 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_COMPLETE; 484 485 return 0; 486 487 error: 488 adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR; 489 return r; 490 } 491 492 /**************************************************************************** 493 * UALink info and configuration in sysfs 494 * 495 * Using kobj_attribute and not device_attribute here because the UALink 496 * attributes are part of their own nested kobjects and not the amdgpu 497 * device itself or a subdevice. 498 ****************************************************************************/ 499 500 /* UALink configuration APIs are being deprecated before upstreaming and are 501 * not enabled by default. They are replaced by a unified in-band/side-band 502 * approach through the IFoE driver. Querying of the current configuration 503 * will continue to be supported in sysfs. 504 * 505 * Uncomment the following line to enable the deprecated configuration APIs 506 * for testing purposes only. The code to support these APIs will be removed 507 * in a future patch. 508 */ 509 /* #define UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS */ 510 511 #define UALINK_VALUE_SHOW(prefix, name, field, format) \ 512 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 513 struct kobj_attribute *attr,\ 514 char *buf) \ 515 { \ 516 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 517 \ 518 return sysfs_emit(buf, format"\n", info->field); \ 519 } 520 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 521 #define UALINK_VALUE_STORE(prefix, name, field, type, base) \ 522 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 523 struct kobj_attribute *attr,\ 524 const char *buf, size_t count)\ 525 { \ 526 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 527 int r; \ 528 \ 529 r = kstrto##type(buf, base, &info->field); \ 530 if (r < 0) \ 531 return r; \ 532 return count; \ 533 } 534 #endif 535 536 static ssize_t show_idbits(unsigned long *bits, unsigned int nbits, char *buf) 537 { 538 ssize_t len = 0; 539 unsigned int id; 540 541 for_each_set_bit(id, bits, nbits) 542 len += sysfs_emit_at(buf, len, "%u ", id); 543 if (len) 544 len--; 545 return len + sysfs_emit_at(buf, len, "\n"); 546 } 547 #define UALINK_IDBITS_SHOW(prefix, name, field) \ 548 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 549 struct kobj_attribute *attr,\ 550 char *buf) \ 551 { \ 552 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 553 \ 554 return show_idbits(info->field, sizeof(info->field)*8, buf); \ 555 } 556 557 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 558 static ssize_t store_idbits(unsigned long *bits, unsigned int nbits, 559 const char *buf, size_t count) 560 { 561 char *dup_buf, *str, *tok; 562 ssize_t r; 563 u32 id; 564 565 bitmap_zero(bits, nbits); 566 if (!count) 567 return 0; 568 569 dup_buf = kstrndup(buf, count, GFP_KERNEL); 570 if (unlikely(!dup_buf)) 571 return -ENOMEM; 572 str = dup_buf; 573 do { 574 str += strspn(str, " "); 575 tok = strsep(&str, ", "); 576 r = kstrtou32(tok, 10, &id); 577 if (r < 0) 578 goto err; 579 if (id >= nbits) { 580 r = -ERANGE; 581 goto err; 582 } 583 set_bit(id, bits); 584 } while (str); 585 kfree(dup_buf); 586 587 return count; 588 589 err: 590 kfree(dup_buf); 591 return r; 592 } 593 #define UALINK_IDBITS_STORE(prefix, name, field, nbits) \ 594 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 595 struct kobj_attribute *attr,\ 596 const char *buf, size_t count)\ 597 { \ 598 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 599 DECLARE_BITMAP(bits, nbits); \ 600 ssize_t r = store_idbits(bits, nbits, buf, count); \ 601 \ 602 if (r >= 0) \ 603 bitmap_copy(info->field, bits, nbits); \ 604 return r; \ 605 } 606 #endif 607 608 static ssize_t show_idarray(const u32 *array, u32 size, char *buf) 609 { 610 ssize_t len = 0; 611 unsigned int i; 612 613 for (i = 0; i < size; i++) 614 len += sysfs_emit_at(buf, len, "%u ", array[i]); 615 if (len) 616 len--; 617 return len + sysfs_emit_at(buf, len, "\n"); 618 } 619 #define UALINK_IDARRAY_SHOW(prefix, name, field, size) \ 620 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 621 struct kobj_attribute *attr,\ 622 char *buf) \ 623 { \ 624 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 625 \ 626 return show_idarray(info->field, info->size, buf); \ 627 } 628 629 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 630 static ssize_t store_idarray(u32 *array, u32 *size, u32 max_id, u32 max_size, 631 const char *buf, size_t count) 632 { 633 char *dup_buf, *str, *tok; 634 unsigned int index = 0; 635 ssize_t r; 636 u32 id; 637 638 *size = 0; 639 if (!count) 640 return 0; 641 642 dup_buf = kstrndup(buf, count, GFP_KERNEL); 643 if (unlikely(!dup_buf)) 644 return -ENOMEM; 645 str = dup_buf; 646 do { 647 str += strspn(str, " "); 648 tok = strsep(&str, ", "); 649 r = kstrtou32(tok, 10, &id); 650 if (r < 0) 651 goto err; 652 if (index >= max_size) { 653 r = -ENOSPC; 654 goto err; 655 } 656 if (id >= max_id) { 657 r = -ERANGE; 658 goto err; 659 } 660 array[index++] = id; 661 } while (str); 662 kfree(dup_buf); 663 664 *size = index; 665 return count; 666 667 err: 668 kfree(dup_buf); 669 return r; 670 } 671 #define UALINK_IDARRAY_STORE(prefix, name, field, size, max_id) \ 672 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 673 struct kobj_attribute *attr,\ 674 const char *buf, size_t count)\ 675 { \ 676 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 677 u32 new_array[ARRAY_SIZE(info->field)]; \ 678 u32 new_size; \ 679 ssize_t r = store_idarray(new_array, &new_size, max_id, \ 680 ARRAY_SIZE(new_array), buf, count); \ 681 \ 682 if (r >= 0) { \ 683 memcpy(info->field, new_array, sizeof(new_array)); \ 684 info->size = new_size; \ 685 } \ 686 return r; \ 687 } 688 #endif 689 690 #define UALINK_UUID_SHOW(prefix, name, field) \ 691 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 692 struct kobj_attribute *attr,\ 693 char *buf) \ 694 { \ 695 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 696 \ 697 return sysfs_emit(buf, "%pU\n", &info->field); \ 698 } 699 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 700 #define UALINK_UUID_STORE(prefix, name, field) \ 701 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 702 struct kobj_attribute *attr,\ 703 const char *buf, size_t count)\ 704 { \ 705 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 706 int r; \ 707 \ 708 r = uuid_parse(buf, &info->field); \ 709 return r ? r : count; \ 710 } 711 #endif 712 713 static ssize_t show_enum(u32 x, const char * const values[], unsigned int n, 714 char *buf) 715 { 716 return sysfs_emit(buf, "%s\n", x < n && values[x] ? values[x] : "invalid"); 717 } 718 #define UALINK_ENUM_SHOW(prefix, name, field) \ 719 static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj, \ 720 struct kobj_attribute *attr,\ 721 char *buf) \ 722 { \ 723 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 724 \ 725 return show_enum(info->field, ualink_##name##_values, \ 726 ARRAY_SIZE(ualink_##name##_values), buf); \ 727 } 728 729 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 730 static ssize_t store_enum(u32 *x, const char * const values[], unsigned int n, 731 const char *buf, size_t count) 732 { 733 unsigned int i; 734 735 for (i = 0; i < n; i++) { 736 if (values[i] && sysfs_streq(buf, values[i])) { 737 *x = i; 738 return count; 739 } 740 } 741 return -EINVAL; 742 } 743 #define UALINK_ENUM_STORE(prefix, name, field) \ 744 static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj, \ 745 struct kobj_attribute *attr,\ 746 const char *buf, size_t count)\ 747 { \ 748 struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj); \ 749 \ 750 return store_enum(&info->field, ualink_##name##_values, \ 751 ARRAY_SIZE(ualink_##name##_values), \ 752 buf, count); \ 753 } 754 #endif 755 static const char * const ualink_link_type_values[] = { 756 "UALoE", "UALink" 757 }; 758 static const char * const ualink_addr_mode_values[] = { 759 "source-aliasing", "source-identification" 760 }; 761 static const char * const ualink_accel_state_values[] = { 762 [AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED] = "unconfigured", 763 [AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED] = "ppod_configured", 764 [AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED] = "vpod_configured", 765 [AMDGPU_UALINK_ACCEL_STATE_READY] = "ready", 766 [AMDGPU_UALINK_ACCEL_STATE_ACTIVE] = "active", 767 [AMDGPU_UALINK_ACCEL_STATE_ERROR] = "error", 768 }; 769 static_assert(ARRAY_SIZE(ualink_accel_state_values) == 770 AMDGPU_UALINK_ACCEL_STATE_MAX); 771 772 UALINK_ENUM_SHOW(info, link_type, link_type); 773 UALINK_VALUE_SHOW(info, accel_id, ppod.accel_id, "%u"); 774 UALINK_VALUE_SHOW(info, bandwidth, ppod.bandwidth, "%u"); 775 UALINK_VALUE_SHOW(info, latency, ppod.latency, "%u"); 776 UALINK_UUID_SHOW(info, ppod_id, ppod.id); 777 UALINK_VALUE_SHOW(info, ppod_size, ppod.size, "%u"); 778 UALINK_VALUE_SHOW(info, vpod_id, vpod.id, "%u"); 779 UALINK_VALUE_SHOW(info, vpod_size, vpod.size, "%u"); 780 UALINK_IDBITS_SHOW(info, vpod_active_accels, vpod.active_accel_bits); 781 UALINK_ENUM_SHOW(info, addr_mode, vpod.addr_mode); 782 UALINK_ENUM_SHOW(info, accel_state, accel_state); 783 UALINK_IDARRAY_SHOW(info, local_accels, local_accels, n_local_accels); 784 785 static struct amdgpu_device *ualink_xcp_kobj_to_adev(struct kobject *kobj) 786 { 787 struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp, 788 ualink.kobj); 789 790 return xcp->xcp_mgr->adev; 791 } 792 793 #define UALINK_XCP_INFO_SHOW(name) \ 794 static ssize_t ualink_xcp_info_##name##_show(struct kobject *kobj, \ 795 struct kobj_attribute *attr, char *buf) \ 796 { \ 797 struct amdgpu_device *adev = ualink_xcp_kobj_to_adev(kobj); \ 798 \ 799 return ualink_info_##name##_show(&adev->ualink.info->kobj, \ 800 attr, buf); \ 801 } 802 803 UALINK_XCP_INFO_SHOW(link_type) 804 UALINK_XCP_INFO_SHOW(accel_id) 805 UALINK_XCP_INFO_SHOW(bandwidth) 806 UALINK_XCP_INFO_SHOW(latency) 807 UALINK_XCP_INFO_SHOW(ppod_id) 808 UALINK_XCP_INFO_SHOW(ppod_size) 809 UALINK_XCP_INFO_SHOW(vpod_id) 810 UALINK_XCP_INFO_SHOW(vpod_size) 811 UALINK_XCP_INFO_SHOW(vpod_active_accels) 812 UALINK_XCP_INFO_SHOW(addr_mode) 813 UALINK_XCP_INFO_SHOW(accel_state) 814 UALINK_XCP_INFO_SHOW(local_accels) 815 816 #define UALINK_INFO_ATTR(name) __ATTR(name, 0444, ualink_info_##name##_show, NULL) 817 static struct kobj_attribute ualink_info_link_type = UALINK_INFO_ATTR(link_type); 818 static struct kobj_attribute ualink_info_accel_id = UALINK_INFO_ATTR(accel_id); 819 static struct kobj_attribute ualink_info_bandwidth = UALINK_INFO_ATTR(bandwidth); 820 static struct kobj_attribute ualink_info_latency = UALINK_INFO_ATTR(latency); 821 static struct kobj_attribute ualink_info_ppod_id = UALINK_INFO_ATTR(ppod_id); 822 static struct kobj_attribute ualink_info_ppod_size = UALINK_INFO_ATTR(ppod_size); 823 static struct kobj_attribute ualink_info_vpod_id = UALINK_INFO_ATTR(vpod_id); 824 static struct kobj_attribute ualink_info_vpod_size = UALINK_INFO_ATTR(vpod_size); 825 static struct kobj_attribute ualink_info_vpod_active_accels = UALINK_INFO_ATTR(vpod_active_accels); 826 static struct kobj_attribute ualink_info_addr_mode = UALINK_INFO_ATTR(addr_mode); 827 static struct kobj_attribute ualink_info_accel_state = UALINK_INFO_ATTR(accel_state); 828 static struct kobj_attribute ualink_info_local_accels = UALINK_INFO_ATTR(local_accels); 829 830 static const struct attribute *ualink_info_attrs[] = { 831 &ualink_info_link_type.attr, 832 &ualink_info_accel_id.attr, 833 &ualink_info_bandwidth.attr, 834 &ualink_info_latency.attr, 835 &ualink_info_ppod_id.attr, 836 &ualink_info_ppod_size.attr, 837 &ualink_info_vpod_id.attr, 838 &ualink_info_vpod_size.attr, 839 &ualink_info_vpod_active_accels.attr, 840 &ualink_info_addr_mode.attr, 841 &ualink_info_accel_state.attr, 842 &ualink_info_local_accels.attr, 843 NULL 844 }; 845 846 #define UALINK_XCP_INFO_ATTR(name) \ 847 __ATTR(name, 0444, ualink_xcp_info_##name##_show, NULL) 848 static struct kobj_attribute ualink_xcp_info_link_type = UALINK_XCP_INFO_ATTR(link_type); 849 static struct kobj_attribute ualink_xcp_info_accel_id = UALINK_XCP_INFO_ATTR(accel_id); 850 static struct kobj_attribute ualink_xcp_info_bandwidth = UALINK_XCP_INFO_ATTR(bandwidth); 851 static struct kobj_attribute ualink_xcp_info_latency = UALINK_XCP_INFO_ATTR(latency); 852 static struct kobj_attribute ualink_xcp_info_ppod_id = UALINK_XCP_INFO_ATTR(ppod_id); 853 static struct kobj_attribute ualink_xcp_info_ppod_size = UALINK_XCP_INFO_ATTR(ppod_size); 854 static struct kobj_attribute ualink_xcp_info_vpod_id = UALINK_XCP_INFO_ATTR(vpod_id); 855 static struct kobj_attribute ualink_xcp_info_vpod_size = UALINK_XCP_INFO_ATTR(vpod_size); 856 static struct kobj_attribute ualink_xcp_info_vpod_active_accels = UALINK_XCP_INFO_ATTR(vpod_active_accels); 857 static struct kobj_attribute ualink_xcp_info_addr_mode = UALINK_XCP_INFO_ATTR(addr_mode); 858 static struct kobj_attribute ualink_xcp_info_accel_state = UALINK_XCP_INFO_ATTR(accel_state); 859 static struct kobj_attribute ualink_xcp_info_local_accels = UALINK_XCP_INFO_ATTR(local_accels); 860 861 static struct attribute *ualink_xcp_info_attrs[] = { 862 &ualink_xcp_info_link_type.attr, 863 &ualink_xcp_info_accel_id.attr, 864 &ualink_xcp_info_bandwidth.attr, 865 &ualink_xcp_info_latency.attr, 866 &ualink_xcp_info_ppod_id.attr, 867 &ualink_xcp_info_ppod_size.attr, 868 &ualink_xcp_info_vpod_id.attr, 869 &ualink_xcp_info_vpod_size.attr, 870 &ualink_xcp_info_vpod_active_accels.attr, 871 &ualink_xcp_info_addr_mode.attr, 872 &ualink_xcp_info_accel_state.attr, 873 &ualink_xcp_info_local_accels.attr, 874 NULL 875 }; 876 877 static umode_t ualink_xcp_info_is_visible(struct kobject *kobj, 878 struct attribute *attr, int n) 879 { 880 struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp, 881 ualink.kobj); 882 883 if (!xcp->valid) 884 return 0; 885 886 return attr->mode; 887 } 888 889 static const struct attribute_group ualink_xcp_info_group = { 890 .attrs = ualink_xcp_info_attrs, 891 .is_visible = ualink_xcp_info_is_visible, 892 }; 893 894 static const struct kobj_type ualink_xcp_info_ktype = { 895 .sysfs_ops = &kobj_sysfs_ops 896 }; 897 898 static void ualink_info_release(struct kobject *kobj) 899 { 900 kfree(to_ualink_info(kobj)); 901 } 902 903 static const struct kobj_type ualink_info_ktype = { 904 .release = ualink_info_release, 905 .sysfs_ops = &kobj_sysfs_ops 906 }; 907 908 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 909 UALINK_VALUE_SHOW(ppod_setup, accel_id, ppod.accel_id, "%u"); 910 UALINK_VALUE_SHOW(ppod_setup, bandwidth, ppod.bandwidth, "%u"); 911 UALINK_VALUE_SHOW(ppod_setup, latency, ppod.latency, "%u"); 912 UALINK_UUID_SHOW(ppod_setup, ppod_id, ppod.id); 913 UALINK_VALUE_SHOW(ppod_setup, ppod_size, ppod.size, "%u"); 914 UALINK_IDARRAY_SHOW(ppod_setup, local_accels, local_accels, n_local_accels); 915 916 UALINK_VALUE_STORE(ppod_setup, accel_id, ppod.accel_id, u32, 10); 917 UALINK_VALUE_STORE(ppod_setup, bandwidth, ppod.bandwidth, u32, 10); 918 UALINK_VALUE_STORE(ppod_setup, latency, ppod.latency, u32, 10); 919 UALINK_UUID_STORE(ppod_setup, ppod_id, ppod.id); 920 UALINK_VALUE_STORE(ppod_setup, ppod_size, ppod.size, u32, 10); 921 UALINK_IDARRAY_STORE(ppod_setup, local_accels, local_accels, n_local_accels, 922 AMDGPU_UALINK_ACCEL_MAX); 923 924 static enum amdgpu_ualink_accel_state 925 check_ppod_state(struct amdgpu_device *adev, 926 const struct amdgpu_ualink_ppod_setup *setup) 927 { 928 if (setup->ppod.size <= 0 || setup->ppod.size > AMDGPU_UALINK_ACCEL_MAX) { 929 dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n", 930 setup->ppod.size, AMDGPU_UALINK_ACCEL_MAX); 931 return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 932 } 933 if (setup->ppod.accel_id >= setup->ppod.size) { 934 dev_dbg(adev->dev, "Accelerator ID %u greater or equal pPod size %u\n", 935 setup->ppod.accel_id, setup->ppod.size); 936 return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED; 937 } 938 return AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 939 } 940 941 static ssize_t ualink_ppod_setup_commit_store(struct kobject *kobj, 942 struct kobj_attribute *attr, 943 const char *buf, size_t count) 944 { 945 struct amdgpu_ualink_ppod_setup *setup = to_ualink_ppod_setup(kobj); 946 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 947 struct device *dev = kobj_to_dev(info->kobj.parent); 948 struct drm_device *ddev = dev_get_drvdata(dev); 949 struct amdgpu_device *adev = drm_to_adev(ddev); 950 int r; 951 952 if (!sysfs_streq(buf, "true")) 953 return -EINVAL; 954 955 r = psp_ual_set_ppod_config(&adev->psp, adev->ualink.psp_if_ver, 956 setup); 957 if (r) 958 return r; 959 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL); 960 if (r) 961 return r; 962 963 /* 964 * Hold mgpu_info.mutex to serialize with activate_local_vpod() in 965 * vpod_config_commit_store which also holds this lock. Without it, 966 * deactivate_accelerator() can race with activate_accelerator(), 967 * causing concurrent vm_fini / vm_init on the same NPA VM. 968 */ 969 mutex_lock(&mgpu_info.mutex); 970 if (info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_READY) 971 deactivate_accelerator(adev); 972 info->accel_state = check_ppod_state(adev, setup); 973 /* PPOD is expected to be configured first */ 974 info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID; 975 mutex_unlock(&mgpu_info.mutex); 976 977 /* TODO: If accel_state was ACTIVE, reset all connections */ 978 979 return count; 980 } 981 982 #define UALINK_PPOD_SETUP_ATTR(name) __ATTR(name, 0600, \ 983 ualink_ppod_setup_##name##_show, \ 984 ualink_ppod_setup_##name##_store) 985 static struct kobj_attribute ualink_ppod_setup_accel_id = UALINK_PPOD_SETUP_ATTR(accel_id); 986 static struct kobj_attribute ualink_ppod_setup_bandwidth = UALINK_PPOD_SETUP_ATTR(bandwidth); 987 static struct kobj_attribute ualink_ppod_setup_latency = UALINK_PPOD_SETUP_ATTR(latency); 988 static struct kobj_attribute ualink_ppod_setup_ppod_id = UALINK_PPOD_SETUP_ATTR(ppod_id); 989 static struct kobj_attribute ualink_ppod_setup_ppod_size = UALINK_PPOD_SETUP_ATTR(ppod_size); 990 static struct kobj_attribute ualink_ppod_setup_local_accels = UALINK_PPOD_SETUP_ATTR(local_accels); 991 static struct kobj_attribute ualink_ppod_setup_commit = __ATTR(commit, 0200, NULL, 992 ualink_ppod_setup_commit_store); 993 994 static const struct attribute *ualink_ppod_setup_attrs[] = { 995 &ualink_ppod_setup_accel_id.attr, 996 &ualink_ppod_setup_bandwidth.attr, 997 &ualink_ppod_setup_latency.attr, 998 &ualink_ppod_setup_ppod_id.attr, 999 &ualink_ppod_setup_ppod_size.attr, 1000 &ualink_ppod_setup_local_accels.attr, 1001 &ualink_ppod_setup_commit.attr, 1002 NULL 1003 }; 1004 #endif 1005 1006 static void ualink_ppod_setup_release(struct kobject *kobj) 1007 { 1008 kfree(to_ualink_ppod_setup(kobj)); 1009 } 1010 1011 static const struct kobj_type ualink_ppod_setup_ktype = { 1012 .release = ualink_ppod_setup_release, 1013 .sysfs_ops = &kobj_sysfs_ops 1014 }; 1015 1016 static struct amdgpu_device *find_peer_adev(unsigned int accel_id) 1017 { 1018 unsigned int i; 1019 1020 for (i = 0; i < mgpu_info.num_gpu; i++) { 1021 struct amdgpu_device *peer_adev = mgpu_info.gpu_ins[i].adev; 1022 1023 if (peer_adev->ualink.info && 1024 peer_adev->ualink.info->ppod.accel_id == accel_id) 1025 return peer_adev; 1026 } 1027 1028 return NULL; 1029 } 1030 1031 static bool amdgpu_ualink_is_local_accel(struct amdgpu_device *adev, 1032 u32 accel_id) 1033 { 1034 struct amdgpu_ualink_info *info = adev->ualink.info; 1035 unsigned int i; 1036 1037 if (!info) 1038 return false; 1039 1040 for (i = 0; i < info->n_local_accels; i++) { 1041 if (info->local_accels[i] == accel_id) 1042 return true; 1043 } 1044 return false; 1045 } 1046 1047 static void activate_accelerator(struct amdgpu_device *adev) 1048 { 1049 int r; 1050 1051 if (adev->ualink.info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 1052 return; 1053 1054 dev_info(adev->dev, 1055 "UALINK: activating accelerator accel_id=%u (accel_state=%d)\n", 1056 adev->ualink.info->ppod.accel_id, 1057 adev->ualink.info->accel_state); 1058 1059 /* Enable incoming NPA address translation with NPA VMID */ 1060 r = psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver, 1061 adev->vm_manager.npa_vmid, true); 1062 if (r) { 1063 dev_err(adev->dev, "Failed to set NPA config\n"); 1064 return; 1065 } 1066 1067 r = amdgpu_ualink_manager_start(adev); 1068 if (r) { 1069 dev_err(adev->dev, "Failed to start UALink manager\n"); 1070 return; 1071 } 1072 1073 r = amdgpu_ualink_sw_init(adev); 1074 if (r) { 1075 dev_err(adev->dev, "Failed to init UALink sw r=%d\n", r); 1076 amdgpu_ualink_manager_stop(adev); 1077 return; 1078 } 1079 1080 adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ACTIVE; 1081 dev_info(adev->dev, 1082 "UALINK: accelerator accel_id=%u is now ACTIVE\n", 1083 adev->ualink.info->ppod.accel_id); 1084 } 1085 1086 static void deactivate_accelerator(struct amdgpu_device *adev) 1087 { 1088 if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 1089 return; 1090 1091 dev_info(adev->dev, 1092 "UALINK: deactivating accelerator accel_id=%u (accel_state=%d -> PPOD_CONFIGURED)\n", 1093 adev->ualink.info->ppod.accel_id, 1094 adev->ualink.info->accel_state); 1095 1096 /* Disable incoming NPA address translation with NPA VMID */ 1097 psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver, 1098 adev->vm_manager.npa_vmid, false); 1099 /* ignore return value */ 1100 adev->ualink.info->accel_state = 1101 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED; 1102 1103 amdgpu_ualink_sw_fini(adev); 1104 amdgpu_ualink_manager_stop(adev); 1105 } 1106 1107 static void activate_local_vpod(struct amdgpu_device *adev) 1108 { 1109 struct amdgpu_ualink_info *info = adev->ualink.info; 1110 struct amdgpu_device *peer_adev; 1111 unsigned int accel_id; 1112 unsigned int i; 1113 1114 for (i = 0; i < info->n_local_accels; i++) { 1115 accel_id = info->local_accels[i]; 1116 1117 peer_adev = find_peer_adev(accel_id); 1118 if (WARN_ON(!peer_adev || !peer_adev->ualink.info)) 1119 /* info->local_accels is corrupted? */ 1120 continue; 1121 1122 /* Bring the peer up to match the current vPod membership. 1123 * 1124 * A peer that is not yet ACTIVE (or whose ACTIVE membership 1125 * changed on a grow/shrink) has its remote metadata, links and 1126 * GART mappings built for a stale member set (or none at all). 1127 * Firmware only accepts a full metadata reload while halted, so 1128 * surgical per-peer deltas are not possible: fully bounce the 1129 * accelerator. deactivate_accelerator() is a no-op when the peer 1130 * is not ACTIVE, so this handles first-time bring-up too. 1131 * Unchanged ACTIVE peers are left untouched. 1132 */ 1133 if (amdgpu_ualink_vpod_membership_changed(peer_adev)) { 1134 dev_info(peer_adev->dev, 1135 "UALINK: (re)configuring vpod for accel_id=%u\n", 1136 peer_adev->ualink.info->ppod.accel_id); 1137 deactivate_accelerator(peer_adev); 1138 activate_accelerator(peer_adev); 1139 } 1140 } 1141 } 1142 1143 static inline bool __is_vpod_peer(struct amdgpu_ualink_info *info, 1144 struct amdgpu_ualink_info *peer_info) 1145 { 1146 return peer_info->vpod.id == info->vpod.id && 1147 uuid_equal(&peer_info->ppod.id, &info->ppod.id); 1148 } 1149 1150 static int __check_local_vpod_integrity(struct amdgpu_device *adev) 1151 { 1152 DECLARE_BITMAP(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX); 1153 struct amdgpu_ualink_info *info = adev->ualink.info; 1154 u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX]; 1155 struct amdgpu_ualink_info *peer_info; 1156 struct amdgpu_device *peer_adev; 1157 unsigned int i, n_local_accels; 1158 unsigned int accel_id; 1159 /* Check that all local accelerators listed in vpod_active_accels have 1160 * matching pod IDs 1161 */ 1162 bitmap_zero(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX); 1163 n_local_accels = 0; 1164 __set_bit(info->ppod.accel_id, local_accel_ids); 1165 local_accels[n_local_accels++] = info->ppod.accel_id; 1166 1167 for (i = 0; i < mgpu_info.num_gpu; i++) { 1168 peer_adev = mgpu_info.gpu_ins[i].adev; 1169 if (peer_adev == adev || !peer_adev->ualink.info) 1170 continue; 1171 1172 peer_info = peer_adev->ualink.info; 1173 /* peer device ppod not configured */ 1174 if (peer_info->accel_state < 1175 AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 1176 dev_info(adev->dev, 1177 "UALINK: integrity EAGAIN: peer accel_id=%u not ppod-configured (accel_state=%d)\n", 1178 peer_info->ppod.accel_id, 1179 peer_info->accel_state); 1180 return -EAGAIN; 1181 } 1182 1183 accel_id = peer_info->ppod.accel_id; 1184 if (!test_bit(accel_id, info->vpod.active_accel_bits)) 1185 continue; 1186 /* peer device vpod not configured */ 1187 if (peer_info->accel_state < 1188 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) { 1189 dev_info(adev->dev, 1190 "UALINK: integrity EAGAIN: vpod peer accel_id=%u not vpod-configured (accel_state=%d)\n", 1191 accel_id, peer_info->accel_state); 1192 return -EAGAIN; 1193 } 1194 1195 if (!uuid_equal(&peer_info->ppod.id, &info->ppod.id)) { 1196 dev_warn(adev->dev, 1197 "UALINK: integrity fail: peer %u ppod_id doesn't match: %pU != %pU\n", 1198 peer_info->ppod.accel_id, &peer_info->ppod.id, 1199 &info->ppod.id); 1200 return -EINVAL; 1201 } 1202 1203 if (peer_info->ppod.size != info->ppod.size) { 1204 dev_warn(adev->dev, 1205 "UALINK: integrity fail: peer %u ppod_size doesn't match: %u != %u\n", 1206 accel_id, peer_info->ppod.size, 1207 info->ppod.size); 1208 return -EINVAL; 1209 } 1210 1211 if (peer_info->vpod.id != info->vpod.id) { 1212 dev_warn(adev->dev, 1213 "UALINK: integrity fail: peer %u vpod_id doesn't match: %u != %u\n", 1214 accel_id, peer_info->vpod.id, info->vpod.id); 1215 return -EINVAL; 1216 } 1217 if (peer_info->vpod.size != info->vpod.size) { 1218 dev_warn(adev->dev, 1219 "UALINK: integrity fail: peer %u vpod_size doesn't match: %u != %u\n", 1220 accel_id, peer_info->vpod.size, 1221 info->vpod.size); 1222 return -EINVAL; 1223 } 1224 if (peer_info->vpod.addr_mode != info->vpod.addr_mode) { 1225 dev_warn(adev->dev, 1226 "UALINK: integrity fail: peer %u addr_mode doesn't match: %u != %u\n", 1227 accel_id, peer_info->vpod.addr_mode, 1228 info->vpod.addr_mode); 1229 return -EINVAL; 1230 } 1231 if (!bitmap_equal(peer_info->vpod.active_accel_bits, 1232 info->vpod.active_accel_bits, 1233 AMDGPU_UALINK_ACCEL_MAX)) { 1234 dev_warn(adev->dev, 1235 "UALINK: integrity fail: peer %u vpod_active_accels don't match\n", 1236 accel_id); 1237 return -EINVAL; 1238 } 1239 1240 if (__test_and_set_bit(accel_id, local_accel_ids)) { 1241 dev_warn(adev->dev, 1242 "UALINK: integrity fail: duplicate accel_id %u among local vpod peers\n", 1243 accel_id); 1244 return -EINVAL; 1245 } 1246 local_accels[n_local_accels++] = accel_id; 1247 } 1248 1249 for (i = 0; i < mgpu_info.num_gpu; i++) { 1250 peer_adev = mgpu_info.gpu_ins[i].adev; 1251 peer_info = peer_adev->ualink.info; 1252 1253 if (!peer_info) 1254 continue; 1255 if (peer_adev != adev && !__is_vpod_peer(info, peer_info)) 1256 continue; 1257 1258 peer_info->n_local_accels = n_local_accels; 1259 memcpy(peer_info->local_accels, local_accels, 1260 sizeof(local_accels)); 1261 } 1262 1263 return 0; 1264 } 1265 1266 static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev) 1267 { 1268 int ret; 1269 1270 if (adev->ualink.info->accel_state < 1271 AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED || 1272 adev->ualink.info->accel_state == 1273 AMDGPU_UALINK_ACCEL_STATE_ERROR) 1274 return; 1275 1276 ret = __check_local_vpod_integrity(adev); 1277 if (ret && ret != -EAGAIN) { 1278 dev_err(adev->dev, 1279 "Local vpod integrity check failed: %d\n", ret); 1280 return; 1281 } 1282 if (ret == -EAGAIN) { 1283 dev_info(adev->dev, 1284 "UALINK: activate deferred, waiting for local vpod peers to reach VPOD_CONFIGURED\n"); 1285 return; 1286 } 1287 1288 dev_info(adev->dev, 1289 "UALINK: integrity OK, applying local vpod (%u local accels)\n", 1290 adev->ualink.info->n_local_accels); 1291 activate_local_vpod(adev); 1292 } 1293 1294 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1295 UALINK_VALUE_SHOW(vpod_config, vpod_id, vpod.id, "%u"); 1296 UALINK_VALUE_SHOW(vpod_config, vpod_size, vpod.size, "%u"); 1297 UALINK_IDBITS_SHOW(vpod_config, vpod_active_accels, vpod.active_accel_bits); 1298 UALINK_ENUM_SHOW(vpod_config, addr_mode, vpod.addr_mode); 1299 1300 UALINK_VALUE_STORE(vpod_config, vpod_id, vpod.id, u32, 10); 1301 UALINK_VALUE_STORE(vpod_config, vpod_size, vpod.size, u32, 10); 1302 UALINK_IDBITS_STORE(vpod_config, vpod_active_accels, vpod.active_accel_bits, 1303 AMDGPU_UALINK_ACCEL_MAX); 1304 UALINK_ENUM_STORE(vpod_config, addr_mode, vpod.addr_mode); 1305 1306 static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj, 1307 struct kobj_attribute *attr, 1308 const char *buf, size_t count) 1309 { 1310 struct amdgpu_ualink_vpod_config *config = to_ualink_vpod_config(kobj); 1311 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 1312 struct device *dev = kobj_to_dev(info->kobj.parent); 1313 struct drm_device *ddev = dev_get_drvdata(dev); 1314 struct amdgpu_device *adev = drm_to_adev(ddev); 1315 enum amdgpu_ualink_accel_state prev_state; 1316 u32 prev_vpod_id; 1317 int r; 1318 1319 if (!sysfs_streq(buf, "true")) 1320 return -EINVAL; 1321 1322 dev_dbg(adev->dev, 1323 "UALINK: vpod-commit enter: accel_state=%d staged vpod_id=%u vpod_size=%u addr_mode=%u\n", 1324 info->accel_state, config->vpod.id, config->vpod.size, 1325 config->vpod.addr_mode); 1326 1327 if (info->accel_state < AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) { 1328 dev_warn(adev->dev, 1329 "UALINK: vpod-commit rejected, ppod not configured (accel_state=%d)\n", 1330 info->accel_state); 1331 return -EINVAL; 1332 } 1333 1334 prev_state = info->accel_state; 1335 prev_vpod_id = info->vpod.id; 1336 r = psp_ual_set_vpod_config(&adev->psp, adev->ualink.psp_if_ver, 1337 config); 1338 if (r) { 1339 dev_warn(adev->dev, 1340 "UALINK: vpod-commit psp_ual_set_vpod_config failed: %d\n", 1341 r); 1342 return r; 1343 } 1344 1345 r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL); 1346 if (r) { 1347 dev_warn(adev->dev, 1348 "UALINK: vpod-commit psp_ual_query_info failed: %d\n", 1349 r); 1350 return r; 1351 } 1352 1353 dev_dbg(adev->dev, 1354 "UALINK: vpod-commit fw read-back: vpod_id=%u vpod_size=%u accel_id=%u ppod.size=%u addr_mode=%u\n", 1355 info->vpod.id, info->vpod.size, info->ppod.accel_id, 1356 info->ppod.size, info->vpod.addr_mode); 1357 1358 if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID && 1359 !__check_vpod_info(adev, info)) { 1360 dev_err(adev->dev, 1361 "UALINK: vpod-commit __check_vpod_info() failed\n"); 1362 return -EINVAL; 1363 } 1364 1365 /* The integrity check makes sure each new GPU is consistent with the 1366 * other GPUs already in the vPod. All known local GPUs can become 1367 * "ready" at the same time. 1368 * 1369 * Misconfiguration of one GPU does not reduce the state of other GPUs 1370 * already in the vPod. GPU is intentionally not put to ERROR state if 1371 * misconfiguration occurs. 1372 */ 1373 r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id, 1374 UAL_CFG_VPOD); 1375 if (r) 1376 return r; 1377 1378 return count; 1379 } 1380 1381 #define UALINK_VPOD_CONFIG_ATTR(name) __ATTR(name, 0600, \ 1382 ualink_vpod_config_##name##_show, \ 1383 ualink_vpod_config_##name##_store) 1384 static struct kobj_attribute ualink_vpod_config_vpod_id = UALINK_VPOD_CONFIG_ATTR(vpod_id); 1385 static struct kobj_attribute ualink_vpod_config_vpod_size = UALINK_VPOD_CONFIG_ATTR(vpod_size); 1386 static struct kobj_attribute ualink_vpod_config_vpod_active_accels = 1387 UALINK_VPOD_CONFIG_ATTR(vpod_active_accels); 1388 static struct kobj_attribute ualink_vpod_config_addr_mode = UALINK_VPOD_CONFIG_ATTR(addr_mode); 1389 static struct kobj_attribute ualink_vpod_config_commit = __ATTR(commit, 0200, NULL, 1390 ualink_vpod_config_commit_store); 1391 1392 static const struct attribute *ualink_vpod_config_attrs[] = { 1393 &ualink_vpod_config_vpod_id.attr, 1394 &ualink_vpod_config_vpod_size.attr, 1395 &ualink_vpod_config_vpod_active_accels.attr, 1396 &ualink_vpod_config_addr_mode.attr, 1397 &ualink_vpod_config_commit.attr, 1398 NULL 1399 }; 1400 #endif 1401 1402 static void ualink_vpod_config_release(struct kobject *kobj) 1403 { 1404 kfree(to_ualink_vpod_config(kobj)); 1405 } 1406 1407 static const struct kobj_type ualink_vpod_config_ktype = { 1408 .release = ualink_vpod_config_release, 1409 .sysfs_ops = &kobj_sysfs_ops 1410 }; 1411 1412 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1413 UALINK_VALUE_SHOW(station_config, flags, flags, "0x%x"); 1414 1415 UALINK_VALUE_STORE(station_config, flags, flags, u8, 16); 1416 1417 static size_t lane_bitmap_to_string(char *str, 1418 const struct amdgpu_ualink_station_config *stations) 1419 { 1420 size_t i; 1421 1422 for (i = 0; i < stations->n_stations; i++) 1423 str[i] = stations->lane_en_bitmap[i] + 1424 (stations->lane_en_bitmap[i] < 10 ? '0' : 'a' - 10); 1425 str[i] = '\0'; 1426 1427 return i; 1428 } 1429 static ssize_t 1430 ualink_station_config_lane_en_bitmap_show(struct kobject *kobj, 1431 struct kobj_attribute *attr, 1432 char *buf) 1433 { 1434 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1435 char str[AMDGPU_UALINK_STATIONS_MAX + 1]; 1436 1437 lane_bitmap_to_string(str, stations); 1438 return sysfs_emit(buf, "%s\n", str); 1439 } 1440 static ssize_t 1441 ualink_station_config_lane_en_bitmap_store(struct kobject *kobj, 1442 struct kobj_attribute *attr, 1443 const char *buf, size_t count) 1444 { 1445 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1446 u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX]; 1447 u32 n_stations = 0; 1448 bool end = false; 1449 size_t i; 1450 1451 for (i = 0; i < count; i++) { 1452 /* Accept any number of \n in the end */ 1453 if (buf[i] == '\n') { 1454 end = true; 1455 continue; 1456 } 1457 /* Don't accept any more data after \n */ 1458 if (end) 1459 return -EINVAL; 1460 /* Don't accept more data than the array size */ 1461 if (i >= sizeof(lane_en_bitmap)) 1462 return -ENOSPC; 1463 /* Accept hex digits */ 1464 if (buf[i] >= '0' && buf[i] <= '9') 1465 lane_en_bitmap[n_stations++] = buf[i] - '0'; 1466 else if (buf[i] >= 'a' && buf[i] <= 'f') 1467 lane_en_bitmap[n_stations++] = buf[i] + 10 - 'a'; 1468 else if (buf[i] >= 'A' && buf[i] <= 'F') 1469 lane_en_bitmap[n_stations++] = buf[i] + 10 - 'A'; 1470 else 1471 return -EINVAL; 1472 } 1473 1474 memcpy(stations->lane_en_bitmap, lane_en_bitmap, sizeof(lane_en_bitmap)); 1475 stations->n_stations = n_stations; 1476 return i; 1477 } 1478 1479 static ssize_t ualink_station_config_commit_store(struct kobject *kobj, 1480 struct kobj_attribute *attr, 1481 const char *buf, size_t count) 1482 { 1483 struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj); 1484 struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent); 1485 struct device *dev = kobj_to_dev(info->kobj.parent); 1486 struct drm_device *ddev = dev_get_drvdata(dev); 1487 struct amdgpu_device *adev = drm_to_adev(ddev); 1488 int r; 1489 1490 /* DF reconfiguration does not interact with accelerator state */ 1491 1492 if (!sysfs_streq(buf, "true")) 1493 return -EINVAL; 1494 1495 r = psp_ual_set_station_config(&adev->psp, adev->ualink.psp_if_ver, stations); 1496 if (r) 1497 return r; 1498 1499 return count; 1500 } 1501 1502 #define UALINK_STATION_CONFIG_ATTR(name) __ATTR(name, 0600, \ 1503 ualink_station_config_##name##_show, \ 1504 ualink_station_config_##name##_store) 1505 static struct kobj_attribute ualink_station_config_flags = UALINK_STATION_CONFIG_ATTR(flags); 1506 static struct kobj_attribute ualink_station_config_lane_en_bitmap = 1507 UALINK_STATION_CONFIG_ATTR(lane_en_bitmap); 1508 static struct kobj_attribute ualink_station_config_commit = 1509 __ATTR(commit, 0200, NULL, ualink_station_config_commit_store); 1510 1511 static const struct attribute *ualink_station_config_attrs[] = { 1512 &ualink_station_config_flags.attr, 1513 &ualink_station_config_lane_en_bitmap.attr, 1514 &ualink_station_config_commit.attr, 1515 NULL 1516 }; 1517 #endif 1518 1519 static void ualink_station_config_release(struct kobject *kobj) 1520 { 1521 kfree(to_ualink_station_config(kobj)); 1522 } 1523 1524 static const struct kobj_type ualink_station_config_ktype = { 1525 .release = ualink_station_config_release, 1526 .sysfs_ops = &kobj_sysfs_ops 1527 }; 1528 1529 int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block) 1530 { 1531 struct amdgpu_ualink_station_config *stations; 1532 struct amdgpu_ualink_vpod_config *vpod_config; 1533 struct amdgpu_ualink_ppod_setup *ppod_setup; 1534 struct amdgpu_device *adev = ip_block->adev; 1535 struct amdgpu_ualink_info *info; 1536 1537 info = kzalloc_obj(*info); 1538 ppod_setup = kzalloc_obj(*ppod_setup); 1539 vpod_config = kzalloc_obj(*vpod_config); 1540 stations = kzalloc_obj(*stations); 1541 if (!info || !ppod_setup || !vpod_config || !stations) { 1542 kfree(info); 1543 kfree(ppod_setup); 1544 kfree(vpod_config); 1545 kfree(stations); 1546 return -ENOMEM; 1547 } 1548 1549 info->ppod.accel_id = 0xffffffff; 1550 info->ppod.bandwidth = 0xffffffff; 1551 info->ppod.latency = 0xffffffff; 1552 info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID; 1553 info->vpod.addr_mode = AMDGPU_UALINK_ADDR_MODE_MAX; 1554 1555 /* 1556 * Initialize the kobjects here so their lifetime is tied to the UALink 1557 * manager software state. amdgpu_ualink_object_fini() drops the final 1558 * reference via kobject_put(). 1559 */ 1560 kobject_init(&info->kobj, &ualink_info_ktype); 1561 kobject_init(&ppod_setup->kobj, &ualink_ppod_setup_ktype); 1562 kobject_init(&vpod_config->kobj, &ualink_vpod_config_ktype); 1563 kobject_init(&stations->kobj, &ualink_station_config_ktype); 1564 1565 adev->ualink.info = info; 1566 adev->ualink.setup = ppod_setup; 1567 adev->ualink.config = vpod_config; 1568 adev->ualink.stations = stations; 1569 1570 return 0; 1571 } 1572 1573 int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block) 1574 { 1575 amdgpu_ualink_object_fini(ip_block->adev); 1576 return 0; 1577 } 1578 1579 static int ualink_kobj_add(struct kobject *kobj, struct kobject *parent, 1580 const char *name, const struct attribute **attrs) 1581 { 1582 int r; 1583 1584 r = kobject_add(kobj, parent, "%s", name); 1585 if (r) 1586 return r; 1587 r = sysfs_create_files(kobj, attrs); 1588 if (r) 1589 kobject_del(kobj); 1590 1591 return r; 1592 } 1593 1594 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev); 1595 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev); 1596 1597 int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev) 1598 { 1599 struct amdgpu_ualink_info *info = adev->ualink.info; 1600 int r; 1601 1602 if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) 1603 return 0; 1604 1605 /* ualink parent node */ 1606 r = ualink_kobj_add(&info->kobj, &adev->dev->kobj, "ualink", 1607 ualink_info_attrs); 1608 if (r) 1609 goto err; 1610 1611 #ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS 1612 r = ualink_kobj_add(&adev->ualink.setup->kobj, &info->kobj, "setup", 1613 ualink_ppod_setup_attrs); 1614 if (r) 1615 goto err_info; 1616 1617 r = ualink_kobj_add(&adev->ualink.config->kobj, &info->kobj, "config", 1618 ualink_vpod_config_attrs); 1619 if (r) 1620 goto err_setup; 1621 1622 r = ualink_kobj_add(&adev->ualink.stations->kobj, &info->kobj, 1623 "stations", ualink_station_config_attrs); 1624 if (r) 1625 goto err_config; 1626 #else 1627 r = kobject_add(&adev->ualink.setup->kobj, &info->kobj, "%s", 1628 "setup"); 1629 if (r) 1630 goto err_info; 1631 1632 r = kobject_add(&adev->ualink.config->kobj, &info->kobj, "%s", 1633 "config"); 1634 if (r) 1635 goto err_setup; 1636 1637 r = kobject_add(&adev->ualink.stations->kobj, &info->kobj, "%s", 1638 "stations"); 1639 if (r) 1640 goto err_config; 1641 #endif 1642 1643 amdgpu_ualink_xcp_sysfs_init(adev); 1644 adev->ualink.sysfs_init = true; 1645 1646 return 0; 1647 1648 err_config: 1649 kobject_del(&adev->ualink.config->kobj); 1650 err_setup: 1651 kobject_del(&adev->ualink.setup->kobj); 1652 err_info: 1653 kobject_del(&info->kobj); 1654 err: 1655 dev_warn(adev->dev, "Failed to create UALink sysfs: %d\n", r); 1656 return 0; 1657 } 1658 1659 void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev) 1660 { 1661 if (!adev->ualink.sysfs_init) 1662 return; 1663 1664 amdgpu_ualink_xcp_sysfs_fini(adev); 1665 kobject_del(&adev->ualink.stations->kobj); 1666 kobject_del(&adev->ualink.config->kobj); 1667 kobject_del(&adev->ualink.setup->kobj); 1668 kobject_del(&adev->ualink.info->kobj); 1669 adev->ualink.sysfs_init = false; 1670 } 1671 1672 static int amdgpu_ualink_xcp_sysfs_add(struct amdgpu_xcp *xcp) 1673 { 1674 struct amdgpu_device *adev = xcp->xcp_mgr->adev; 1675 int r; 1676 1677 r = kobject_init_and_add(&xcp->ualink.kobj, &ualink_xcp_info_ktype, 1678 &xcp->ddev->dev->kobj, "ualink"); 1679 if (r) 1680 goto err; 1681 1682 r = sysfs_create_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1683 if (r) 1684 goto err; 1685 1686 /* pin info so it outlives this node regardless of teardown order */ 1687 kobject_get(&adev->ualink.info->kobj); 1688 xcp->ualink.sysfs = true; 1689 return 0; 1690 err: 1691 kobject_put(&xcp->ualink.kobj); 1692 return r; 1693 } 1694 1695 void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp) 1696 { 1697 if (!xcp->ualink.sysfs) 1698 return; 1699 1700 sysfs_update_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1701 } 1702 1703 static void amdgpu_ualink_xcp_sysfs_remove(struct amdgpu_xcp *xcp) 1704 { 1705 struct amdgpu_device *adev = xcp->xcp_mgr->adev; 1706 1707 if (!xcp->ualink.sysfs) 1708 return; 1709 1710 /* group is only populated for valid partitions */ 1711 if (xcp->valid) 1712 sysfs_remove_group(&xcp->ualink.kobj, &ualink_xcp_info_group); 1713 kobject_put(&adev->ualink.info->kobj); 1714 kobject_put(&xcp->ualink.kobj); 1715 xcp->ualink.sysfs = false; 1716 } 1717 1718 static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev) 1719 { 1720 struct amdgpu_xcp *xcp; 1721 int i; 1722 1723 if (!adev->xcp_mgr) 1724 return; 1725 1726 for (i = 0; i < MAX_XCP; i++) { 1727 xcp = &adev->xcp_mgr->xcp[i]; 1728 if (!xcp->ddev || amdgpu_xcp_is_primary(xcp)) 1729 continue; 1730 amdgpu_ualink_xcp_sysfs_add(xcp); 1731 } 1732 } 1733 1734 static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev) 1735 { 1736 struct amdgpu_xcp *xcp; 1737 int i; 1738 1739 if (!adev->xcp_mgr) 1740 return; 1741 1742 for (i = 0; i < MAX_XCP; i++) { 1743 xcp = &adev->xcp_mgr->xcp[i]; 1744 amdgpu_ualink_xcp_sysfs_remove(xcp); 1745 } 1746 } 1747 1748 static int amdgpu_ualink_npa_alloc_va(struct amdgpu_device *adev, 1749 struct drm_mm_node *mm_node, 1750 u64 va, u64 range_start, 1751 u64 range_end, int size) 1752 { 1753 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1754 u64 alignment = 0; 1755 int ret; 1756 1757 mutex_lock(&npa_mm->mm_lock); 1758 if (va) { 1759 mm_node->start = va; 1760 mm_node->size = size; 1761 ret = drm_mm_reserve_node(&npa_mm->mm, mm_node); 1762 } else { 1763 if (!range_start && !range_end) { 1764 range_start = npa_mm->va_start; 1765 range_end = npa_mm->va_start + npa_mm->va_size; 1766 } 1767 1768 /* If size is greater than 2MB (or 512 pages), 1769 * align it to 2MB (or 512 pages) granularity. 1770 */ 1771 if (size >= 0x200) 1772 alignment = 0x200; 1773 1774 ret = drm_mm_insert_node_in_range(&npa_mm->mm, mm_node, size, 1775 alignment, 0, range_start, 1776 range_end, 0); 1777 } 1778 mutex_unlock(&npa_mm->mm_lock); 1779 1780 if (ret) 1781 dev_err(adev->dev, "Failed to allocate NPA address\n"); 1782 1783 return ret; 1784 } 1785 1786 static void amdgpu_ualink_npa_free_va(struct amdgpu_device *adev, 1787 struct drm_mm_node *mm_node) 1788 { 1789 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1790 1791 mutex_lock(&npa_mm->mm_lock); 1792 drm_mm_remove_node(mm_node); 1793 mutex_unlock(&npa_mm->mm_lock); 1794 } 1795 1796 static void amdgpu_ualink_npa_mm_init(struct amdgpu_device *adev) 1797 { 1798 struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm; 1799 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 1800 u64 va_size; 1801 1802 mutex_init(&npa_mm->mm_lock); 1803 1804 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 1805 /* 51 bit address space in Source-Identification mode. 1806 * 39 bits in page granularity. 1807 */ 1808 npa_mm->va_start = 0; 1809 va_size = GENMASK_ULL(38, 0); 1810 } else { 1811 /* 41 bit address space in Source-Alias mode. 1812 * 29 bits in page granularity. 1813 */ 1814 /* First 8MB is reserved for Metadata NPAs */ 1815 npa_mm->va_start = 0x800; 1816 va_size = GENMASK_ULL(28, 0); 1817 } 1818 1819 npa_mm->va_size = va_size - npa_mm->va_start; 1820 drm_mm_init(&npa_mm->mm, npa_mm->va_start, npa_mm->va_size); 1821 } 1822 1823 static void amdgpu_ualink_npa_mm_fini(struct amdgpu_device *adev) 1824 { 1825 mutex_destroy(&adev->ualink.npa_mm.mm_lock); 1826 drm_mm_takedown(&adev->ualink.npa_mm.mm); 1827 } 1828 1829 /* The caller of this function is expected to hold the XA lock when calling 1830 * this function. 1831 */ 1832 static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev, 1833 struct amdgpu_ualink_handle *handle) 1834 { 1835 bool unique = false; 1836 1837 do { 1838 handle->handle_lo = get_random_u64(); 1839 /* Replace bottom 10 bits in handle_lo with accId */ 1840 handle->handle_lo &= ~AMDGPU_UALINK_HANDLE_ACCID_MASK; 1841 handle->handle_lo |= adev->ualink.info->ppod.accel_id; 1842 1843 /* Don't generate/store a Handle with value 0. */ 1844 if (!handle->handle_lo) 1845 continue; 1846 1847 /* Find if the handle already exists in the exporter xarray. 1848 * If it already exists, then regenerate the handle since we 1849 * want the handle to be unique. 1850 */ 1851 unique = !xa_load(&adev->ualink.exp_xa, handle->handle_lo); 1852 } while (!unique); 1853 1854 handle->handle_hi = get_random_u64(); 1855 dev_dbg(adev->dev, "GENERATE-HANDLE: generated handle: %llx:%llx\n", 1856 handle->handle_hi, handle->handle_lo); 1857 } 1858 1859 static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref) 1860 { 1861 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 1862 struct amdgpu_device *adev; 1863 1864 exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node, 1865 refcount); 1866 adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev); 1867 queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work); 1868 } 1869 1870 static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref) 1871 { 1872 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 1873 struct amdgpu_device *adev; 1874 1875 imp_xa_node = container_of(ref, struct amdgpu_ualink_imp_xa_node, 1876 refcount); 1877 adev = imp_xa_node->adev; 1878 1879 dev_dbg(adev->dev, "IMP-CLEANUP: Remove entry from XA for handle:%llx:%llx\n", 1880 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1881 1882 /* Remove node from the Xarray */ 1883 if (!xa_erase(&adev->ualink.imp_xa, imp_xa_node->handle.handle_lo)) 1884 dev_err(adev->dev, "IMP-CLEANUP: Failed to find entry in XA for handle:%llx:%llx\n", 1885 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1886 1887 if (imp_xa_node->npa_addr && imp_xa_node->size && imp_xa_node->dmabuf) { 1888 dev_dbg(adev->dev, 1889 "IMP-CLEANUP: dmabuf free for NPA:%llx handle:%llx:%llx, fc:%lu\n", 1890 imp_xa_node->npa_addr, imp_xa_node->handle.handle_hi, 1891 imp_xa_node->handle.handle_lo, 1892 file_count(imp_xa_node->dmabuf->file)); 1893 dma_buf_put(imp_xa_node->dmabuf); 1894 drm_gem_handle_delete(adev->ualink.client.file, 1895 imp_xa_node->gem_handle); 1896 1897 } 1898 1899 dev_dbg(adev->dev, "IMP-CLEANUP: Freeing XA entry for handle:%llx:%llx\n", 1900 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo); 1901 1902 kfree(imp_xa_node); 1903 } 1904 1905 static int amdgpu_ualink_exp_xa_entry_get(struct amdgpu_ualink_exp_xa_node *exp_xa_node) 1906 { 1907 return kref_get_unless_zero(&exp_xa_node->refcount); 1908 } 1909 1910 static void amdgpu_ualink_exp_xa_entry_put(struct amdgpu_ualink_exp_xa_node *exp_xa_node) 1911 { 1912 kref_put(&exp_xa_node->refcount, amdgpu_ualink_cleanup_exp_xa_node); 1913 } 1914 1915 static int amdgpu_ualink_imp_xa_entry_get(struct amdgpu_ualink_imp_xa_node *imp_xa_node) 1916 { 1917 return kref_get_unless_zero(&imp_xa_node->refcount); 1918 } 1919 1920 static void amdgpu_ualink_imp_xa_entry_put(struct amdgpu_ualink_imp_xa_node *imp_xa_node) 1921 { 1922 kref_put(&imp_xa_node->refcount, amdgpu_ualink_cleanup_imp_xa_node); 1923 } 1924 1925 static int amdgpu_ualink_send_npa_release_msg(struct amdgpu_device *adev, 1926 u32 remote_acc_id, 1927 struct amdgpu_ualink_handle handle) 1928 { 1929 u32 dw0, dw1, dw2, dw3; 1930 1931 dw0 = lower_32_bits(handle.handle_lo); 1932 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1933 dw0 |= AMDGPU_UALINK_NPA_RELEASE_MSG; 1934 1935 dw1 = upper_32_bits(handle.handle_lo); 1936 dw2 = lower_32_bits(handle.handle_hi); 1937 dw3 = upper_32_bits(handle.handle_hi); 1938 1939 dev_dbg(adev->dev, "SEND NPA-RELEASE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1940 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1941 1942 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1943 dw2, dw3); 1944 } 1945 1946 static int amdgpu_ualink_send_npa_revoke_msg(struct amdgpu_device *adev, 1947 u32 remote_acc_id, 1948 struct amdgpu_ualink_handle handle) 1949 { 1950 u32 dw0, dw1, dw2, dw3; 1951 1952 dw0 = lower_32_bits(handle.handle_lo); 1953 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1954 dw0 |= AMDGPU_UALINK_NPA_REVOKE_MSG; 1955 1956 dw1 = upper_32_bits(handle.handle_lo); 1957 dw2 = lower_32_bits(handle.handle_hi); 1958 dw3 = upper_32_bits(handle.handle_hi); 1959 1960 dev_dbg(adev->dev, "SEND NPA-REVOKE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1961 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1962 1963 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1964 dw2, dw3); 1965 } 1966 1967 static int amdgpu_ualink_send_npa_fail_msg(struct amdgpu_device *adev, 1968 u32 remote_acc_id, 1969 struct amdgpu_ualink_handle handle, 1970 u32 fail_reason) 1971 { 1972 u32 dw0, dw1, dw2, dw3; 1973 1974 dw0 = lower_32_bits(handle.handle_lo); 1975 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1976 dw0 |= AMDGPU_UALINK_NPA_FAIL_MSG; 1977 1978 dw1 = upper_32_bits(handle.handle_lo); 1979 dw2 = fail_reason & 0xFF; 1980 dw3 = 0; 1981 1982 dev_dbg(adev->dev, "SEND NPA-FAIL: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 1983 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 1984 1985 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 1986 dw2, dw3); 1987 } 1988 1989 static int amdgpu_ualink_send_npa_rsp_msg(struct amdgpu_device *adev, 1990 u32 remote_acc_id, 1991 struct amdgpu_ualink_handle handle, 1992 u32 npa_addr, u32 size) 1993 { 1994 u32 dw0, dw1, dw2, dw3; 1995 1996 dw0 = lower_32_bits(handle.handle_lo); 1997 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 1998 dw0 |= AMDGPU_UALINK_NPA_RSP_MSG; 1999 2000 dw1 = upper_32_bits(handle.handle_lo); 2001 dw2 = size; 2002 dw3 = npa_addr; 2003 2004 dev_dbg(adev->dev, "SEND NPA-RSP: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 2005 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 2006 2007 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 2008 dw2, dw3); 2009 } 2010 2011 static int amdgpu_ualink_send_npa_req_msg(struct amdgpu_device *adev, 2012 u32 remote_acc_id, 2013 struct amdgpu_ualink_handle handle) 2014 { 2015 u32 dw0, dw1, dw2, dw3; 2016 2017 dw0 = lower_32_bits(handle.handle_lo); 2018 dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK; 2019 dw0 |= AMDGPU_UALINK_NPA_REQ_MSG; 2020 2021 dw1 = upper_32_bits(handle.handle_lo); 2022 dw2 = lower_32_bits(handle.handle_hi); 2023 dw3 = upper_32_bits(handle.handle_hi); 2024 2025 dev_dbg(adev->dev, "SEND NPA-REQ: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 2026 remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3); 2027 2028 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1, 2029 dw2, dw3); 2030 } 2031 2032 static u64 amdgpu_ualink_get_export_pte_flags(struct amdgpu_device *adev, 2033 struct amdgpu_bo *bo, 2034 u64 mapping_flags) 2035 { 2036 u64 pte_flags = adev->gmc.init_pte_flags; 2037 2038 pte_flags |= (AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE | 2039 AMDGPU_PTE_WRITEABLE); 2040 mapping_flags |= AMDGPU_VM_MTYPE_DEFAULT; 2041 2042 amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, bo, mapping_flags, 2043 &pte_flags); 2044 2045 return pte_flags; 2046 } 2047 2048 static int amdgpu_ualink_unmap_npa_addr(struct amdgpu_device *adev, 2049 struct amdgpu_bo *bo, 2050 u64 npa_addr, u64 size) 2051 { 2052 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2053 uint64_t pte_value = adev->gmc.noretry_flags; 2054 struct amdgpu_bo *bos[] = { bo }; 2055 struct dma_fence *fence = NULL; 2056 struct drm_exec exec; 2057 int r; 2058 2059 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false); 2060 if (unlikely(r)) { 2061 dev_err(adev->dev, "Failed to reserve VM and BO in unmap_npa_addr\n"); 2062 return r; 2063 } 2064 2065 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2066 false, NULL, npa_addr, npa_addr + size - 1, 2067 pte_value, 0, 0, NULL, NULL, &vm->last_update); 2068 if (r) { 2069 dev_err(adev->dev, 2070 "Failed to unmap NPA addr (%llx) from NPA VM\n", npa_addr); 2071 goto out; 2072 } 2073 2074 r = amdgpu_vm_update_pdes(adev, vm, false); 2075 if (r) { 2076 dev_err(adev->dev, 2077 "Failed %d to update page directories during unmapping NPA: 0x%llx\n", 2078 r, npa_addr); 2079 goto out; 2080 } 2081 2082 fence = dma_fence_get(vm->last_update); 2083 if (fence) { 2084 r = dma_fence_wait(fence, false); 2085 dma_fence_put(fence); 2086 fence = NULL; 2087 if (r) { 2088 dev_dbg(adev->dev, 2089 "UNMAP-NPA: dma fence wait failed, error: %d\n", r); 2090 goto out; 2091 } 2092 } 2093 2094 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2095 out: 2096 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2097 2098 return r; 2099 } 2100 2101 static int amdgpu_ualink_map_npa_addr(struct amdgpu_device *adev, u64 npa_addr, 2102 u64 size, struct amdgpu_bo *bo, u64 offset, 2103 u64 pte_flags) 2104 { 2105 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2106 struct amdgpu_bo *bos[] = { bo }; 2107 struct dma_fence *fence = NULL; 2108 struct drm_exec exec; 2109 int r; 2110 2111 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false); 2112 if (unlikely(r)) { 2113 dev_err(adev->dev, "Failed to reserve VM and BO in map_npa_addr\n"); 2114 return r; 2115 } 2116 2117 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2118 false, NULL, npa_addr, npa_addr + size - 1, 2119 pte_flags, offset, adev->vm_manager.vram_base_offset, 2120 bo->tbo.resource, NULL, &vm->last_update); 2121 if (r) { 2122 dev_warn(adev->dev, 2123 "Failed to map NPA addr (%llx) into NPA VM\n", npa_addr); 2124 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2125 goto out; 2126 } 2127 2128 r = amdgpu_vm_update_pdes(adev, vm, false); 2129 if (r) { 2130 dev_err(adev->dev, 2131 "failed %d to update page directories for NPA: 0x%llx\n", 2132 r, npa_addr); 2133 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2134 goto unmap_npa; 2135 } 2136 2137 fence = dma_fence_get(vm->last_update); 2138 if (fence) { 2139 r = dma_fence_wait(fence, false); 2140 dma_fence_put(fence); 2141 fence = NULL; 2142 if (r) { 2143 pr_debug("failed %d to dma fence wait\n", r); 2144 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2145 goto unmap_npa; 2146 } 2147 } 2148 2149 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2150 2151 /* TLB flush may be needed after updated page directories */ 2152 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2153 2154 return 0; 2155 2156 unmap_npa: 2157 amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size); 2158 out: 2159 return r; 2160 } 2161 2162 static int amdgpu_ualink_send_hello_ack_msg(struct amdgpu_device *adev, 2163 u32 remote_acc_id) 2164 { 2165 dev_dbg(adev->dev, "SEND HELLO-ACK: HELLO-ACK message to remote AccId:%u\n", 2166 remote_acc_id); 2167 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, 2168 AMDGPU_UALINK_HELLO_ACK_MSG, 2169 0, 0, 0); 2170 } 2171 2172 static int amdgpu_ualink_send_hello_msg(struct amdgpu_device *adev, 2173 u32 remote_acc_id) 2174 { 2175 u32 dw0; 2176 2177 dw0 = AMDGPU_UALINK_HELLO_MSG; 2178 dw0 |= (remote_acc_id << AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT); 2179 dw0 |= (adev->ualink.info->ppod.accel_id << 2180 AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT); 2181 2182 dev_dbg(adev->dev, "SEND HELLO: HELLO message to remote AccId:%u\n", 2183 remote_acc_id); 2184 return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, 0, 2185 0, 0); 2186 } 2187 2188 static u32 amdgpu_ualink_check_conn_ready(struct amdgpu_device *adev, 2189 u32 remote_acc_id, u32 gen_count) 2190 { 2191 struct amdgpu_ualink_connection *conn_state; 2192 u32 current_gen_count = 0; 2193 2194 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2195 2196 /* Check if the connection is established. */ 2197 mutex_lock(&conn_state->lock); 2198 if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) && 2199 (!gen_count || conn_state->generation_count == gen_count)) 2200 current_gen_count = conn_state->generation_count; 2201 mutex_unlock(&conn_state->lock); 2202 2203 return current_gen_count; 2204 } 2205 2206 static void amdgpu_ualink_process_hello_ack_msg(struct amdgpu_device *adev, 2207 u32 sender_acc_id) 2208 { 2209 struct amdgpu_ualink_connection *conn_state; 2210 2211 if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 2212 dev_err(adev->dev, 2213 "HELLO-ACK: sender AccId out of range:%u\n", 2214 sender_acc_id); 2215 return; 2216 } 2217 2218 conn_state = &adev->ualink.conn_state[sender_acc_id]; 2219 2220 /* If we are in IN_PROGRESS state, then transition the connection 2221 * state to established and signal that the HELLO ACK is received. 2222 * Otherwise, ignore the HELLO ACK. 2223 */ 2224 mutex_lock(&conn_state->lock); 2225 2226 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) { 2227 conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED; 2228 conn_state->generation_count++; 2229 complete(&conn_state->hello_done); 2230 } else { 2231 dev_dbg(adev->dev, 2232 "HELLO-ACK: already connected, ignoring from AccId:%u\n", 2233 sender_acc_id); 2234 } 2235 mutex_unlock(&conn_state->lock); 2236 } 2237 2238 static void amdgpu_ualink_process_hello_msg(struct amdgpu_device *adev, 2239 u32 receiver_acc_id, 2240 u32 sender_acc_id, 2241 u32 src_acc_id) 2242 { 2243 struct amdgpu_ualink_connection *conn_state; 2244 u32 generation_count; 2245 int r; 2246 2247 if (receiver_acc_id != adev->ualink.info->ppod.accel_id) { 2248 dev_err(adev->dev, 2249 "HELLO: receiver AccId mismatch got:%u self:%u\n", 2250 receiver_acc_id, adev->ualink.info->ppod.accel_id); 2251 return; 2252 } 2253 2254 /* src_acc_id is the AccId received in IH cookie. Confirm it 2255 * matches with the sender AccId. 2256 */ 2257 if (sender_acc_id != src_acc_id) { 2258 dev_err(adev->dev, 2259 "HELLO: sender AccId mismatch sender:%u IH cookie:%u\n", 2260 sender_acc_id, src_acc_id); 2261 return; 2262 } 2263 2264 if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 2265 dev_err(adev->dev, 2266 "HELLO: sender AccId out of range:%u\n", 2267 sender_acc_id); 2268 return; 2269 } 2270 2271 conn_state = &adev->ualink.conn_state[sender_acc_id]; 2272 2273 /* Check if connection is already established. If yes, then receiving HELLO msg 2274 * triggers a reset handling scenario. 2275 * If the connection is not ready, and we receive a HELLO msg, then 2276 * transition the state to PENDING. 2277 * Otherwise, leave it IN_PROGRESS. 2278 */ 2279 mutex_lock(&conn_state->lock); 2280 if (conn_state->state != AMDGPU_UALINK_CONN_ESTABLISHED) { 2281 if (conn_state->state == AMDGPU_UALINK_CONN_NOT_READY) 2282 conn_state->state = AMDGPU_UALINK_CONN_PENDING; 2283 /* otherwise, leave it IN_PROGRESS to signal the completion below */ 2284 mutex_unlock(&conn_state->lock); 2285 } else { 2286 /* Leave state ESTABLISHED so handle_connection_reset() 2287 * performs the transition and imp/exp XA cleanup. 2288 */ 2289 generation_count = conn_state->generation_count; 2290 mutex_unlock(&conn_state->lock); 2291 2292 amdgpu_ualink_handle_connection_reset(adev, sender_acc_id, 2293 AMDGPU_UALINK_CONN_PENDING, 2294 generation_count); 2295 } 2296 2297 /* Peer may have rebooted. */ 2298 r = amdgpu_ualink_ring_resync_ptrs(adev, sender_acc_id); 2299 if (r < 0) { 2300 dev_err(adev->dev, 2301 "HELLO: resync sender AccId %u ptrs failed %d\n", 2302 sender_acc_id, r); 2303 goto out_set_state; 2304 } 2305 if (r) 2306 amdgpu_ualink_reset_peer_rings(adev, sender_acc_id); 2307 2308 r = amdgpu_ualink_send_hello_ack_msg(adev, sender_acc_id); 2309 if (r) 2310 dev_err(adev->dev, "HELLO-ACK: send failed to remote AccId:%u\n", 2311 sender_acc_id); 2312 2313 out_set_state: 2314 mutex_lock(&conn_state->lock); 2315 if (r) { 2316 conn_state->state = AMDGPU_UALINK_CONN_NOT_READY; 2317 } else { 2318 /* If we are in IN_PROGRESS state and we receivied the HELLO message, 2319 * upon receiving the HELLO message, transition the state to ESTABLISHED 2320 * and signal the completion. 2321 */ 2322 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) 2323 complete(&conn_state->hello_done); 2324 conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED; 2325 conn_state->generation_count++; 2326 } 2327 mutex_unlock(&conn_state->lock); 2328 } 2329 2330 /* True if nothing is exported to, or imported from, this remote accelerator. */ 2331 static bool amdgpu_ualink_peer_idle(struct amdgpu_device *adev, u32 remote_accel_id) 2332 { 2333 /* Unlocked: a racing import at worst costs one redundant ring resync. */ 2334 return list_empty(&adev->ualink.imp_handles_list[remote_accel_id]) && 2335 list_empty(&adev->ualink.exp_handles_list[remote_accel_id]); 2336 } 2337 2338 static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev, 2339 u32 remote_acc_id) 2340 { 2341 struct amdgpu_ualink_connection *conn_state; 2342 int r; 2343 2344 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2345 2346 /* Connection state management goes through different states. 2347 * The states are: 2348 * - NOT_READY: The connection is not ready. 2349 * - IN_PROGRESS: GPU sent HELLO message and is waiting for the HELLO_ACK. 2350 * - PENDING: GPU received HELLO message and is in the process of sending 2351 * the HELLO_ACK. 2352 * - ESTABLISHED: The connection is established. 2353 */ 2354 2355 /* Grab the lock and check if connection establishment was 2356 * already done by another thread. 2357 */ 2358 mutex_lock(&conn_state->lock); 2359 if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS || 2360 conn_state->state == AMDGPU_UALINK_CONN_PENDING) { 2361 r = -EAGAIN; 2362 goto out; 2363 } 2364 2365 if (conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED && 2366 !amdgpu_ualink_peer_idle(adev, remote_acc_id)) { 2367 r = 0; 2368 goto out; 2369 } 2370 2371 /* Peer is idle or not connected, it may have rebooted. Skip the 2372 * handshake if the rings are still in sync, it would reset the remote. 2373 */ 2374 r = amdgpu_ualink_ring_resync_ptrs(adev, remote_acc_id); 2375 if (r < 0) 2376 goto out; 2377 if (!r && conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) 2378 goto out; 2379 2380 amdgpu_ualink_reset_peer_rings(adev, remote_acc_id); 2381 2382 conn_state->state = AMDGPU_UALINK_CONN_IN_PROGRESS; 2383 2384 mutex_unlock(&conn_state->lock); 2385 2386 /* Send HELLO message */ 2387 r = amdgpu_ualink_send_hello_msg(adev, remote_acc_id); 2388 if (r) { 2389 dev_warn(adev->dev, 2390 "HELLO: Send failed to remote AccId:%u\n", 2391 remote_acc_id); 2392 goto reset_state; 2393 } 2394 2395 /* Wait for the HELLO_ACK to come back */ 2396 /* complete(conn_state->hello_done) should be called from the IRQ 2397 * handler when the HELLO_ACK is received. 2398 */ 2399 r = wait_for_completion_interruptible_timeout(&conn_state->hello_done, 2400 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 2401 if (r == -ERESTARTSYS) { 2402 dev_err_ratelimited(adev->dev, 2403 "HELLO-ACK: interrupted by signal\n"); 2404 goto reset_state; 2405 } else if (r == 0) { 2406 dev_warn(adev->dev, 2407 "HELLO-ACK: Timeout from remote AccId:%u\n", 2408 remote_acc_id); 2409 r = -ETIMEDOUT; 2410 goto reset_state; 2411 } 2412 2413 return 0; 2414 2415 reset_state: 2416 mutex_lock(&conn_state->lock); 2417 conn_state->state = AMDGPU_UALINK_CONN_NOT_READY; 2418 out: 2419 mutex_unlock(&conn_state->lock); 2420 2421 return r; 2422 } 2423 2424 /* Invalidate an importer node's mappings and drop its last ref (frees the 2425 * node, dma-buf and GEM handle). Caller must have unlinked it from the 2426 * per-remote list and must not hold the imp_xa lock. 2427 */ 2428 static void amdgpu_ualink_release_imp_xa_node(struct amdgpu_device *adev, 2429 struct amdgpu_ualink_imp_xa_node *imp_xa_node) 2430 { 2431 struct amdgpu_bo *bo; 2432 2433 dev_dbg(adev->dev, 2434 "IMP-CLEANUP: handle:%llx:%llx npa:%llx size:%llx\n", 2435 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 2436 imp_xa_node->npa_addr, imp_xa_node->size); 2437 2438 if (imp_xa_node->dmabuf) { 2439 bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv); 2440 amdgpu_ualink_invalidate_import_mappings(bo); 2441 } 2442 2443 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 2444 } 2445 2446 static void amdgpu_ualink_cleanup_imp_xa_entries(struct amdgpu_device *adev, 2447 u32 remote_acc_id) 2448 { 2449 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 2450 struct list_head *imp_handles_list; 2451 2452 dev_dbg(adev->dev, 2453 "IMP-RESET: Cleaning up all XA entries for remote:%u\n", 2454 remote_acc_id); 2455 2456 imp_handles_list = &adev->ualink.imp_handles_list[remote_acc_id]; 2457 2458 xa_lock(&adev->ualink.imp_xa); 2459 while (!list_empty(imp_handles_list)) { 2460 imp_xa_node = list_first_entry(imp_handles_list, 2461 struct amdgpu_ualink_imp_xa_node, list); 2462 list_del_init(&imp_xa_node->list); 2463 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 2464 xa_unlock(&adev->ualink.imp_xa); 2465 2466 amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node); 2467 2468 xa_lock(&adev->ualink.imp_xa); 2469 } 2470 xa_unlock(&adev->ualink.imp_xa); 2471 } 2472 2473 static void amdgpu_ualink_cleanup_exp_xa_entries(struct amdgpu_device *adev, 2474 u32 remote_acc_id) 2475 { 2476 struct amdgpu_ualink_importer_entry *importer_entry; 2477 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2478 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 2479 struct list_head *exp_handles_list; 2480 struct drm_mm_node *mm_node; 2481 u64 npa_addr, size; 2482 2483 /* Get the list head for the list containing all the handles 2484 * exported to this remote GPU. 2485 */ 2486 exp_handles_list = &adev->ualink.exp_handles_list[remote_acc_id]; 2487 2488 dev_dbg(adev->dev, 2489 "EXP-RESET: Cleaning up all XA entries for remote:%u\n", 2490 remote_acc_id); 2491 2492 xa_lock(&adev->ualink.exp_xa); 2493 while (!list_empty(exp_handles_list)) { 2494 importer_entry = list_first_entry(exp_handles_list, 2495 struct amdgpu_ualink_importer_entry, list); 2496 list_del_init(&importer_entry->list); 2497 2498 exp_xa_node = importer_entry->parent; 2499 if (!amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) 2500 continue; 2501 2502 xa_unlock(&adev->ualink.exp_xa); 2503 /* Clear the bit corresponding to this remote GPU in 2504 * the importer bitmap. 2505 */ 2506 if (!test_and_clear_bit(remote_acc_id, 2507 exp_xa_node->importers_bitmap)) { 2508 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 2509 xa_lock(&adev->ualink.exp_xa); 2510 continue; 2511 } 2512 2513 mutex_lock(&exp_xa_node->node_lock); 2514 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2515 /* Unpin the BO */ 2516 if (likely(!amdgpu_bo_reserve(exp_xa_node->bo, true))) { 2517 amdgpu_bo_unpin(exp_xa_node->bo); 2518 amdgpu_bo_unreserve(exp_xa_node->bo); 2519 } else { 2520 dev_warn(adev->dev, 2521 "EXP-RESET: BO reserve to unpin failed handle:%llx:%llx\n", 2522 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo); 2523 } 2524 2525 dev_dbg(adev->dev, 2526 "EXP-RESET: handle:%llx:%llx pin_count:%d, importers:%d\n", 2527 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 2528 exp_xa_node->bo->tbo.pin_count, 2529 bitmap_weight(exp_xa_node->importers_bitmap, 2530 AMDGPU_UALINK_ACCEL_MAX)); 2531 2532 WARN_ON(exp_xa_node->bo->tbo.pin_count < 2533 bitmap_weight(exp_xa_node->importers_bitmap, 2534 AMDGPU_UALINK_ACCEL_MAX)); 2535 2536 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 2537 npa_addr = importer_entry->npa_addr; 2538 mm_node = importer_entry->mm_node; 2539 mutex_unlock(&exp_xa_node->node_lock); 2540 2541 dev_dbg(adev->dev, 2542 "EXP-RESET: Unmap NPA:%llx size: %llx remote:%u handle:%llx:%llx\n", 2543 npa_addr, size, remote_acc_id, 2544 exp_xa_node->handle.handle_hi, 2545 exp_xa_node->handle.handle_lo); 2546 amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo, 2547 npa_addr, size); 2548 amdgpu_ualink_npa_free_va(adev, mm_node); 2549 kfree(mm_node); 2550 2551 /* Reset the NPA addr and mm_node */ 2552 mutex_lock(&exp_xa_node->node_lock); 2553 importer_entry->npa_addr = 0; 2554 importer_entry->mm_node = NULL; 2555 } else if (bitmap_empty(exp_xa_node->importers_bitmap, 2556 AMDGPU_UALINK_ACCEL_MAX)) { 2557 /* In Source-Aliasing mode, if there are no importers 2558 * for this handle, then we can unmap and free the 2559 * NPA address. 2560 */ 2561 mm_node = exp_xa_node->importer_entries[0].mm_node; 2562 npa_addr = exp_xa_node->importer_entries[0].npa_addr; 2563 mutex_unlock(&exp_xa_node->node_lock); 2564 2565 dev_dbg(adev->dev, 2566 "EXP-RESET: Unmap NPA:%llx size: %llx handle:%llx:%llx\n", 2567 npa_addr, size, exp_xa_node->handle.handle_hi, 2568 exp_xa_node->handle.handle_lo); 2569 amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo, 2570 npa_addr, size); 2571 amdgpu_ualink_npa_free_va(adev, mm_node); 2572 kfree(mm_node); 2573 mutex_lock(&exp_xa_node->node_lock); 2574 exp_xa_node->importer_entries[0].npa_addr = 0; 2575 exp_xa_node->importer_entries[0].mm_node = NULL; 2576 } 2577 2578 mutex_unlock(&exp_xa_node->node_lock); 2579 xa_lock(&adev->ualink.exp_xa); 2580 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 2581 } 2582 xa_unlock(&adev->ualink.exp_xa); 2583 } 2584 2585 static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev, 2586 u32 remote_acc_id, u32 state, 2587 u32 generation_count) 2588 { 2589 struct amdgpu_ualink_connection *conn_state; 2590 2591 conn_state = &adev->ualink.conn_state[remote_acc_id]; 2592 2593 mutex_lock(&conn_state->lock); 2594 if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) && 2595 (conn_state->generation_count == generation_count)) { 2596 conn_state->state = state; 2597 mutex_unlock(&conn_state->lock); 2598 2599 amdgpu_ualink_cleanup_imp_xa_entries(adev, remote_acc_id); 2600 amdgpu_ualink_cleanup_exp_xa_entries(adev, remote_acc_id); 2601 } else { 2602 mutex_unlock(&conn_state->lock); 2603 } 2604 } 2605 2606 /* Set PTE.X = 1 for all importer entries to retry RPCs. */ 2607 static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev, 2608 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2609 { 2610 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2611 struct amdgpu_ualink_importer_entry *imp_entry; 2612 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2613 u64 pte_flags, npa_addr, size; 2614 struct dma_fence *fence = NULL; 2615 struct amdgpu_bo *bo; 2616 struct drm_exec exec; 2617 u32 remote_acc_id; 2618 int r; 2619 2620 bo = exp_xa_node->bo; 2621 size = amdgpu_bo_ngpu_pages(bo); 2622 2623 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false); 2624 if (unlikely(r)) { 2625 dev_err(adev->dev, "Failed to reserve VM and BO in force_retry_rpcs\n"); 2626 return; 2627 } 2628 2629 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2630 AMDGPU_UALINK_ACCEL_MAX) { 2631 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2632 imp_entry = &exp_xa_node->importer_entries[0]; 2633 else 2634 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2635 npa_addr = imp_entry->npa_addr; 2636 2637 /* If the connection state changed while we are freeing 2638 * the BO, then ignore this importer. We will unmap this 2639 * address eventually in amdgpu_ualink_unmap_all_npa_addr() 2640 * function. Also, we will clear the corresponding bit in 2641 * the importer_bitmap in the same function. 2642 */ 2643 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2644 imp_entry->generation_count)) 2645 continue; 2646 2647 /* Set PTE.X = 1 */ 2648 pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 2649 AMDGPU_VM_PAGE_EXECUTABLE); 2650 dev_dbg(adev->dev, 2651 "RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n", 2652 npa_addr, remote_acc_id, pte_flags); 2653 2654 r = amdgpu_vm_update_range(adev, vm, false, false, true, 2655 false, NULL, npa_addr, npa_addr + size - 1, 2656 pte_flags, 0, adev->vm_manager.vram_base_offset, 2657 bo->tbo.resource, NULL, &vm->last_update); 2658 2659 if (r) 2660 dev_warn(adev->dev, 2661 "RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n", 2662 npa_addr, remote_acc_id, r); 2663 2664 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2665 break; 2666 } 2667 2668 r = amdgpu_vm_update_pdes(adev, vm, false); 2669 if (r) { 2670 dev_err(adev->dev, 2671 "Failed %d to update page directories during force retry rpcs\n", 2672 r); 2673 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2674 return; 2675 } 2676 2677 fence = dma_fence_get(vm->last_update); 2678 if (fence) { 2679 r = dma_fence_wait(fence, false); 2680 dma_fence_put(fence); 2681 fence = NULL; 2682 if (r) 2683 dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r); 2684 } 2685 2686 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2687 2688 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2689 } 2690 2691 static void amdgpu_ualink_send_tlb_shootdown(struct amdgpu_device *adev, 2692 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2693 { 2694 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2695 u32 accel_id = adev->ualink.info->ppod.accel_id; 2696 struct amdgpu_ualink_importer_entry *imp_entry; 2697 u64 npa_addr, size; 2698 u32 remote_acc_id; 2699 int r; 2700 2701 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2702 2703 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2704 AMDGPU_UALINK_ACCEL_MAX) { 2705 /* 2706 * Replace the GPU-id bits with local accel_id 2707 */ 2708 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2709 imp_entry = &exp_xa_node->importer_entries[0]; 2710 else 2711 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2712 2713 npa_addr = STRIP_NPA(imp_entry->npa_addr); 2714 npa_addr = GENERATE_NPA(npa_addr, accel_id); 2715 2716 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2717 imp_entry->generation_count)) { 2718 clear_bit(remote_acc_id, 2719 exp_xa_node->importers_bitmap); 2720 continue; 2721 } 2722 2723 dev_dbg(adev->dev, 2724 "EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n", 2725 remote_acc_id); 2726 r = amdgpu_ualink_remote_shootdown(adev, remote_acc_id, 2727 npa_addr, size, 2728 AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN); 2729 if (r) { 2730 dev_err(adev->dev, 2731 "EXP-CLEANUP: TLB shootdown send failed to remote:%u\n", 2732 remote_acc_id); 2733 2734 /* Send failed: mark the connection NOT_READY so the 2735 * rest of this cleanup (and other cleanups) skip the 2736 * unreachable peer via amdgpu_ualink_check_conn_ready(). 2737 */ 2738 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 2739 AMDGPU_UALINK_CONN_NOT_READY, 2740 imp_entry->generation_count); 2741 } 2742 } 2743 } 2744 2745 /* Unmap all NPA addresses associated with a BO (UALink handle). This function is used 2746 * only in Source Identification mode. 2747 */ 2748 static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev, 2749 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 2750 { 2751 struct amdgpu_vm *vm = &adev->ualink.npa_vm; 2752 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2753 struct amdgpu_ualink_importer_entry *imp_entry; 2754 u64 pte_value = adev->gmc.noretry_flags; 2755 struct dma_fence *fence = NULL; 2756 struct drm_exec exec; 2757 u64 npa_addr, size; 2758 u32 remote_acc_id; 2759 int r; 2760 2761 size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); 2762 2763 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1, 2764 &exec, false); 2765 if (unlikely(r)) { 2766 dev_err(adev->dev, "Failed to reserve VM and BO in unmap_all_npa_addr\n"); 2767 return; 2768 } 2769 2770 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2771 AMDGPU_UALINK_ACCEL_MAX) { 2772 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2773 imp_entry = &exp_xa_node->importer_entries[0]; 2774 else 2775 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2776 npa_addr = imp_entry->npa_addr; 2777 2778 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2779 imp_entry->generation_count)) { 2780 clear_bit(remote_acc_id, 2781 exp_xa_node->importers_bitmap); 2782 continue; 2783 } 2784 2785 dev_dbg(adev->dev, 2786 "UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n", 2787 npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 2788 remote_acc_id, pte_value); 2789 2790 r = amdgpu_vm_update_range(adev, vm, false, 2791 false, true, false, NULL, npa_addr, 2792 npa_addr + size - 1, pte_value, 0, 2793 0, NULL, NULL, &vm->last_update); 2794 2795 if (r) 2796 dev_err(adev->dev, 2797 "UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n", 2798 npa_addr, exp_xa_node->handle.handle_hi, 2799 exp_xa_node->handle.handle_lo, remote_acc_id); 2800 2801 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2802 break; 2803 } 2804 2805 r = amdgpu_vm_update_pdes(adev, vm, false); 2806 if (r) { 2807 dev_err(adev->dev, 2808 "Failed %d to update page directories during all NPA addresses unmapping\n", 2809 r); 2810 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2811 return; 2812 } 2813 2814 fence = dma_fence_get(vm->last_update); 2815 if (fence) { 2816 r = dma_fence_wait(fence, false); 2817 dma_fence_put(fence); 2818 fence = NULL; 2819 if (r) 2820 dev_err(adev->dev, 2821 "UNMAP-NPA: dma fence wait failed\n"); 2822 } 2823 2824 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 2825 2826 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 2827 } 2828 2829 static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev, 2830 struct amdgpu_ualink_exp_xa_node *exp_xa_node, 2831 unsigned long *importers_bitmap) 2832 { 2833 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 2834 struct drm_mm_node *mm_node; 2835 u32 remote_acc_id; 2836 2837 for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) { 2838 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2839 mm_node = exp_xa_node->importer_entries[0].mm_node; 2840 else 2841 mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node; 2842 2843 if (!mm_node) { 2844 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2845 break; 2846 continue; 2847 } 2848 2849 dev_dbg(adev->dev, 2850 "FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n", 2851 mm_node->start, exp_xa_node->handle.handle_hi, 2852 exp_xa_node->handle.handle_lo, remote_acc_id); 2853 2854 amdgpu_ualink_npa_free_va(adev, mm_node); 2855 kfree(mm_node); 2856 2857 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 2858 break; 2859 } 2860 } 2861 2862 static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work) 2863 { 2864 DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 2865 struct amdgpu_ualink_importer_entry *imp_entry; 2866 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 2867 struct amdgpu_ualink_handle handle; 2868 struct amdgpu_device *adev; 2869 struct amdgpu_bo *bo; 2870 u32 remote_acc_id; 2871 int r; 2872 2873 exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node, 2874 cleanup_work); 2875 bo = exp_xa_node->bo; 2876 adev = amdgpu_ttm_adev(bo->tbo.bdev); 2877 handle = exp_xa_node->handle; 2878 2879 /* Revoking access to an exported memory follows the steps: 2880 * 1. Set PTE.X = 1 to retry for RPCs. 2881 * 2. Send Remote TLB Shootdowns to all importers. 2882 * 3. Unmap the NPA address from NPA VM. 2883 * 4. Drop the ref count for the BO. 2884 * 5. Send NPA_REVOKE to all importers. 2885 * 6. Wait for NPA_RELEASE from all importers. 2886 * 7. Free the NPA address once all responses are received. 2887 */ 2888 /* If there are no importers for this BO/handle */ 2889 if (bitmap_empty(exp_xa_node->importers_bitmap, 2890 AMDGPU_UALINK_ACCEL_MAX)) { 2891 /* Release the dma_buf created at export time (used for the 2892 * local-import shortcut). The has-importers path below drops 2893 * it too; this branch must not skip it or the dma_buf and the 2894 * BO it pins are leaked. 2895 */ 2896 dma_buf_put(exp_xa_node->dmabuf); 2897 /* Drop the BO reference so it can be freed. */ 2898 amdgpu_bo_unref(&bo); 2899 exp_xa_node->bo = NULL; 2900 goto free_node; 2901 } 2902 2903 dev_dbg(adev->dev, 2904 "EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n", 2905 handle.handle_hi, handle.handle_lo, 2906 AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap); 2907 2908 bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap, 2909 AMDGPU_UALINK_ACCEL_MAX); 2910 2911 /* Set PTE.X = 1 for NPA addresses from all importers*/ 2912 amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node); 2913 2914 /* Send TLB-shootdown to all importer GPUs */ 2915 amdgpu_ualink_send_tlb_shootdown(adev, exp_xa_node); 2916 2917 /* Unmap all NPA addresses for this BO from NPA VM */ 2918 amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node); 2919 2920 dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n", 2921 handle.handle_hi, handle.handle_lo, bo->tbo.pin_count, 2922 bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)); 2923 WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap, 2924 AMDGPU_UALINK_ACCEL_MAX)); 2925 2926 /* Unpin the BO */ 2927 if (likely(!amdgpu_bo_reserve(bo, true))) { 2928 bo->ualink_handle_lo = 0ULL; 2929 for_each_set_bit(remote_acc_id, orig_importers_bitmap, 2930 AMDGPU_UALINK_ACCEL_MAX) 2931 amdgpu_bo_unpin(bo); 2932 amdgpu_bo_unreserve(bo); 2933 } else { 2934 dev_warn(adev->dev, 2935 "EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n", 2936 handle.handle_hi, handle.handle_lo); 2937 } 2938 2939 /* Free the DMABuf */ 2940 dma_buf_put(exp_xa_node->dmabuf); 2941 /* Drop the reference to the BO so it can be freed. */ 2942 amdgpu_bo_unref(&bo); 2943 exp_xa_node->bo = NULL; 2944 2945 /* Seed npa_release_bitmap with all importers before any NPA-REVOKE, so 2946 * the release IRQ only sees it empty once every responder has replied. 2947 * Dead peers are dropped from it in the send loop below. 2948 */ 2949 bitmap_copy(exp_xa_node->npa_release_bitmap, 2950 exp_xa_node->importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 2951 2952 /* Send NPA-REVOKE to all importers which have imported this memory. 2953 * On send failure clear the bit (no response will arrive) and mark the 2954 * connection NOT_READY so other cleanups skip this unreachable peer. 2955 */ 2956 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 2957 AMDGPU_UALINK_ACCEL_MAX) { 2958 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 2959 2960 /* Re-check the connection right before the send: concurrent 2961 * cleanup workers all seed the bitmap while the peer is still 2962 * ESTABLISHED, so once its egress dies and the first worker 2963 * marks it NOT_READY, this skips the rest instead of flooding 2964 * the log with failed LSDMA sends to the same dead peer. 2965 */ 2966 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 2967 imp_entry->generation_count)) { 2968 clear_bit(remote_acc_id, 2969 exp_xa_node->npa_release_bitmap); 2970 continue; 2971 } 2972 2973 dev_dbg(adev->dev, 2974 "EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n", 2975 remote_acc_id); 2976 r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle); 2977 if (r) { 2978 dev_err(adev->dev, 2979 "EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n", 2980 remote_acc_id); 2981 clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap); 2982 2983 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 2984 AMDGPU_UALINK_CONN_NOT_READY, 2985 imp_entry->generation_count); 2986 } 2987 } 2988 2989 /* Wait for the NPA_RELEASE to come back from all importers */ 2990 if (!bitmap_empty(exp_xa_node->npa_release_bitmap, 2991 AMDGPU_UALINK_ACCEL_MAX)) { 2992 dev_dbg(adev->dev, 2993 "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n", 2994 handle.handle_hi, handle.handle_lo, 2995 AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap); 2996 2997 /* Wait for the NPA_RELEASE to come back from all importers */ 2998 r = wait_for_completion_timeout(&exp_xa_node->npa_done, 2999 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 3000 3001 if (r == 0) 3002 dev_warn(adev->dev, 3003 "EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n", 3004 handle.handle_hi, handle.handle_lo); 3005 } 3006 3007 /* Free the NPA addresses given to all the importers */ 3008 amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, 3009 orig_importers_bitmap); 3010 3011 /* Warn about all importers that didn't respond back with 3012 * NPA-RELEASE message and trigger connection timeout handling. 3013 */ 3014 for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap, 3015 AMDGPU_UALINK_ACCEL_MAX) { 3016 dev_warn(adev->dev, 3017 "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n", 3018 handle.handle_hi, handle.handle_lo, remote_acc_id); 3019 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3020 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 3021 AMDGPU_UALINK_CONN_NOT_READY, 3022 imp_entry->generation_count); 3023 } 3024 3025 free_node: 3026 xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo); 3027 mutex_destroy(&exp_xa_node->node_lock); 3028 kfree(exp_xa_node); 3029 } 3030 3031 /* This function is a copy of amdgpu_dma_buf_move_notify() function. 3032 * amdgpu_dma_buf_move_notify is only called for import attachments. 3033 * But NPA DMABufs don't use attachments because they are imported 3034 * on the same device. So we need to invalidate the GPUVM mappings 3035 * manually. 3036 */ 3037 static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo) 3038 { 3039 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 3040 struct ttm_operation_ctx ctx = { false, false }; 3041 struct ttm_placement placement = {}; 3042 struct amdgpu_vm_bo_base *bo_base; 3043 struct drm_exec exec; 3044 int r; 3045 3046 /* 3047 * Lock the BO together with every client VM page directory it is 3048 * mapped into in a single drm_exec transaction. 3049 */ 3050 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 3051 drm_exec_until_all_locked(&exec) { 3052 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 3053 drm_exec_retry_on_contention(&exec); 3054 if (unlikely(r)) 3055 goto fini; 3056 3057 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 3058 r = amdgpu_vm_lock_pd(bo_base->vm, &exec, 0); 3059 drm_exec_retry_on_contention(&exec); 3060 if (unlikely(r)) 3061 goto fini; 3062 } 3063 } 3064 3065 amdgpu_vm_bo_move(bo, NULL, false); 3066 3067 r = ttm_bo_validate(&bo->tbo, &placement, &ctx); 3068 if (r) { 3069 dev_err(adev->dev, "Failed to invalidate NPA DMA-buf import (%d)\n", 3070 r); 3071 goto fini; 3072 } 3073 3074 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 3075 struct amdgpu_vm *vm = bo_base->vm; 3076 3077 /* 3078 * Fences for the two SDMA page table updates were already 3079 * reserved by amdgpu_vm_lock_pd() above (it reserves 2 + 3080 * num_fences on the same VM root PD dma_resv). 3081 */ 3082 r = amdgpu_vm_clear_freed(adev, vm, NULL); 3083 if (!r) 3084 r = amdgpu_vm_handle_moved(adev, vm, &exec.ticket); 3085 3086 if (r && r != -EBUSY) 3087 dev_err(adev->dev, "Failed to invalidate VM page tables (%d))\n", 3088 r); 3089 } 3090 3091 fini: 3092 drm_exec_fini(&exec); 3093 } 3094 3095 static int amdgpu_ualink_map_npa_to_dmabuf(struct amdgpu_device *adev, 3096 struct amdgpu_ualink_imp_xa_node *imp_xa_node) 3097 { 3098 struct ttm_operation_ctx ctx = { false, false }; 3099 u32 initial_domain = AMDGPU_GEM_DOMAIN_NPA; 3100 struct drm_gem_object *gobj = NULL; 3101 u64 alloc_flags, npa_addr, size; 3102 struct dma_buf *dmabuf; 3103 struct amdgpu_bo *bo; 3104 u32 handle; 3105 int r; 3106 3107 npa_addr = imp_xa_node->npa_addr; 3108 size = imp_xa_node->size; 3109 3110 dev_dbg(adev->dev, "Create NPA BO addr 0x%llx size in pages 0x%llx\n", 3111 npa_addr, size); 3112 3113 /* 3114 * Create the BO directly in the NPA domain. 3115 */ 3116 alloc_flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 3117 r = amdgpu_gem_object_create(adev, size * AMDGPU_GPU_PAGE_SIZE, 1, 3118 initial_domain, alloc_flags, 3119 ttm_bo_type_device, NULL, &gobj, 0); 3120 if (r) { 3121 dev_err(adev->dev, "Failed to create NPA BO. ret %d\n", r); 3122 return r; 3123 } 3124 3125 bo = gem_to_amdgpu_bo(gobj); 3126 3127 r = amdgpu_bo_reserve(bo, false); 3128 if (unlikely(r != 0)) { 3129 dev_err(adev->dev, "Failed to reserve NPA BO, r: %d\n", r); 3130 goto err_reserve_failed; 3131 } 3132 3133 /* 3134 * Drop the arbitrarily-placed NPA node and re-create it at the exact 3135 * remote window, mirroring amdgpu_bo_create_kernel_at(). 3136 */ 3137 ttm_resource_free(&bo->tbo, &bo->tbo.resource); 3138 3139 bo->placements[0].fpfn = npa_addr; 3140 bo->placements[0].lpfn = npa_addr + size; 3141 3142 r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.resource, &ctx); 3143 amdgpu_bo_unreserve(bo); 3144 if (r) { 3145 dev_err(adev->dev, 3146 "Failed to place NPA BO at 0x%llx, r: %d\n", npa_addr, r); 3147 goto err_validate_failed; 3148 } 3149 3150 r = drm_gem_handle_create(adev->ualink.client.file, gobj, &handle); 3151 if (r) { 3152 dev_err(adev->dev, 3153 "Failed to get handle for NPA GEM object, r: %d\n", r); 3154 goto err_validate_failed; 3155 } 3156 drm_gem_object_put(gobj); 3157 3158 dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, adev->ualink.client.file, 3159 handle, DRM_CLOEXEC | DRM_RDWR); 3160 if (IS_ERR(dmabuf)) { 3161 r = PTR_ERR(dmabuf); 3162 dev_err(adev->dev, 3163 "Failed to generate DMABuf for NPA GEM object\n"); 3164 goto err_dmabuf_failed; 3165 } 3166 3167 imp_xa_node->dmabuf = dmabuf; 3168 imp_xa_node->gem_handle = handle; 3169 3170 return 0; 3171 3172 err_dmabuf_failed: 3173 drm_gem_handle_delete(adev->ualink.client.file, handle); 3174 return r; 3175 err_validate_failed: 3176 err_reserve_failed: 3177 drm_gem_object_put(gobj); 3178 3179 return r; 3180 } 3181 3182 void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo) 3183 { 3184 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 3185 struct amdgpu_ualink_importer_entry *imp_entry; 3186 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3187 u32 remote_acc_id; 3188 int r; 3189 3190 if (!bo->ualink_handle_lo) 3191 return; 3192 3193 /* Remove the entry from the Xarray. */ 3194 xa_lock(&adev->ualink.exp_xa); 3195 exp_xa_node = __xa_erase(&adev->ualink.exp_xa, 3196 bo->ualink_handle_lo); 3197 if (!exp_xa_node) { 3198 xa_unlock(&adev->ualink.exp_xa); 3199 dev_warn(adev->dev, 3200 "Exp XA: handle_lo:%llx not found\n", 3201 bo->ualink_handle_lo); 3202 return; 3203 } 3204 3205 for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, 3206 AMDGPU_UALINK_ACCEL_MAX) { 3207 imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3208 list_del_init(&imp_entry->list); 3209 } 3210 xa_unlock(&adev->ualink.exp_xa); 3211 3212 /* Add it to the Handle_Invalid xarray */ 3213 r = xa_err(xa_store(&adev->ualink.handle_invalid_xa, 3214 bo->ualink_handle_lo, 3215 exp_xa_node, GFP_KERNEL)); 3216 if (r) 3217 dev_err(adev->dev, 3218 "Handle_Invalid XA store failed handle:%llx:%llx error:%d\n", 3219 exp_xa_node->handle.handle_hi, 3220 exp_xa_node->handle.handle_lo, r); 3221 3222 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3223 } 3224 3225 static void amdgpu_ualink_process_npa_release_msg(struct amdgpu_device *adev, 3226 u32 remote_acc_id, 3227 struct amdgpu_ualink_handle handle) 3228 { 3229 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3230 3231 xa_lock(&adev->ualink.handle_invalid_xa); 3232 exp_xa_node = xa_load(&adev->ualink.handle_invalid_xa, handle.handle_lo); 3233 if (!exp_xa_node) { 3234 dev_warn(adev->dev, 3235 "NPA-RELEASE: Handle (%llx:%llx) not found\n", 3236 handle.handle_hi, handle.handle_lo); 3237 goto out; 3238 } 3239 3240 /* Confirm that the complete handle matches. */ 3241 if (handle.handle_hi != exp_xa_node->handle.handle_hi) { 3242 dev_warn(adev->dev, 3243 "NPA-RELEASE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n", 3244 exp_xa_node->handle.handle_hi, 3245 handle.handle_hi, handle.handle_lo, remote_acc_id); 3246 goto out; 3247 } 3248 3249 /* Test and clear the bit corresponding to the remote GPU id to signal 3250 * the arrival of NPA_RELEASE message from it. 3251 * If the corresponding bit wasn't set, then raise a warning and ignore 3252 * the NPA-Release message from the remote GPU. 3253 */ 3254 if (!test_and_clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap)) { 3255 dev_warn(adev->dev, 3256 "NPA-RELEASE: unexpected from remote:%u handle:%llx:%llx\n", 3257 remote_acc_id, handle.handle_hi, handle.handle_lo); 3258 goto out; 3259 } 3260 3261 /* Signal completion if NPA_Release received from all importers */ 3262 if (bitmap_empty(exp_xa_node->npa_release_bitmap, 3263 AMDGPU_UALINK_ACCEL_MAX)) 3264 complete(&exp_xa_node->npa_done); 3265 3266 out: 3267 xa_unlock(&adev->ualink.handle_invalid_xa); 3268 } 3269 3270 static void amdgpu_ualink_process_npa_revoke_msg(struct amdgpu_device *adev, 3271 u32 remote_acc_id, 3272 struct amdgpu_ualink_handle handle) 3273 { 3274 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3275 struct amdgpu_bo *bo; 3276 int r = 0; 3277 3278 /* Remove the entry from the Xarray. */ 3279 xa_lock(&adev->ualink.imp_xa); 3280 imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo); 3281 if (!imp_xa_node) { 3282 xa_unlock(&adev->ualink.imp_xa); 3283 dev_warn(adev->dev, 3284 "NPA-REVOKE: Handle (%llx:%llx) not found\n", 3285 handle.handle_hi, handle.handle_lo); 3286 return; 3287 } 3288 3289 /* Confirm that the complete handle matches. */ 3290 if (handle.handle_hi != imp_xa_node->handle.handle_hi) { 3291 xa_unlock(&adev->ualink.imp_xa); 3292 dev_warn(adev->dev, 3293 "NPA-REVOKE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n", 3294 imp_xa_node->handle.handle_hi, 3295 handle.handle_hi, handle.handle_lo, remote_acc_id); 3296 return; 3297 } 3298 3299 switch (READ_ONCE(imp_xa_node->node_state)) { 3300 case AMDGPU_UALINK_NODE_READY: 3301 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 3302 list_del_init(&imp_xa_node->list); 3303 xa_unlock(&adev->ualink.imp_xa); 3304 3305 /* Invalidate the GPUVM mappings */ 3306 bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv); 3307 amdgpu_ualink_invalidate_import_mappings(bo); 3308 3309 /* Drop the refcount for the node */ 3310 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 3311 break; 3312 case AMDGPU_UALINK_NODE_PENDING: 3313 /* The import is still building the dma-buf and nothing has 3314 * been handed to user-space yet. The importing thread sees 3315 * the teardown state and unwinds. 3316 */ 3317 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 3318 xa_unlock(&adev->ualink.imp_xa); 3319 break; 3320 default: 3321 /* NPA-REVOKE always follows NPA-RSP, so a NOT_READY node means 3322 * a stale revoke, and a node in teardown is already being 3323 * released by whoever moved it there. 3324 */ 3325 xa_unlock(&adev->ualink.imp_xa); 3326 return; 3327 } 3328 3329 r = amdgpu_ualink_send_npa_release_msg(adev, remote_acc_id, handle); 3330 if (r) 3331 dev_err(adev->dev, 3332 "NPA-Release send failed remote:%u handle:%llx:%llx error:%d\n", 3333 remote_acc_id, handle.handle_hi, handle.handle_lo, r); 3334 } 3335 3336 static void amdgpu_ualink_process_npa_fail_msg(struct amdgpu_device *adev, 3337 u32 remote_acc_id, u64 partial_handle, 3338 u32 fail_reason) 3339 { 3340 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3341 int r = 0; 3342 3343 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) { 3344 dev_warn(adev->dev, 3345 "NPA-FAIL: no connection with remote AccId:%u\n", 3346 remote_acc_id); 3347 goto conn_setup; 3348 } 3349 3350 xa_lock(&adev->ualink.imp_xa); 3351 imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle); 3352 if (!imp_xa_node) { 3353 xa_unlock(&adev->ualink.imp_xa); 3354 dev_warn(adev->dev, 3355 "NPA-FAIL: imp XA handle not found:%llx\n", 3356 partial_handle); 3357 return; 3358 } 3359 3360 imp_xa_node->fail_reason = fail_reason; 3361 /* Signal completion done to signal response received for NPA-REQ 3362 * message. 3363 * If the node is in NOT_READY state, then set the node state to 3364 * PENDING and signal the completion. If the node is not in NOT_READY 3365 * state, then it is an unsolicited NPA-FAIL message and we 3366 * log a debug message. 3367 */ 3368 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) { 3369 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING); 3370 complete(&imp_xa_node->npa_done); 3371 } else { 3372 dev_dbg(adev->dev, 3373 "NPA-FAIL: unsolicited for handle:%llx:%llx from AccId:%u\n", 3374 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 3375 remote_acc_id); 3376 } 3377 xa_unlock(&adev->ualink.imp_xa); 3378 3379 return; 3380 3381 conn_setup: 3382 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3383 if (r) 3384 dev_warn(adev->dev, 3385 "NPA-FAIL: connection setup failed with remote AccId:%u\n", 3386 remote_acc_id); 3387 } 3388 3389 static void amdgpu_ualink_process_npa_rsp_msg(struct amdgpu_device *adev, 3390 u32 remote_acc_id, u64 partial_handle, 3391 u64 npa_addr, u64 size) 3392 { 3393 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3394 int r = 0; 3395 3396 /* Check if the connection is established. If it is not, then start 3397 * connection setup. 3398 */ 3399 if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) { 3400 dev_warn(adev->dev, 3401 "NPA-RSP: no connection with remote AccId:%u\n", 3402 remote_acc_id); 3403 goto conn_setup; 3404 } 3405 3406 xa_lock(&adev->ualink.imp_xa); 3407 imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle); 3408 if (!imp_xa_node) { 3409 xa_unlock(&adev->ualink.imp_xa); 3410 dev_warn(adev->dev, 3411 "NPA-RSP: imp XA handle not found:%llx\n", partial_handle); 3412 return; 3413 } 3414 3415 /* NPA addr received in NPA-RSP is page aligned and without the remote 3416 * GPU-id in Bits 41-50. Assemble back the NPA address before storing 3417 * it. 3418 */ 3419 imp_xa_node->npa_addr = GENERATE_NPA(npa_addr, remote_acc_id); 3420 /* Size is in number of GPU pages granularity. */ 3421 imp_xa_node->size = size; 3422 3423 /* Signal completion done to signal NPA_RSP received. 3424 * If the node is in NOT_READY state, then set the node state to 3425 * PENDING and signal the completion. If the node is not in NOT_READY 3426 * state, then it is an unsolicited NPA-RSP message and we 3427 * log a debug message. 3428 */ 3429 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) { 3430 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING); 3431 complete(&imp_xa_node->npa_done); 3432 } else { 3433 dev_dbg(adev->dev, 3434 "NPA-RSP: unsolicited for handle:%llx:%llx from AccId:%u\n", 3435 imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo, 3436 remote_acc_id); 3437 } 3438 xa_unlock(&adev->ualink.imp_xa); 3439 3440 return; 3441 3442 conn_setup: 3443 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3444 if (r) 3445 dev_warn(adev->dev, 3446 "NPA-RSP: connection setup failed with remote AccId:%u\n", 3447 remote_acc_id); 3448 } 3449 3450 static void amdgpu_ualink_process_npa_req_msg(struct amdgpu_device *adev, 3451 u32 remote_acc_id, 3452 struct amdgpu_ualink_handle handle) 3453 { 3454 struct amdgpu_ualink_importer_entry *importer_entry, *npa_addr_entry; 3455 u32 addr_mode = adev->ualink.info->vpod.addr_mode; 3456 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3457 u64 range_start, range_end, pte_flags; 3458 struct drm_mm_node *mm_node = NULL; 3459 int r = 0, fail_reason = 0; 3460 bool send_npa_fail = true; 3461 u64 npa_addr = 0, size; 3462 struct amdgpu_bo *bo; 3463 u32 gen_count; 3464 3465 /* Check if the connection is established. If it is not, then start 3466 * connection setup. 3467 */ 3468 gen_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0); 3469 if (!gen_count) { 3470 dev_warn(adev->dev, 3471 "NPA-REQ: no connection with remote AccId:%u\n", 3472 remote_acc_id); 3473 goto conn_setup; 3474 } 3475 3476 /* Check entry exists in Exporter XA. If yes, increase the refcount 3477 * for the node. 3478 */ 3479 xa_lock(&adev->ualink.exp_xa); 3480 exp_xa_node = xa_load(&adev->ualink.exp_xa, handle.handle_lo); 3481 if (!exp_xa_node || (handle.handle_hi != exp_xa_node->handle.handle_hi) || 3482 !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) { 3483 xa_unlock(&adev->ualink.exp_xa); 3484 dev_warn(adev->dev, 3485 "NPA-REQ: exp XA handle not found handle:%llx:%llx\n", 3486 handle.handle_hi, handle.handle_lo); 3487 fail_reason = AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE; 3488 goto handle_invalid_fail; 3489 } 3490 xa_unlock(&adev->ualink.exp_xa); 3491 3492 bo = exp_xa_node->bo; 3493 size = amdgpu_bo_ngpu_pages(bo); 3494 3495 /* Pin the BO */ 3496 r = amdgpu_bo_reserve(bo, true); 3497 if (unlikely(r)) { 3498 dev_warn(adev->dev, 3499 "NPA-REQ: BO reserve failed handle:%llx:%llx\n", 3500 handle.handle_hi, handle.handle_lo); 3501 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3502 goto bo_reserve_fail; 3503 } 3504 r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM); 3505 amdgpu_bo_unreserve(bo); 3506 if (r) { 3507 dev_warn(adev->dev, 3508 "NPA-REQ: BO pin failed handle:%llx:%llx\n", 3509 handle.handle_hi, handle.handle_lo); 3510 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3511 goto bo_pin_fail; 3512 } 3513 3514 if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 3515 mutex_lock(&exp_xa_node->node_lock); 3516 importer_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3517 npa_addr_entry = importer_entry; 3518 mutex_unlock(&exp_xa_node->node_lock); 3519 /* Check if NPA address is already allocated for this importer. 3520 * If yes, then send the NPA-FAIL message back to the remote GPU. 3521 */ 3522 if (importer_entry->npa_addr) { 3523 fail_reason = AMDGPU_UALINK_NPA_FAIL_DUPLICATE; 3524 goto npa_duplicate_fail; 3525 } 3526 3527 range_start = ((u64)remote_acc_id << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT) | 3528 AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED; 3529 range_end = range_start | AMDGPU_UALINK_NPA_ADDR_RANGE_MASK; 3530 } else { 3531 /* We store NPA-address in importer_entries[0] in 3532 * Source-Aliasing mode. 3533 */ 3534 mutex_lock(&exp_xa_node->node_lock); 3535 npa_addr_entry = &exp_xa_node->importer_entries[0]; 3536 importer_entry = &exp_xa_node->importer_entries[remote_acc_id]; 3537 /* Check if NPA address is already allocated for this importer. 3538 * If yes, then set the corresponding bit in the importers_bitmap, 3539 * set the generation count and send the NPA-RSP back to the remote GPU. 3540 */ 3541 if (npa_addr_entry->npa_addr) { 3542 npa_addr = npa_addr_entry->npa_addr; 3543 set_bit(remote_acc_id, exp_xa_node->importers_bitmap); 3544 importer_entry->generation_count = gen_count; 3545 mutex_unlock(&exp_xa_node->node_lock); 3546 dev_dbg(adev->dev, 3547 "NPA-REQ: NPA:%llx size:%llx handle:%llx:%llx\n", 3548 npa_addr, size, handle.handle_hi, handle.handle_lo); 3549 3550 goto send_npa_rsp; 3551 } 3552 mutex_unlock(&exp_xa_node->node_lock); 3553 3554 range_start = 0; 3555 range_end = 0; 3556 } 3557 3558 mm_node = kzalloc_obj(*mm_node); 3559 if (!mm_node) { 3560 dev_warn(adev->dev, 3561 "NPA-REQ: mm_node alloc failed handle:%llx:%llx\n", 3562 handle.handle_hi, handle.handle_lo); 3563 fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE; 3564 goto mem_alloc_fail; 3565 } 3566 3567 /* Allocate NPA address */ 3568 r = amdgpu_ualink_npa_alloc_va(adev, mm_node, 0, range_start, 3569 range_end, size); 3570 if (r) { 3571 dev_warn(adev->dev, 3572 "NPA-REQ: NPA addr alloc failed handle:%llx:%llx\n", 3573 handle.handle_hi, handle.handle_lo); 3574 fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE; 3575 goto npa_alloc_fail; 3576 } 3577 npa_addr = mm_node->start; 3578 3579 pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 0); 3580 dev_dbg(adev->dev, 3581 "NPA-REQ: Allocated NPA:%llx size:%llx PTE:%llx handle:%llx:%llx\n", 3582 npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo); 3583 3584 /* Map the NPA address into NPA VM*/ 3585 r = amdgpu_ualink_map_npa_addr(adev, npa_addr, size, bo, 0, pte_flags); 3586 if (r) { 3587 fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR; 3588 dev_warn(adev->dev, 3589 "NPA-REQ: NPA addr (%llx) map failed handle:%llx:%llx\n", 3590 npa_addr, handle.handle_hi, handle.handle_lo); 3591 goto map_npa_fail; 3592 } 3593 3594 dev_dbg(adev->dev, 3595 "NPA-REQ: Mapped NPA:%llx size:%llx pte:%llx handle:%llx:%llx\n", 3596 npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo); 3597 send_npa_rsp: 3598 /* Send NPA-RSP back to the remote GPU */ 3599 r = amdgpu_ualink_send_npa_rsp_msg(adev, remote_acc_id, handle, 3600 STRIP_NPA(npa_addr), size); 3601 if (r) { 3602 dev_warn(adev->dev, 3603 "NPA-REQ: send NPA-RSP failed remote:%u handle:%llx:%llx\n", 3604 remote_acc_id, handle.handle_hi, handle.handle_lo); 3605 send_npa_fail = false; 3606 goto send_npa_rsp_fail; 3607 } 3608 3609 dev_dbg(adev->dev, 3610 "NPA-REQ: Sent NPA-RSP with NPA:%llx size:%llx handle:%llx:%llx\n", 3611 npa_addr, size, handle.handle_hi, handle.handle_lo); 3612 3613 /* If this is the first time we are setting the bit for this importer, 3614 * then store the NPA address, mm_node and generation count. 3615 */ 3616 mutex_lock(&exp_xa_node->node_lock); 3617 if (!test_and_set_bit(remote_acc_id, exp_xa_node->importers_bitmap)) { 3618 npa_addr_entry->npa_addr = npa_addr; 3619 npa_addr_entry->mm_node = mm_node; 3620 importer_entry->generation_count = gen_count; 3621 } 3622 mutex_unlock(&exp_xa_node->node_lock); 3623 3624 dev_dbg(adev->dev, 3625 "NPA-REQ: BO pin_count:%d, importers:%d, handle:%llx:%llx\n", 3626 bo->tbo.pin_count, bitmap_weight(exp_xa_node->importers_bitmap, 3627 AMDGPU_UALINK_ACCEL_MAX), handle.handle_hi, handle.handle_lo); 3628 WARN_ON(bo->tbo.pin_count < bitmap_weight(exp_xa_node->importers_bitmap, 3629 AMDGPU_UALINK_ACCEL_MAX)); 3630 3631 /* Add this node to the exported handles list for the remote GPU, 3632 * but only if the node is still in exp_xa. If revoke already erased 3633 * it, skip the list_add to avoid a dangling list entry. The cleanup 3634 * worker is guaranteed to run after we drop our ref, so it will see 3635 * this importer in the bitmap and send NPA-REVOKE. 3636 */ 3637 xa_lock(&adev->ualink.exp_xa); 3638 if (xa_load(&adev->ualink.exp_xa, exp_xa_node->handle.handle_lo) == exp_xa_node) 3639 list_add(&importer_entry->list, &adev->ualink.exp_handles_list[remote_acc_id]); 3640 xa_unlock(&adev->ualink.exp_xa); 3641 3642 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3643 3644 return; 3645 3646 send_npa_rsp_fail: 3647 mutex_lock(&exp_xa_node->node_lock); 3648 clear_bit(remote_acc_id, exp_xa_node->importers_bitmap); 3649 mutex_unlock(&exp_xa_node->node_lock); 3650 if (mm_node) 3651 amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size); 3652 3653 map_npa_fail: 3654 if (mm_node) 3655 amdgpu_ualink_npa_free_va(adev, mm_node); 3656 3657 npa_alloc_fail: 3658 kfree(mm_node); 3659 mem_alloc_fail: 3660 npa_duplicate_fail: 3661 r = amdgpu_bo_reserve(bo, true); 3662 if (likely(!r)) { 3663 amdgpu_bo_unpin(bo); 3664 amdgpu_bo_unreserve(bo); 3665 } else { 3666 dev_warn(adev->dev, 3667 "NPA-REQ: BO reserve to unpin failed for handle:%llx:%llx\n", 3668 handle.handle_hi, handle.handle_lo); 3669 } 3670 3671 bo_pin_fail: 3672 bo_reserve_fail: 3673 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3674 3675 handle_invalid_fail: 3676 if (send_npa_fail) { 3677 r = amdgpu_ualink_send_npa_fail_msg(adev, remote_acc_id, 3678 handle, fail_reason); 3679 if (r) 3680 dev_warn(adev->dev, 3681 "NPA-REQ: send NPA-FAIL failed remote:%u handle:%llx:%llx\n", 3682 remote_acc_id, handle.handle_hi, handle.handle_lo); 3683 } 3684 return; 3685 conn_setup: 3686 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3687 if (r) 3688 dev_warn(adev->dev, 3689 "NPA-REQ: connection setup failed with remote AccId:%u\n", 3690 remote_acc_id); 3691 } 3692 3693 static int amdgpu_ualink_translate_npa_fail_reason(struct amdgpu_device *adev, 3694 u32 fail_reason) 3695 { 3696 switch (fail_reason) { 3697 case AMDGPU_UALINK_NPA_FAIL_NOSPACE: 3698 return -ENOSPC; 3699 case AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE: 3700 case AMDGPU_UALINK_NPA_FAIL_DUPLICATE: 3701 case AMDGPU_UALINK_NPA_FAIL_ERROR: 3702 return -EINVAL; 3703 default: 3704 dev_err(adev->dev, 3705 "IMPORT: invalid NPA-FAIL reason:%u\n", 3706 fail_reason); 3707 return -EINVAL; 3708 } 3709 } 3710 3711 static int amdgpu_ualink_do_import_handle(struct amdgpu_device *adev, 3712 struct amdgpu_ualink_imp_xa_node *imp_xa_node, 3713 u32 remote_acc_id) 3714 { 3715 struct amdgpu_ualink_handle handle = imp_xa_node->handle; 3716 u32 generation_count; 3717 int r; 3718 3719 /* First check if the connection is setup with the 3720 * remote GPU. If yes, then initiate the NPA protocol to 3721 * get the NPA address. 3722 * If not, then initiate the HELLO protocol to first setup 3723 * the connection and once the connection is setup, then 3724 * initiate the NPA protocol. 3725 */ 3726 r = amdgpu_ualink_setup_connection(adev, remote_acc_id); 3727 if (r) { 3728 if (r != -EAGAIN) 3729 dev_warn(adev->dev, 3730 "IMPORT: connection setup failed with remote AccId:%u\n", 3731 remote_acc_id); 3732 return r; 3733 } 3734 3735 /* Send NPA_REQ message */ 3736 r = amdgpu_ualink_send_npa_req_msg(adev, remote_acc_id, handle); 3737 if (r) { 3738 dev_warn(adev->dev, 3739 "IMPORT: NPA-REQ send failed to remote AccId:%u\n", 3740 remote_acc_id); 3741 goto reset_conn; 3742 } 3743 3744 /* Wait for the NPA_RSP to come back */ 3745 r = wait_for_completion_interruptible_timeout(&imp_xa_node->npa_done, 3746 msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); 3747 if (r == -ERESTARTSYS) { 3748 dev_err_ratelimited(adev->dev, 3749 "IMPORT: NPA-RSP wait interrupted by signal\n"); 3750 return r; 3751 } else if (r == 0) { 3752 dev_warn(adev->dev, 3753 "IMPORT: NPA-RSP timeout from remote AccId:%u\n", 3754 remote_acc_id); 3755 r = -ETIMEDOUT; 3756 goto reset_conn; 3757 } 3758 3759 /* If the NPA addr/size isn't filled with valid values, then either 3760 * we got a NPA_FAIL or something bad happened. In either case, we 3761 * return the error back to user-space. 3762 */ 3763 if (imp_xa_node->fail_reason) { 3764 dev_warn(adev->dev, 3765 "IMPORT: NPA-REQ failed with fail_reason:%d handle:%llx:%llx\n", 3766 imp_xa_node->fail_reason, handle.handle_hi, handle.handle_lo); 3767 return amdgpu_ualink_translate_npa_fail_reason(adev, 3768 imp_xa_node->fail_reason); 3769 } 3770 3771 if (!imp_xa_node->npa_addr || !imp_xa_node->size) { 3772 dev_warn(adev->dev, 3773 "IMPORT: invalid npa:%llx or size:%llx\n", 3774 imp_xa_node->npa_addr, imp_xa_node->size); 3775 imp_xa_node->npa_addr = 0; 3776 imp_xa_node->size = 0; 3777 return -EINVAL; 3778 } 3779 3780 r = amdgpu_ualink_map_npa_to_dmabuf(adev, imp_xa_node); 3781 if (r) { 3782 dev_warn(adev->dev, 3783 "IMPORT: dmabuf creation failed npa:%llx size:%llx\n", 3784 imp_xa_node->npa_addr, imp_xa_node->size); 3785 imp_xa_node->npa_addr = 0; 3786 imp_xa_node->size = 0; 3787 return r; 3788 } 3789 3790 /* Add this node to the imported handles list for the remote GPU, 3791 * unless the exporter revoked the handle while the import was in 3792 * flight. The dmabuf is released with the last node reference. 3793 */ 3794 xa_lock(&adev->ualink.imp_xa); 3795 if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_TEARDOWN) { 3796 xa_unlock(&adev->ualink.imp_xa); 3797 dev_dbg(adev->dev, 3798 "IMPORT: handle:%llx:%llx revoked during import\n", 3799 handle.handle_hi, handle.handle_lo); 3800 return -EINVAL; 3801 } 3802 list_add(&imp_xa_node->list, &adev->ualink.imp_handles_list[remote_acc_id]); 3803 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_READY); 3804 xa_unlock(&adev->ualink.imp_xa); 3805 3806 return 0; 3807 3808 reset_conn: 3809 dev_dbg(adev->dev, 3810 "IMPORT: Resetting connection for remote:%u\n", remote_acc_id); 3811 generation_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0); 3812 amdgpu_ualink_handle_connection_reset(adev, remote_acc_id, 3813 AMDGPU_UALINK_CONN_NOT_READY, 3814 generation_count); 3815 return r; 3816 } 3817 3818 static int amdgpu_ualink_local_import(struct amdgpu_device *adev, 3819 u32 remote_acc_id, 3820 struct amdgpu_ualink_handle *handle, 3821 int *fd_out) 3822 { 3823 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 3824 struct amdgpu_device *peer_adev; 3825 int fd; 3826 3827 mutex_lock(&mgpu_info.mutex); 3828 peer_adev = find_peer_adev(remote_acc_id); 3829 mutex_unlock(&mgpu_info.mutex); 3830 if (!peer_adev) { 3831 dev_err(adev->dev, 3832 "IMPORT LOCAL: peer adev not found for AccId:%u\n", 3833 remote_acc_id); 3834 return -ENODEV; 3835 } 3836 3837 xa_lock(&peer_adev->ualink.exp_xa); 3838 exp_xa_node = xa_load(&peer_adev->ualink.exp_xa, handle->handle_lo); 3839 if (!exp_xa_node || 3840 exp_xa_node->handle.handle_hi != handle->handle_hi || 3841 !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) { 3842 xa_unlock(&peer_adev->ualink.exp_xa); 3843 dev_err(adev->dev, 3844 "IMPORT LOCAL: handle:%llx:%llx not found in peer exp_xa\n", 3845 handle->handle_hi, handle->handle_lo); 3846 return -EINVAL; 3847 } 3848 xa_unlock(&peer_adev->ualink.exp_xa); 3849 3850 get_dma_buf(exp_xa_node->dmabuf); 3851 /* Get a new fd for the DMABuf */ 3852 fd = dma_buf_fd(exp_xa_node->dmabuf, O_CLOEXEC | O_RDWR); 3853 if (fd < 0) { 3854 dev_err(adev->dev, 3855 "IMPORT LOCAL: dma-buf fd failed handle:%llx:%llx\n", 3856 handle->handle_hi, handle->handle_lo); 3857 dma_buf_put(exp_xa_node->dmabuf); 3858 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3859 return fd; 3860 } 3861 3862 amdgpu_ualink_exp_xa_entry_put(exp_xa_node); 3863 *fd_out = fd; 3864 return 0; 3865 } 3866 3867 int amdgpu_ualink_import_handle(struct drm_device *dev, 3868 const struct amdgpu_ualink_handle *ualink_handle, 3869 int *fd_out) 3870 { 3871 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 3872 struct amdgpu_device *adev = drm_to_adev(dev); 3873 struct amdgpu_ualink_handle handle = *ualink_handle; 3874 u32 remote_acc_id, node_state; 3875 int r = 0, fd; 3876 3877 remote_acc_id = (handle.handle_lo & 3878 AMDGPU_UALINK_HANDLE_ACCID_MASK); 3879 3880 if (remote_acc_id >= AMDGPU_UALINK_ACCEL_MAX) { 3881 dev_err(adev->dev, 3882 "IMPORT: invalid remote AccId:%u\n", remote_acc_id); 3883 return -EINVAL; 3884 } 3885 3886 if (amdgpu_ualink_is_local_accel(adev, remote_acc_id)) 3887 return amdgpu_ualink_local_import(adev, remote_acc_id, 3888 &handle, fd_out); 3889 3890 xa_lock(&adev->ualink.imp_xa); 3891 imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo); 3892 3893 if (imp_xa_node) { 3894 /* If node state is Not_ready/Pending, then some other 3895 * thread is already trying the NPA protocol for the same 3896 * ualink handle. Back off and let the thread finish. 3897 * If the node state is Teardown, then it means this 3898 * node is about to be removed. So let user-space know 3899 * that this handle is invalid. 3900 */ 3901 node_state = READ_ONCE(imp_xa_node->node_state); 3902 if (node_state == AMDGPU_UALINK_NODE_NOT_READY || 3903 node_state == AMDGPU_UALINK_NODE_PENDING) { 3904 xa_unlock(&adev->ualink.imp_xa); 3905 r = -EAGAIN; 3906 goto out; 3907 } else if (node_state == AMDGPU_UALINK_NODE_TEARDOWN) { 3908 xa_unlock(&adev->ualink.imp_xa); 3909 r = -EINVAL; 3910 goto out; 3911 } 3912 3913 /* Increase the refcount while we are processing the request */ 3914 r = amdgpu_ualink_imp_xa_entry_get(imp_xa_node) ? 0 : -EINVAL; 3915 xa_unlock(&adev->ualink.imp_xa); 3916 3917 /* If the refcount has become 0 but the entry is not yet 3918 * removed from the Xarray, then return error to user-space. 3919 */ 3920 if (r) 3921 goto out; 3922 } else { 3923 xa_unlock(&adev->ualink.imp_xa); 3924 /* if the partial handle doesn't exist in the Importer xarray then 3925 * initiate the NPA protocol and generate the DMABuf corresponding 3926 * to the NPA address. 3927 * First store the entry in the Xarray. 3928 */ 3929 imp_xa_node = kzalloc_obj(*imp_xa_node); 3930 if (!imp_xa_node) { 3931 r = -ENOMEM; 3932 goto out; 3933 } 3934 imp_xa_node->adev = adev; 3935 imp_xa_node->node_state = AMDGPU_UALINK_NODE_NOT_READY; 3936 imp_xa_node->handle = handle; 3937 init_completion(&imp_xa_node->npa_done); 3938 INIT_LIST_HEAD(&imp_xa_node->list); 3939 kref_init(&imp_xa_node->refcount); 3940 3941 /* Take an extra reference to store in the Xarray. The error 3942 * handling paths will drop both these references, while a 3943 * successful path will drop only one reference to the Xarray 3944 * entry. 3945 */ 3946 amdgpu_ualink_imp_xa_entry_get(imp_xa_node); 3947 3948 /* Check if another thread created a node for the same handle while 3949 * we were trying to create and initialize the node. 3950 */ 3951 r = xa_insert(&adev->ualink.imp_xa, handle.handle_lo, 3952 imp_xa_node, GFP_KERNEL); 3953 if (r) { 3954 kfree(imp_xa_node); 3955 /* -EBUSY means another thread raced us and inserted a 3956 * node for the same handle. Ask user-space to retry. 3957 */ 3958 if (r == -EBUSY) 3959 r = -EAGAIN; 3960 else 3961 dev_err(adev->dev, 3962 "IMPORT: XA insert failed for handle:%llx:%llx err:%d\n", 3963 handle.handle_hi, handle.handle_lo, r); 3964 goto out; 3965 } 3966 3967 r = amdgpu_ualink_do_import_handle(adev, imp_xa_node, remote_acc_id); 3968 3969 /* If error is returned, then cleanup the xarray entry before returning 3970 * the error back to user-space 3971 */ 3972 if (r) { 3973 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 3974 if (r != -EAGAIN) 3975 dev_err(adev->dev, 3976 "IMPORT: XA import failed for handle:%llx:%llx\n", 3977 handle.handle_hi, handle.handle_lo); 3978 goto cleanup; 3979 } 3980 } 3981 3982 /* dma_buf_fd consumes a reference and assigns it to the fd. 3983 * Therefore take an extra reference to be consumed. It will be 3984 * released when user mode closes the fd. 3985 */ 3986 get_dma_buf(imp_xa_node->dmabuf); 3987 3988 fd = dma_buf_fd(imp_xa_node->dmabuf, O_CLOEXEC | O_RDWR); 3989 if (fd >= 0) { 3990 *fd_out = fd; 3991 } else { 3992 dma_buf_put(imp_xa_node->dmabuf); 3993 r = fd; 3994 dev_err(adev->dev, 3995 "IMPORT: dma-buf fd creation failed handle:%llx:%llx\n", 3996 handle.handle_hi, handle.handle_lo); 3997 } 3998 3999 cleanup: 4000 amdgpu_ualink_imp_xa_entry_put(imp_xa_node); 4001 out: 4002 return r; 4003 } 4004 4005 int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp, 4006 u32 gem_handle, 4007 struct amdgpu_ualink_handle *handle_out) 4008 { 4009 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 4010 struct amdgpu_ualink_handle handle; 4011 struct drm_gem_object *gobj; 4012 struct amdgpu_device *adev; 4013 struct amdgpu_bo *robj; 4014 int r = 0, i; 4015 4016 gobj = drm_gem_object_lookup(filp, gem_handle); 4017 if (!gobj) 4018 return -ENOENT; 4019 4020 robj = gem_to_amdgpu_bo(gobj); 4021 adev = amdgpu_ttm_adev(robj->tbo.bdev); 4022 4023 if (!(robj->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { 4024 dev_err(adev->dev, "Only VRAM BOs can be exported\n"); 4025 r = -EOPNOTSUPP; 4026 goto out; 4027 } 4028 4029 if (!robj->ualink_handle_lo) { 4030 /* If no ualink handle generated for BO, then generate one and 4031 * add it to the exporter Xarray. 4032 */ 4033 exp_xa_node = kzalloc_obj(*exp_xa_node); 4034 if (!exp_xa_node) { 4035 dev_err(adev->dev, "Failed to allocate exp_xa_node\n"); 4036 r = -ENOMEM; 4037 goto out; 4038 } 4039 4040 amdgpu_bo_ref(robj); 4041 exp_xa_node->bo = robj; 4042 init_completion(&exp_xa_node->npa_done); 4043 bitmap_zero(exp_xa_node->importers_bitmap, 4044 AMDGPU_UALINK_ACCEL_MAX); 4045 bitmap_zero(exp_xa_node->npa_release_bitmap, 4046 AMDGPU_UALINK_ACCEL_MAX); 4047 kref_init(&exp_xa_node->refcount); 4048 mutex_init(&exp_xa_node->node_lock); 4049 INIT_WORK(&exp_xa_node->cleanup_work, 4050 amdgpu_ualink_exp_cleanup_worker); 4051 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4052 INIT_LIST_HEAD(&exp_xa_node->importer_entries[i].list); 4053 exp_xa_node->importer_entries[i].parent = exp_xa_node; 4054 } 4055 /* DMABuf handle for local import of fabric handles */ 4056 exp_xa_node->dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, filp, 4057 gem_handle, DRM_CLOEXEC | DRM_RDWR); 4058 if (IS_ERR(exp_xa_node->dmabuf)) { 4059 r = PTR_ERR(exp_xa_node->dmabuf); 4060 dev_err(adev->dev, "Failed to generate DMABuf for the BO\n"); 4061 goto err_unref_bo; 4062 } 4063 4064 xa_lock(&adev->ualink.exp_xa); 4065 amdgpu_generate_ualink_handle(adev, &handle); 4066 exp_xa_node->handle = handle; 4067 r = __xa_insert(&adev->ualink.exp_xa, handle.handle_lo, 4068 exp_xa_node, GFP_KERNEL); 4069 xa_unlock(&adev->ualink.exp_xa); 4070 if (r) { 4071 dev_err(adev->dev, "Failed to insert exp_xa_node into XA: %d\n", r); 4072 goto err_put_dmabuf; 4073 } 4074 4075 r = amdgpu_bo_reserve(robj, false); 4076 if (r) 4077 goto err_erase_xa; 4078 4079 if (robj->ualink_handle_lo) { 4080 amdgpu_bo_unreserve(robj); 4081 r = -EAGAIN; 4082 goto err_erase_xa; 4083 } 4084 4085 robj->ualink_handle_lo = handle.handle_lo; 4086 amdgpu_bo_unreserve(robj); 4087 4088 /* Return the generated handle back to the caller */ 4089 *handle_out = handle; 4090 } else { 4091 handle_out->handle_lo = robj->ualink_handle_lo; 4092 4093 /* Do a sanity check to ensure the handle exists in the XA */ 4094 xa_lock(&adev->ualink.exp_xa); 4095 exp_xa_node = xa_load(&adev->ualink.exp_xa, 4096 robj->ualink_handle_lo); 4097 xa_unlock(&adev->ualink.exp_xa); 4098 WARN(!exp_xa_node, "Exp XA: Handle_Lo: %llx not found", 4099 robj->ualink_handle_lo); 4100 if (exp_xa_node) 4101 handle_out->handle_hi = exp_xa_node->handle.handle_hi; 4102 } 4103 4104 drm_gem_object_put(gobj); 4105 return 0; 4106 4107 err_erase_xa: 4108 xa_erase(&adev->ualink.exp_xa, handle.handle_lo); 4109 err_put_dmabuf: 4110 dma_buf_put(exp_xa_node->dmabuf); 4111 err_unref_bo: 4112 amdgpu_bo_unref(&exp_xa_node->bo); 4113 kfree(exp_xa_node); 4114 out: 4115 drm_gem_object_put(gobj); 4116 return r; 4117 } 4118 4119 int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data, 4120 struct drm_file *filp) 4121 { 4122 union drm_amdgpu_ualink_handle *args = data; 4123 struct amdgpu_device *adev = drm_to_adev(dev); 4124 struct amdgpu_ualink_handle handle = {}; 4125 u32 gem_handle; 4126 int r, fd = -1; 4127 4128 /* The ioctl is registered for every amdgpu device, but the UALink 4129 * software state only exists on ASICs that instantiate the UALink IP 4130 * block. Bail out if it is absent, and short-circuit before the 4131 * accel_state dereference below. 4132 */ 4133 if (!adev->ualink.info || 4134 adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) { 4135 dev_dbg(adev->dev, 4136 "ualink device not available or not in active state\n"); 4137 return -EOPNOTSUPP; 4138 } 4139 4140 /* The input and output members of the ioctl argument alias each other. 4141 * Latch every input field before invoking the handlers, and only write 4142 * the output fields afterwards. 4143 */ 4144 switch (args->in.op) { 4145 case DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT: 4146 gem_handle = args->in.gem_handle; 4147 r = amdgpu_ualink_export_handle(dev, filp, gem_handle, &handle); 4148 if (!r) { 4149 args->out.export_ualink_handle[0] = handle.handle_lo; 4150 args->out.export_ualink_handle[1] = handle.handle_hi; 4151 } 4152 break; 4153 case DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT: 4154 handle.handle_lo = args->in.import_ualink_handle[0]; 4155 handle.handle_hi = args->in.import_ualink_handle[1]; 4156 r = amdgpu_ualink_import_handle(dev, &handle, &fd); 4157 if (!r) 4158 args->out.import_dmabuf_handle = fd; 4159 break; 4160 default: 4161 r = -EINVAL; 4162 break; 4163 } 4164 4165 return r; 4166 } 4167 4168 int amdgpu_ualink_manager_start(struct amdgpu_device *adev) 4169 { 4170 int i, r; 4171 4172 r = amdgpu_vm_init(adev, &adev->ualink.npa_vm, 0); 4173 if (r) 4174 goto out; 4175 4176 /* For pinning page tables and using CPU for page table updates. */ 4177 r = amdgpu_vm_make_npa(adev, &adev->ualink.npa_vm); 4178 if (r) 4179 goto uninit_vm; 4180 4181 adev->ualink.npa_wq = alloc_workqueue("NPA WQ", WQ_UNBOUND, 0); 4182 if (unlikely(!adev->ualink.npa_wq)) { 4183 dev_err(adev->dev, "Failed to allocate NPA WQ\n"); 4184 r = -ENOMEM; 4185 goto uninit_vm; 4186 } 4187 4188 /* Map this VM to NPA VMID */ 4189 adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 4190 amdgpu_gmc_pd_addr(adev->ualink.npa_vm.root.bo)); 4191 4192 xa_init(&adev->ualink.exp_xa); 4193 xa_init(&adev->ualink.imp_xa); 4194 xa_init(&adev->ualink.handle_invalid_xa); 4195 4196 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4197 init_completion(&adev->ualink.conn_state[i].hello_done); 4198 mutex_init(&adev->ualink.conn_state[i].lock); 4199 adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY; 4200 INIT_LIST_HEAD(&adev->ualink.exp_handles_list[i]); 4201 INIT_LIST_HEAD(&adev->ualink.imp_handles_list[i]); 4202 } 4203 4204 amdgpu_ualink_npa_mm_init(adev); 4205 4206 return 0; 4207 4208 uninit_vm: 4209 amdgpu_vm_fini(adev, &adev->ualink.npa_vm); 4210 out: 4211 return r; 4212 } 4213 4214 /* Free every importer entry left in imp_xa at manager stop. */ 4215 static void amdgpu_ualink_teardown_imp_xa_entries(struct amdgpu_device *adev) 4216 { 4217 struct amdgpu_ualink_imp_xa_node *imp_xa_node; 4218 unsigned long index; 4219 4220 xa_for_each(&adev->ualink.imp_xa, index, imp_xa_node) { 4221 list_del_init(&imp_xa_node->list); 4222 WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN); 4223 amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node); 4224 } 4225 } 4226 4227 /* Free a single exporter node at manager stop. This is the message-free 4228 * portion of amdgpu_ualink_exp_cleanup_worker(). 4229 */ 4230 static void amdgpu_ualink_teardown_exp_xa_node(struct amdgpu_device *adev, 4231 struct amdgpu_ualink_exp_xa_node *exp_xa_node) 4232 { 4233 DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); 4234 struct amdgpu_bo *bo = exp_xa_node->bo; 4235 u32 remote_acc_id; 4236 4237 bitmap_copy(importers_bitmap, exp_xa_node->importers_bitmap, 4238 AMDGPU_UALINK_ACCEL_MAX); 4239 4240 dev_dbg(adev->dev, 4241 "EXP-STOP: handle:%llx:%llx importers bitmap: %*pbl\n", 4242 exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, 4243 AMDGPU_UALINK_ACCEL_MAX, importers_bitmap); 4244 4245 if (!bitmap_empty(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)) { 4246 amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node); 4247 4248 /* Unlink importers and unpin the BO once per importer. */ 4249 if (likely(!amdgpu_bo_reserve(bo, true))) { 4250 bo->ualink_handle_lo = 0ULL; 4251 for_each_set_bit(remote_acc_id, importers_bitmap, 4252 AMDGPU_UALINK_ACCEL_MAX) { 4253 list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list); 4254 amdgpu_bo_unpin(bo); 4255 } 4256 amdgpu_bo_unreserve(bo); 4257 } else { 4258 dev_warn(adev->dev, 4259 "EXP-STOP: BO reserve to unpin failed handle:%llx:%llx\n", 4260 exp_xa_node->handle.handle_hi, 4261 exp_xa_node->handle.handle_lo); 4262 for_each_set_bit(remote_acc_id, importers_bitmap, 4263 AMDGPU_UALINK_ACCEL_MAX) 4264 list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list); 4265 } 4266 4267 amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, importers_bitmap); 4268 } 4269 4270 /* Release the export dma-buf and drop the BO ref. */ 4271 dma_buf_put(exp_xa_node->dmabuf); 4272 amdgpu_bo_unref(&bo); 4273 exp_xa_node->bo = NULL; 4274 4275 mutex_destroy(&exp_xa_node->node_lock); 4276 kfree(exp_xa_node); 4277 } 4278 4279 /* Free every exporter entry left in exp_xa at manager stop. These were 4280 * exported but never revoked, so no cleanup worker was queued for them. 4281 */ 4282 static void amdgpu_ualink_teardown_exp_xa_entries(struct amdgpu_device *adev) 4283 { 4284 struct amdgpu_ualink_exp_xa_node *exp_xa_node; 4285 unsigned long index; 4286 4287 xa_for_each(&adev->ualink.exp_xa, index, exp_xa_node) { 4288 xa_erase(&adev->ualink.exp_xa, index); 4289 amdgpu_ualink_teardown_exp_xa_node(adev, exp_xa_node); 4290 } 4291 } 4292 4293 void amdgpu_ualink_manager_stop(struct amdgpu_device *adev) 4294 { 4295 int i; 4296 4297 adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 0); 4298 4299 /* Mark connections down so the drained workers and the teardown below 4300 * take the message-free path in amdgpu_ualink_check_conn_ready(). 4301 */ 4302 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) { 4303 mutex_lock(&adev->ualink.conn_state[i].lock); 4304 adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY; 4305 mutex_unlock(&adev->ualink.conn_state[i].lock); 4306 } 4307 4308 /* Drain in-flight exporter cleanup work so revoked nodes in 4309 * handle_invalid_xa free themselves and empty that xarray. 4310 */ 4311 drain_workqueue(adev->ualink.npa_wq); 4312 4313 /* Free live entries before the NPA allocator is torn down, so NPA 4314 * addresses can still be unmapped and freed. 4315 */ 4316 amdgpu_ualink_teardown_imp_xa_entries(adev); 4317 amdgpu_ualink_teardown_exp_xa_entries(adev); 4318 4319 amdgpu_ualink_npa_mm_fini(adev); 4320 4321 /* All three xarrays are empty by now. */ 4322 xa_destroy(&adev->ualink.exp_xa); 4323 xa_destroy(&adev->ualink.imp_xa); 4324 xa_destroy(&adev->ualink.handle_invalid_xa); 4325 4326 for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) 4327 mutex_destroy(&adev->ualink.conn_state[i].lock); 4328 4329 destroy_workqueue(adev->ualink.npa_wq); 4330 amdgpu_vm_fini(adev, &adev->ualink.npa_vm); 4331 } 4332 4333 /* 4334 * UALink remote interrupt and shootdown 4335 */ 4336 4337 /* UALINK ring buffer size, same for both remote shootdown and interrupt ring */ 4338 #define AMDGPU_UALINK_RB_SIZE 4096 4339 4340 #define AMDGPU_UALINK_METADATA_HEADER 0x4E485446 4341 4342 /* 2MB NPA start address for 2MB page mapping */ 4343 #define AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET SZ_2M 4344 4345 struct amdgpu_ualink_metadata { 4346 u32 header; 4347 4348 /* 4349 * "ring entries" as unit. So the ring-size-in-bytes could be calculated 4350 * as entry-size * 2^RBsize. This would make the minimum size a single 4351 * entry and the maximum size 32786 entries. 4352 * 4353 * u32 rb_size:4; 4354 * u32 reserved0:4; 4355 * u32 vmid:4; 4356 * u32 reserved1:20; 4357 */ 4358 u32 rb_size; 4359 4360 /* ring buffer base address for remote interrupt */ 4361 u64 ri_rb; 4362 4363 /* tlb invalidate ring buffer base address for remote shootdown */ 4364 u64 tlb_inv_rb; 4365 4366 /* remote interrupt Tail pointer, write pointer address */ 4367 u64 tailptr_ri; 4368 4369 /* TLB invalidate ring buffer's Tail pointer, write pointer address */ 4370 u64 tailptr_tlb_inv; 4371 }; 4372 4373 struct amdgpu_ualink_wb { 4374 /* last finished command seq number */ 4375 u32 data; 4376 4377 /* command complete error code */ 4378 u32 status; 4379 4380 /* ring rptr updated by FW */ 4381 u64 rptr; 4382 }; 4383 4384 /* 4385 * For remote interrupt and shootdown 4386 */ 4387 struct amdgpu_ualink_ring { 4388 u32 rb_size; 4389 4390 /* writeback data */ 4391 u32 seq; 4392 4393 /* local copy */ 4394 u64 wptr, rptr; 4395 4396 /* NPA gart mapping for SDMA */ 4397 u64 rb_npa_gart; 4398 u64 wptr_npa_gart; 4399 u64 doorbell_npa_gart; 4400 4401 /* gart mapping node */ 4402 struct drm_mm_node mm_node_rb; 4403 struct drm_mm_node mm_node_wptr; 4404 struct drm_mm_node mm_node_doorbell; 4405 4406 /* seq, complete status write back cpu address */ 4407 struct amdgpu_ualink_wb *wb_cpu; 4408 4409 /* true if fw write back address updated successfully */ 4410 bool ready; 4411 }; 4412 4413 struct amdgpu_ualink_peer { 4414 /* remote interrupt and shootdown uses same SDMA entity */ 4415 struct mutex lock; 4416 4417 /* SDMA engine to send remote command via NPA */ 4418 struct drm_sched_entity entity; 4419 4420 /* to select different DXS ports, cycles through different values */ 4421 u32 dxs_port; 4422 4423 struct amdgpu_ualink_ring interrupt; 4424 struct amdgpu_ualink_ring shootdown; 4425 }; 4426 4427 struct amdgpu_ualink_remote { 4428 /* ualink metadata passed to MPNHT FW */ 4429 struct amdgpu_bo *metadata_bo; 4430 u64 metadata_gpu_addr; 4431 void *metadata_cpu_addr; 4432 4433 /* ualink ring, tlb ring buffer, wptr */ 4434 struct amdgpu_bo *ring_bo; 4435 u64 rb_gpu_addr; 4436 void *rb_cpu_addr; 4437 4438 /* ualink rptr, wb data, statuss for address alias mode */ 4439 struct amdgpu_bo *rptr_bo; 4440 u64 rptr_gpu_addr; 4441 void *rptr_cpu_addr; 4442 u64 rptr_npa; 4443 4444 /* address alias mode alloc npa address for shared wb */ 4445 struct drm_mm_node rptr_mm_node; 4446 4447 /* 4448 * Owned snapshot of the vPod's active accelerator bitmap, taken at 4449 * sw_init time. Must NOT alias info->vpod.active_accel_bits, which 4450 * psp_ual_query_info() refreshes on every commit: aliasing it would let 4451 * teardown unmap a different set than setup mapped, leaking drm_mm nodes 4452 * in the shared GTT manager. 4453 */ 4454 DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); 4455 u32 num_accel; 4456 4457 /* remote GPUs ring buffer, read, write pointer local copy and gart mapping */ 4458 struct amdgpu_ualink_peer peer[AMDGPU_UALINK_ACCEL_MAX]; 4459 4460 /* use lsdma write to NPA address for remote interrupt */ 4461 bool use_lsdma; 4462 }; 4463 4464 static inline struct amdgpu_ualink_remote *to_remote(struct amdgpu_device *adev) 4465 { 4466 return adev->ualink.remote; 4467 } 4468 4469 /* 4470 * Returns true if the vPod membership most recently reported by firmware 4471 * (info->vpod.active_accel_bits, read back via psp_ual_query_info() on the 4472 * committing GPU) differs from the owned snapshot captured when this 4473 * accelerator was last activated (remote->active_accel_bits). Used to 4474 * decide whether an accelerator must be (re)built to match the new member 4475 * set. A NULL remote means the accelerator has never been activated (no 4476 * snapshot yet), which also counts as "changed" so it gets brought up. 4477 */ 4478 static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev) 4479 { 4480 struct amdgpu_ualink_remote *remote = to_remote(adev); 4481 4482 if (!remote) 4483 return true; 4484 4485 return !bitmap_equal(remote->active_accel_bits, 4486 adev->ualink.info->vpod.active_accel_bits, 4487 AMDGPU_UALINK_ACCEL_MAX); 4488 } 4489 4490 static inline u32 ualink_accel_id(struct amdgpu_device *adev) 4491 { 4492 return adev->ualink.info->ppod.accel_id; 4493 } 4494 4495 static inline enum amdgpu_ualink_addr_mode ualink_addr_mode(struct amdgpu_device *adev) 4496 { 4497 return adev->ualink.info->vpod.addr_mode; 4498 } 4499 4500 /* 4501 * address mode and NPA address for ring wptr, rptr, wb status, wb data 4502 * 4503 * source identification address mode 4504 * 4505 * we need to export different tail pointers for different remote GPU accel_id, 4506 * hence we need to allocate a whole page for each remote GPU separately. 4507 * 4508 * rptr, wb data, status share the whole page with wptr. 4509 * 4510 * 1 page reserved NPA address for 1 active remote GPU, max 255 pages 4511 * starting address: wptr NPA address + accel_id * 4K 4512 * offset: 4513 * remote interrupt ring wptr 0 4514 * remote shootdown ring wptr 8 4515 * remote interrupt wb status 16 4516 * remote interrupt wb data 20 4517 * remote interrupt ring rptr 24 4518 * remote shootdown wb status 32 4519 * remote shootdown wb data 36 4520 * remote shootdown ring rptr 40 4521 * 4522 * source aliasing address mode 4523 * 4524 * wptr for different remote GPU can share same reserved 1 page NPA address page. 4525 * 4526 * to save reserved NPA address space, rptr, wb data, status for different 4527 * remote GPU share same allocated 2 pages NPA address. 4528 * 4529 * 1 page reserved NPA address for wptr of all active remote GPUs 4530 * offset: 4531 * remote interrupt ring wptr accel_id * 16 4532 * remote shootdown ring wptr accel_id * 16 + 8 4533 * 4534 * 2 page allocated NPA address for rptr, writeback of all active remote GPUs 4535 * starting address: at accel_id * 32 for each remote GPU 4536 * offset: 4537 * remote interrupt wb status 0 4538 * remote interrupt wb data 4 4539 * remote interrupt ring rptr 8 4540 * remote shootdown wb status 16 4541 * remote shootdown wb data 20 4542 * remote shootdown ring rptr 24 4543 */ 4544 static inline u32 ualink_wptr_size(struct amdgpu_device *adev) 4545 { 4546 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4547 return AMDGPU_GPU_PAGE_SIZE; 4548 else 4549 return 0; /* wptr uses the ring buffer for itself */ 4550 } 4551 4552 static inline u32 ualink_wb_size(struct amdgpu_device *adev) 4553 { 4554 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4555 return 0; /* share same page with wptr */ 4556 else 4557 return 2 * sizeof(struct amdgpu_ualink_wb); 4558 } 4559 4560 static inline u32 ualink_wptr_offset(struct amdgpu_device *adev, u32 accel_id) 4561 { 4562 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4563 return 0; /* one separate page per GPU */ 4564 else 4565 return accel_id * 2 * sizeof(u64); /* two pointers per GPU */ 4566 } 4567 4568 static inline u32 ualink_tlb_wptr_offset(struct amdgpu_device *adev, u32 accel_id) 4569 { 4570 return ualink_wptr_offset(adev, accel_id) + sizeof(u64); 4571 } 4572 4573 static inline u32 ualink_wb_offset(struct amdgpu_device *adev, u32 accel_id) 4574 { 4575 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) 4576 return 2 * sizeof(u64); /* after two wptr pointers */ 4577 else 4578 return accel_id * ualink_wb_size(adev); 4579 } 4580 4581 static inline u32 ualink_tlb_wb_offset(struct amdgpu_device *adev, u32 accel_id) 4582 { 4583 return ualink_wb_offset(adev, accel_id) + sizeof(struct amdgpu_ualink_wb); 4584 } 4585 4586 static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, u32 flush_type) 4587 { 4588 u32 bit; 4589 4590 bit = AMDGPU_MMHUB0_START; 4591 4592 for_each_set_bit_from(bit, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) 4593 amdgpu_gmc_flush_gpu_tlb(adev, adev->vm_manager.npa_vmid, 4594 bit, flush_type); 4595 } 4596 4597 /** 4598 * amdgpu_ualink_npa_vm_map_range - Map a range in the NPA VM 4599 * @adev: amdgpu device pointer 4600 * @bo: buffer object backing the mapping 4601 * @pte_flags: page table entry flags 4602 * @offset: offset into the buffer object in bytes 4603 * @size_in_pages: size of the range to map in pages 4604 * @npa_in_pages: NPA target address in pages 4605 * 4606 * Maps a buffer object range into the NPA VM page table at the specified 4607 * NPA address. 4608 * 4609 * Return: 0 on success, negative error code on failure 4610 */ 4611 static int amdgpu_ualink_npa_vm_map_range(struct amdgpu_device *adev, struct amdgpu_bo *bo, 4612 u64 pte_flags, u64 offset, u64 size_in_pages, 4613 u64 npa_in_pages) 4614 { 4615 struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm; 4616 int r; 4617 4618 dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n", 4619 offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags, 4620 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4621 4622 r = amdgpu_vm_update_range(adev, npa_vm, false, false, true, 4623 false, NULL, npa_in_pages, 4624 npa_in_pages + size_in_pages - 1, 4625 pte_flags, offset, adev->vm_manager.vram_base_offset, 4626 bo->tbo.resource, NULL, &npa_vm->last_update); 4627 if (r) 4628 dev_dbg(adev->dev, "failed %d to map npa 0x%llx to NPA VM\n", r, 4629 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4630 return r; 4631 } 4632 4633 /** 4634 * amdgpu_npa_vm_unmap_range - Unmap a range from the NPA VM 4635 * @adev: amdgpu device pointer 4636 * @bo: buffer object backing the mapping 4637 * @pte_flags: page table entry flags 4638 * @offset: offset into the buffer object in bytes 4639 * @size_in_pages: size of the range to unmap in pages 4640 * @npa_in_pages: NPA target address in pages 4641 * 4642 * Removes a previously established mapping from the NPA VM page table. 4643 * 4644 * Return: 0 on success, negative error code on failure 4645 */ 4646 static int amdgpu_ualink_npa_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_bo *bo, 4647 u64 pte_flags, u64 offset, u64 size_in_pages, 4648 u64 npa_in_pages) 4649 { 4650 struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm; 4651 int r; 4652 4653 dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n", 4654 offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags, 4655 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4656 4657 r = amdgpu_vm_update_range(adev, npa_vm, false, false, true, 4658 false, NULL, npa_in_pages, 4659 npa_in_pages + size_in_pages - 1, 4660 pte_flags, offset, 0, bo->tbo.resource, NULL, 4661 &npa_vm->last_update); 4662 if (r) 4663 dev_dbg(adev->dev, "failed %d to unmap npa 0x%llx from NPA VM\n", r, 4664 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 4665 return r; 4666 } 4667 4668 /* 4669 * Reserved NPA space for remote shootdown and interrupt ring buffer, 4670 * write pointers, read pointers and writeback buffers 4671 * 4672 * NPA ring type: 0 = shootdown, 1 = interrupt, 2 = tailptr (wptr) 4673 */ 4674 enum ring_buffer_type { 4675 RB_TYPE_TLB_INV = 0, 4676 RB_TYPE_REMOTE_INTERRUPT = 1, 4677 RB_TYPE_TAILPTR = 2 4678 }; 4679 4680 /** 4681 * amdgpu_ualink_npa_addr - Get reserved NPA address for ring buffer 4682 * @adev: amdgpu device pointer 4683 * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR) 4684 * @src_accel_id: Source accelerator ID 4685 * @dst_accel_id: Destination accelerator ID 4686 * 4687 * Return: NPA address for the specified ring buffer type and GPUs 4688 */ 4689 static u64 amdgpu_ualink_npa_addr(struct amdgpu_device *adev, u32 type, 4690 u32 src_accel_id, u32 dst_accel_id) 4691 { 4692 u32 addr_mode = ualink_addr_mode(adev); 4693 u64 npa; 4694 4695 WARN_ON_ONCE(src_accel_id >= AMDGPU_UALINK_ACCEL_MAX || 4696 dst_accel_id >= AMDGPU_UALINK_ACCEL_MAX || 4697 type > RB_TYPE_TAILPTR || 4698 (addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT && 4699 addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)); 4700 4701 switch (addr_mode) { 4702 case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT: 4703 npa = (u64)src_accel_id << 41 | type << 12; 4704 break; 4705 4706 case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS: 4707 default: 4708 if (type == RB_TYPE_TAILPTR) 4709 npa = (u64)dst_accel_id << 13; 4710 else 4711 npa = (u64)src_accel_id << 13 | type << 12; 4712 4713 /* 4714 * In order to avoid address conflicts between source-identification 4715 * and source-aliasing mode, add 2MB to the buffer addresses in 4716 * source-aliasing mode. This way in a misconfigured cluster, a GPU 4717 * configured in the wrong address mode will access unmapped NPA addresses 4718 * rather than the wrong buffer mappings. 4719 */ 4720 npa += AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET; 4721 break; 4722 } 4723 4724 dev_dbg(adev->dev, "addr mode %d from accel %d to accel %d type %d NPA 0x%llx\n", 4725 addr_mode, src_accel_id, dst_accel_id, type, npa); 4726 4727 return npa; 4728 } 4729 4730 /** 4731 * amdgpu_ualink_gart_npa_addr - Get GART-mapped NPA address for ring buffer 4732 * @adev: amdgpu device pointer 4733 * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR) 4734 * @src_accel_id: Source accelerator ID 4735 * @dst_accel_id: Destination accelerator ID 4736 * 4737 * Computes the NPA address suitable for GART mapping by adding dst_accid_id to 4738 * NPA address to access remote by SDMA. 4739 4740 * clears the source GPU ID bits from the NPA address which is set for 4741 * address-identification mode. 4742 * 4743 * Return: NPA address adjusted for GART mapping 4744 */ 4745 4746 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT 41 4747 #define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK GENMASK_ULL(50, 41) 4748 4749 static u64 amdgpu_ualink_gart_npa_addr(struct amdgpu_device *adev, u32 type, 4750 u32 src_accel_id, u32 dst_accel_id) 4751 { 4752 u64 npa; 4753 4754 npa = amdgpu_ualink_npa_addr(adev, type, src_accel_id, dst_accel_id); 4755 4756 /* wiping out source accelerator id */ 4757 npa &= ~AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK; 4758 4759 return npa | ((u64)dst_accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT); 4760 } 4761 4762 /** 4763 * amdgpu_ualink_reserve_npa_vm_and_bos - Reserve the NPA VM page directory and BOs 4764 * @adev: amdgpu device pointer 4765 * @bos: array of buffer objects to reserve 4766 * @n_bos: number of entries in @bos 4767 * @exec: drm_exec context to initialize and use for locking 4768 * @interruptible: true to use the interruptible dma_resv_lock 4769 * 4770 * Initializes @exec and uses it to lock all BOs in @bos together with the 4771 * NPA VM page directory, retrying on contention. On failure, @exec is 4772 * finalized and the error code is returned. 4773 * 4774 * Returns: 0 on success, negative error code on failure. 4775 */ 4776 static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev, 4777 struct amdgpu_bo *bos[], u32 n_bos, 4778 struct drm_exec *exec, 4779 bool interruptible) 4780 { 4781 u32 flags = DRM_EXEC_IGNORE_DUPLICATES; 4782 int i, r = 0; 4783 4784 if (interruptible) 4785 flags |= DRM_EXEC_INTERRUPTIBLE_WAIT; 4786 4787 dev_dbg(adev->dev, "reserve NPA vm and %d bos\n", n_bos); 4788 4789 drm_exec_init(exec, flags, 0); 4790 4791 drm_exec_until_all_locked(exec) { 4792 for (i = 0; i < n_bos; i++) { 4793 r = drm_exec_lock_obj(exec, &bos[i]->tbo.base); 4794 drm_exec_retry_on_contention(exec); 4795 if (unlikely(r)) 4796 goto out; 4797 } 4798 4799 r = amdgpu_vm_lock_pd(&adev->ualink.npa_vm, exec, 0); 4800 drm_exec_retry_on_contention(exec); 4801 if (unlikely(r)) 4802 goto out; 4803 } 4804 4805 out: 4806 if (r) 4807 drm_exec_fini(exec); 4808 return r; 4809 4810 } 4811 4812 /** 4813 * amdgpu_ualink_unreserve_npa_vm_and_bos - Release the NPA VM page directory and BOs 4814 * @adev: amdgpu device pointer 4815 * @exec: drm_exec context previously initialized by 4816 * amdgpu_ualink_reserve_npa_vm_and_bos() 4817 * 4818 * Finalizes @exec, releasing all locks on the NPA VM page directory and 4819 * the associated BOs acquired during reservation. 4820 */ 4821 static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev, 4822 struct drm_exec *exec) 4823 { 4824 dev_dbg(adev->dev, "unreserve NPA vm and bos\n"); 4825 drm_exec_fini(exec); 4826 } 4827 4828 /** 4829 * amdgpu_ualink_metadata_npa_unmapping - Tear down NPA address mappings 4830 * @adev: amdgpu device pointer 4831 * 4832 * Unmaps all NPA address mappings from the NPA VM for ring buffers, 4833 * write pointers, and read pointers. Waits for outstanding DMA fences 4834 * and flushes the TLB. 4835 */ 4836 static void amdgpu_ualink_metadata_npa_unmapping(struct amdgpu_device *adev) 4837 { 4838 struct amdgpu_ualink_remote *remote = to_remote(adev); 4839 u64 timeout = msecs_to_jiffies(2000); 4840 u32 dst_accel_id = ualink_accel_id(adev); 4841 struct amdgpu_bo *bos[2]; 4842 u32 n_bos; 4843 struct dma_fence *fence; 4844 struct drm_exec exec; 4845 u32 rptr_size, rptr_size_in_pages; 4846 u32 rb_size, rb_size_in_pages; 4847 u64 pte_flags = adev->gmc.noretry_flags; 4848 u64 npa; 4849 int r; 4850 4851 if (!remote->ring_bo) 4852 return; 4853 4854 rb_size = AMDGPU_UALINK_RB_SIZE; 4855 rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT; 4856 4857 bos[0] = remote->ring_bo; 4858 n_bos = 1; 4859 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4860 bos[1] = remote->rptr_bo; 4861 n_bos = 2; 4862 } 4863 4864 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false); 4865 if (unlikely(r)) 4866 return; 4867 4868 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4869 /* rptr npa mapping, up to allocated 2 pages npa address */ 4870 rptr_size = ualink_wb_size(adev); 4871 rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel); 4872 rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT; 4873 4874 npa = remote->rptr_npa; 4875 4876 amdgpu_ualink_npa_vm_unmap_range(adev, remote->rptr_bo, 4877 pte_flags, 0, rptr_size_in_pages, 4878 npa >> AMDGPU_GPU_PAGE_SHIFT); 4879 amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node); 4880 4881 /* 4882 * unmap wptr, remote interrupt, shootdown ring. 4883 * wptr page is part of the single 2MB mapping for remote GPUs 4884 * interrupt and shootdown ring buffer 4885 */ 4886 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, 4887 0, dst_accel_id); 4888 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, pte_flags, 4889 0, 4890 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX, 4891 npa >> AMDGPU_GPU_PAGE_SHIFT); 4892 } else { 4893 u32 wptr_offset = 2 * rb_size * remote->num_accel; 4894 u32 idx = 0; 4895 u32 accel_id; 4896 4897 /* source identification mode */ 4898 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 4899 if (accel_id == dst_accel_id) 4900 continue; 4901 4902 /* remote interrupt ring npa mapping */ 4903 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 4904 accel_id, dst_accel_id); 4905 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4906 pte_flags, idx * 2 * rb_size, 4907 rb_size_in_pages, 4908 npa >> AMDGPU_GPU_PAGE_SHIFT); 4909 4910 /* remote shootdown ring npa mapping */ 4911 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id, 4912 dst_accel_id); 4913 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4914 pte_flags, (idx * 2 + 1) * rb_size, 4915 rb_size_in_pages, 4916 npa >> AMDGPU_GPU_PAGE_SHIFT); 4917 4918 /* wptr, rptr npa mapping */ 4919 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id, 4920 dst_accel_id); 4921 amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, 4922 pte_flags, wptr_offset + idx * PAGE_SIZE, 4923 1, npa >> AMDGPU_GPU_PAGE_SHIFT); 4924 idx++; 4925 } 4926 } 4927 4928 r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false); 4929 if (r) { 4930 dev_dbg(adev->dev, "failed %d to update directories\n", r); 4931 goto out_unreserve; 4932 } 4933 4934 fence = dma_fence_get(adev->ualink.npa_vm.last_update); 4935 if (fence) { 4936 r = dma_fence_wait_timeout(fence, true, timeout); 4937 dma_fence_put(fence); 4938 if (r <= 0) 4939 dev_dbg(adev->dev, "failed %d to dma fence wait\n", r); 4940 } 4941 4942 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 4943 out_unreserve: 4944 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 4945 } 4946 4947 /** 4948 * amdgpu_ualink_metadata_npa_mapping - Setup NPA address mapping in VM 4949 * @adev: amdgpu device pointer 4950 * 4951 * Maps NPA addresses to GPA for metadata ring buffers, write pointers, 4952 * on NPA VMID 15. 4953 * 4954 * Return: 0 on success, negative error code on failure 4955 */ 4956 static int amdgpu_ualink_metadata_npa_mapping(struct amdgpu_device *adev) 4957 { 4958 struct amdgpu_ualink_remote *remote = to_remote(adev); 4959 u32 dst_accel_id = ualink_accel_id(adev); 4960 u64 timeout = msecs_to_jiffies(2000); 4961 struct amdgpu_bo *bos[2]; 4962 u32 n_bos; 4963 struct dma_fence *fence; 4964 struct drm_exec exec; 4965 u32 rb_size, rb_size_in_pages; 4966 u64 npa, npa_in_pages, pte_flags; 4967 int r; 4968 4969 rb_size = AMDGPU_UALINK_RB_SIZE; 4970 rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT; 4971 4972 bos[0] = remote->ring_bo; 4973 n_bos = 1; 4974 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4975 bos[1] = remote->rptr_bo; 4976 n_bos = 2; 4977 } 4978 4979 r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false); 4980 if (unlikely(r)) 4981 return r; 4982 4983 pte_flags = amdgpu_ttm_tt_pte_flags(adev, remote->ring_bo->tbo.ttm, 4984 remote->ring_bo->tbo.resource); 4985 dev_dbg(adev->dev, "init pte_flags 0x%llx\n", pte_flags); 4986 4987 amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, remote->ring_bo, 4988 AMDGPU_VM_MTYPE_DEFAULT, &pte_flags); 4989 dev_dbg(adev->dev, "after get coherent pte_flags 0x%llx\n", pte_flags); 4990 4991 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 4992 u32 rptr_size, rptr_size_in_pages; 4993 4994 /* rptr npa mapping, up to allocated 2 pages npa address */ 4995 rptr_size = ualink_wb_size(adev); 4996 rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel); 4997 rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT; 4998 4999 r = amdgpu_ualink_npa_alloc_va(adev, &remote->rptr_mm_node, 5000 0, 0, 0, 5001 rptr_size_in_pages); 5002 if (r) 5003 goto out; 5004 5005 npa_in_pages = remote->rptr_mm_node.start; 5006 5007 dev_dbg(adev->dev, "source aliasing rptr alloc 0x%llx and map to npa vm\n", 5008 npa_in_pages << AMDGPU_GPU_PAGE_SHIFT); 5009 5010 r = amdgpu_ualink_npa_vm_map_range(adev, remote->rptr_bo, pte_flags, 0, 5011 rptr_size_in_pages, 5012 npa_in_pages); 5013 if (r) 5014 goto error_npa_mapping; 5015 5016 remote->rptr_npa = npa_in_pages << AMDGPU_GPU_PAGE_SHIFT; 5017 5018 /* wptr, remote interrupt, shootdown ring, single big 2MB mapping */ 5019 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, 5020 0, dst_accel_id); 5021 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5022 0, 5023 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX, 5024 npa >> AMDGPU_GPU_PAGE_SHIFT); 5025 if (r) 5026 goto error_npa_mapping; 5027 5028 } else { 5029 u32 wptr_offset = 2 * rb_size * remote->num_accel; 5030 u32 accel_id, idx = 0; 5031 5032 /* Source identification mode */ 5033 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5034 if (accel_id == dst_accel_id) 5035 continue; 5036 5037 /* remote interrupt ring npa mapping */ 5038 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 5039 accel_id, dst_accel_id); 5040 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5041 idx * 2 * rb_size, 5042 rb_size_in_pages, 5043 npa >> AMDGPU_GPU_PAGE_SHIFT); 5044 if (r) 5045 goto error_npa_mapping; 5046 5047 /* remote shootdown ring npa mapping */ 5048 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id, 5049 dst_accel_id); 5050 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags, 5051 (idx * 2 + 1) * rb_size, 5052 rb_size_in_pages, 5053 npa >> AMDGPU_GPU_PAGE_SHIFT); 5054 if (r) 5055 goto error_npa_mapping; 5056 5057 /* wptr, rptr npa mapping */ 5058 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id, 5059 dst_accel_id); 5060 r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, 5061 pte_flags, 5062 wptr_offset + idx * PAGE_SIZE, 5063 1, npa >> AMDGPU_GPU_PAGE_SHIFT); 5064 if (r) 5065 goto error_npa_mapping; 5066 5067 idx++; 5068 } 5069 } 5070 5071 r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false); 5072 if (r) { 5073 dev_dbg(adev->dev, "failed %d to update directories\n", r); 5074 goto error_npa_mapping; 5075 } 5076 5077 /* TODO: only wait the last fence, then flush TLB */ 5078 fence = dma_fence_get(adev->ualink.npa_vm.last_update); 5079 if (fence) { 5080 r = dma_fence_wait_timeout(fence, true, timeout); 5081 dma_fence_put(fence); 5082 if (r <= 0) 5083 dev_dbg(adev->dev, "failed %d to dma fence wait\n", r); 5084 } 5085 5086 amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); 5087 5088 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 5089 return 0; 5090 5091 error_npa_mapping: 5092 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 5093 amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node); 5094 if (r) 5095 dev_dbg(adev->dev, "failed %d to map NPA vm\n", r); 5096 5097 out: 5098 amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); 5099 5100 return r; 5101 } 5102 5103 /** 5104 * amdgpu_ualink_sdma_entities_init - Initialize SDMA job scheduler entities 5105 * @adev: amdgpu device pointer 5106 * 5107 * Sets up SDMA job scheduler entities for remote interrupt and shootdown. 5108 * Each entity uses one SDMA scheduler/ring. 5109 * 5110 * Return: 0 on success, negative error code on failure 5111 */ 5112 static int amdgpu_ualink_sdma_entities_init(struct amdgpu_device *adev) 5113 { 5114 struct amdgpu_ualink_remote *remote = to_remote(adev); 5115 struct drm_gpu_scheduler *sched; 5116 u32 accel_id; 5117 int i = 0, r; 5118 5119 dev_dbg(adev->dev, "enter\n"); 5120 5121 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5122 sched = &adev->sdma.instance[accel_id % adev->sdma.num_instances].ring.sched; 5123 5124 r = drm_sched_entity_init(&remote->peer[accel_id].entity, DRM_SCHED_PRIORITY_HIGH, 5125 &sched, 1, NULL); 5126 if (r) 5127 goto out_free; 5128 i++; 5129 } 5130 5131 remote->use_lsdma = true; 5132 5133 dev_dbg(adev->dev, "exit\n"); 5134 return 0; 5135 5136 out_free: 5137 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5138 if (i--) 5139 drm_sched_entity_destroy(&remote->peer[accel_id].entity); 5140 else 5141 break; 5142 } 5143 5144 return r; 5145 } 5146 5147 static void amdgpu_ualink_sdma_entities_fini(struct amdgpu_device *adev) 5148 { 5149 struct amdgpu_ualink_remote *remote = to_remote(adev); 5150 u32 accel_id; 5151 5152 dev_dbg(adev->dev, "enter\n"); 5153 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) 5154 drm_sched_entity_destroy(&remote->peer[accel_id].entity); 5155 dev_dbg(adev->dev, "exit\n"); 5156 } 5157 5158 /** 5159 * amdgpu_ualink_gart_map - Allocate GART entry and map NPA address 5160 * @adev: amdgpu device pointer 5161 * @npages: Number of pages to map 5162 * @npa: NPA address to map 5163 * @mm_node: DRM memory manager node for GART allocation 5164 * @pte_flags: the GART mapping flags 5165 * 5166 * Allocates GART entries and sets up NPA address mapping without TTM BO. 5167 * 5168 * Return: 0 on success, negative error code on failure 5169 */ 5170 static int amdgpu_ualink_gart_map(struct amdgpu_device *adev, u64 npages, 5171 u64 npa, struct drm_mm_node *mm_node, 5172 u64 pte_flags) 5173 { 5174 struct ttm_resource_manager *man = 5175 ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 5176 struct amdgpu_gtt_mgr *mgr = 5177 container_of(man, struct amdgpu_gtt_mgr, manager); 5178 dma_addr_t *dma_addr; 5179 int i, r; 5180 5181 dev_dbg(adev->dev, "npa 0x%llx npages 0x%llx\n", npa, npages); 5182 5183 if (npages > 1) { 5184 dma_addr = kmalloc_objs(*dma_addr, npages); 5185 if (!dma_addr) 5186 return -ENOMEM; 5187 5188 for (i = 0; i < npages; i++) 5189 dma_addr[i] = npa + i * AMDGPU_GPU_PAGE_SIZE; 5190 } else { 5191 dma_addr = (dma_addr_t *)&npa; 5192 } 5193 5194 r = amdgpu_gtt_mgr_alloc_entries(mgr, mm_node, npages, DRM_MM_INSERT_BEST); 5195 if (r) 5196 goto out; 5197 amdgpu_gart_bind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, dma_addr, 5198 pte_flags); 5199 5200 out: 5201 dev_dbg(adev->dev, "npa 0x%llx mapped to 0x%llx npages 0x%llx r=%d\n", 5202 npa, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, r); 5203 5204 if (npages > 1) 5205 kfree(dma_addr); 5206 return r; 5207 } 5208 5209 /** 5210 * amdgpu_ualink_gart_unmap - Unmap and free GART entry 5211 * @adev: amdgpu device pointer 5212 * @npages: Number of pages to unmap 5213 * @mm_node: DRM memory manager node for GART allocation 5214 * 5215 * Unbinds GART mapping and frees allocated entries. 5216 */ 5217 static void amdgpu_ualink_gart_unmap(struct amdgpu_device *adev, u64 npages, 5218 struct drm_mm_node *mm_node) 5219 { 5220 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 5221 struct amdgpu_gtt_mgr *mgr = 5222 container_of(man, struct amdgpu_gtt_mgr, manager); 5223 5224 dev_dbg(adev->dev, "0x%llx npages 0x%llx\n", mm_node->start << AMDGPU_GPU_PAGE_SHIFT, 5225 npages); 5226 5227 if (!drm_mm_node_allocated(mm_node)) 5228 return; 5229 amdgpu_gart_unbind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages); 5230 amdgpu_gtt_mgr_free_entries(mgr, mm_node); 5231 } 5232 5233 /** 5234 * amdgpu_ualink_metadata_fini - Clean up ualink metadata structures 5235 * @adev: amdgpu device pointer 5236 * 5237 * Unmaps NPA addresses and frees all allocated buffers for metadata, 5238 * ring buffers, and pointers. 5239 */ 5240 static void amdgpu_ualink_metadata_fini(struct amdgpu_device *adev) 5241 { 5242 struct amdgpu_ualink_remote *remote = to_remote(adev); 5243 5244 dev_dbg(adev->dev, "accel_id %u\n", ualink_accel_id(adev)); 5245 amdgpu_bo_free_kernel(&remote->rptr_bo, &remote->rptr_gpu_addr, 5246 &remote->rptr_cpu_addr); 5247 amdgpu_bo_free_kernel(&remote->ring_bo, &remote->rb_gpu_addr, 5248 &remote->rb_cpu_addr); 5249 amdgpu_bo_free_kernel(&remote->metadata_bo, &remote->metadata_gpu_addr, 5250 &remote->metadata_cpu_addr); 5251 } 5252 5253 /** 5254 * amdgpu_ualink_metadata_init - Initialize ualink metadata structures 5255 * @adev: amdgpu device pointer 5256 * 5257 * Allocates and initializes metadata structures, ring buffers, and write/read 5258 * pointers for multi-GPU ualink. Communicates with firmware to load metadata. 5259 * 5260 * Return: 0 on success, negative error code on failure 5261 */ 5262 static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev) 5263 { 5264 struct amdgpu_ualink_remote *remote = to_remote(adev); 5265 struct amdgpu_ualink_metadata *metadata; 5266 u32 size, rb_size, wptr_size, rptr_size, metadata_size; 5267 u64 rb_gpu_addr, wptr_gpu_addr; 5268 u32 status, accel_id; 5269 int r; 5270 5271 bitmap_copy(remote->active_accel_bits, 5272 adev->ualink.info->vpod.active_accel_bits, 5273 AMDGPU_UALINK_ACCEL_MAX); 5274 dev_dbg(adev->dev, "%d active accelerators config in vpod\n", 5275 bitmap_weight(remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX)); 5276 5277 /* 5278 * allocate metadata entries and ring buffer for all remote GPUs, 5279 * to get 2MB page ring buffer NPA mapping for remote access. 5280 */ 5281 remote->num_accel = AMDGPU_UALINK_ACCEL_MAX; 5282 5283 status = adev->ualink.msg_ctl->check_status(adev); 5284 if (status != AMDGPU_NHT_FW_ST_PREINIT && 5285 status != AMDGPU_NHT_FW_ST_HALT) { 5286 dev_dbg(adev->dev, "fw status 0x%x not preinit or halt\n", status); 5287 return -ENODEV; 5288 } 5289 5290 dev_dbg(adev->dev, "accel_id %u addr_mode %d fw status 0x%x\n", 5291 ualink_accel_id(adev), ualink_addr_mode(adev), status); 5292 5293 /* Alloc metadata structure for all GPUs */ 5294 metadata_size = sizeof(struct amdgpu_ualink_metadata) * AMDGPU_UALINK_ACCEL_MAX; 5295 metadata_size = AMDGPU_GPU_PAGE_ALIGN(metadata_size); 5296 5297 dev_dbg(adev->dev, "metadata size 0x%x\n", metadata_size); 5298 5299 /* pinned system memory */ 5300 r = amdgpu_bo_create_kernel(adev, metadata_size, PAGE_SIZE, 5301 AMDGPU_GEM_DOMAIN_GTT, 5302 &remote->metadata_bo, &remote->metadata_gpu_addr, 5303 &remote->metadata_cpu_addr); 5304 if (r) 5305 goto out; 5306 5307 memset(remote->metadata_cpu_addr, 0, metadata_size); 5308 metadata = remote->metadata_cpu_addr; 5309 5310 dev_dbg(adev->dev, "metadata gpu address 0x%llx\n", remote->metadata_gpu_addr); 5311 5312 /* Allocate ring buffers, wptr for remote interrupt and shootdown */ 5313 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE); 5314 wptr_size = ualink_wptr_size(adev); 5315 size = (rb_size + wptr_size) * remote->num_accel; 5316 size = AMDGPU_GPU_PAGE_ALIGN(size); 5317 5318 dev_dbg(adev->dev, "rb_size 0x%x wptr_size 0x%x total alloc size 0x%x\n", 5319 rb_size, wptr_size, size); 5320 5321 /* pinned VRAM */ 5322 r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 5323 AMDGPU_GEM_DOMAIN_VRAM, 5324 &remote->ring_bo, 5325 &remote->rb_gpu_addr, 5326 &remote->rb_cpu_addr); 5327 if (r) 5328 goto out; 5329 5330 memset(remote->rb_cpu_addr, 0, size); 5331 5332 dev_dbg(adev->dev, "rb gpu addr 0x%llx cpu addr 0x%p vram_start 0x%llx vram_base 0x%llx\n", 5333 remote->rb_gpu_addr, remote->rb_cpu_addr, adev->gmc.vram_start, 5334 adev->vm_manager.vram_base_offset); 5335 5336 /* 5337 * address aliasing mode, shared wptr pagee left is not enough for wb data, 5338 * wb status and rptr, alloc another BO, and then alloc npa address and map 5339 * to npa vm, for remote to access. 5340 * 5341 * No gart mapping required for wb data, status and rptr because this is 5342 * updated by firmware. 5343 */ 5344 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) { 5345 rptr_size = AMDGPU_GPU_PAGE_ALIGN(ualink_wb_size(adev) * remote->num_accel); 5346 dev_dbg(adev->dev, "source aliasing mode rptr_size 0x%x\n", rptr_size); 5347 5348 /* pinned VRAM */ 5349 r = amdgpu_bo_create_kernel(adev, rptr_size, PAGE_SIZE, 5350 AMDGPU_GEM_DOMAIN_VRAM, 5351 &remote->rptr_bo, 5352 &remote->rptr_gpu_addr, 5353 &remote->rptr_cpu_addr); 5354 if (r) 5355 goto out; 5356 5357 memset(remote->rptr_cpu_addr, 0, rptr_size); 5358 5359 dev_dbg(adev->dev, "source aliasing rptr gpu addr 0x%llx cpu addr 0x%p\n", 5360 remote->rptr_gpu_addr, remote->rptr_cpu_addr); 5361 } 5362 5363 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) 5364 wptr_gpu_addr = remote->rb_gpu_addr + rb_size * ualink_accel_id(adev); 5365 5366 for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 5367 dev_dbg(adev->dev, "init for accel_id %u\n", accel_id); 5368 5369 if (accel_id == ualink_accel_id(adev)) 5370 continue; 5371 5372 rb_gpu_addr = remote->rb_gpu_addr + rb_size * accel_id; 5373 5374 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 5375 wptr_gpu_addr = remote->rb_gpu_addr + rb_size * remote->num_accel; 5376 wptr_gpu_addr += wptr_size * accel_id; 5377 } 5378 5379 metadata[accel_id].header = AMDGPU_UALINK_METADATA_HEADER; 5380 metadata[accel_id].rb_size = fls(AMDGPU_UALINK_RB_SIZE / 64) - 1; 5381 5382 /* 5383 * remote shootdown type is 0, remote command type is 1 5384 * with 2MB ring buffer mapping, remote shootdown ring NPA is before interrupt 5385 */ 5386 metadata[accel_id].ri_rb = rb_gpu_addr + AMDGPU_UALINK_RB_SIZE; 5387 metadata[accel_id].tlb_inv_rb = rb_gpu_addr; 5388 5389 metadata[accel_id].tailptr_ri = wptr_gpu_addr + 5390 ualink_wptr_offset(adev, accel_id); 5391 metadata[accel_id].tailptr_tlb_inv = wptr_gpu_addr + 5392 ualink_tlb_wptr_offset(adev, accel_id); 5393 5394 dev_dbg(adev->dev, "init from accel_id %u to accel_id %u, rb_size 0x%x\n", 5395 accel_id, ualink_accel_id(adev), metadata[accel_id].rb_size); 5396 dev_dbg(adev->dev, "rb 0x%llx tlb rb 0x%llx\n", 5397 metadata[accel_id].ri_rb, metadata[accel_id].tlb_inv_rb); 5398 dev_dbg(adev->dev, "rb wptr at 0x%llx tlb wptr at 0x%llx\n", 5399 metadata[accel_id].tailptr_ri, metadata[accel_id].tailptr_tlb_inv); 5400 } 5401 5402 r = adev->ualink.msg_ctl->send_metadata(adev, 5403 remote->metadata_gpu_addr, 5404 AMDGPU_UALINK_ACCEL_MAX << 8); 5405 5406 out: 5407 if (r) 5408 amdgpu_ualink_metadata_fini(adev); 5409 5410 dev_dbg(adev->dev, "ret 0x%x\n", r); 5411 return r; 5412 } 5413 5414 /* 5415 * nHT Firmware Error code 5416 * 5417 * Success 0x0 5418 * FIFO overflow 0x1 5419 * Invalid RB command 0x2 5420 * Timeout 0x3 5421 * Invalid metadata entry 0x4 5422 * RB overflow 0x5 5423 * Invalid wptr address 0x6 5424 * Interrupt cookie send failure 0x7 5425 * Invalid RB address 0x8 5426 * Invalid writeback address 0x9 5427 */ 5428 static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status) 5429 { 5430 const char *fw_err_code_msg[] = { 5431 "Unknown FW error status", 5432 "FIFO overflow", /* error code 1 */ 5433 "Invalid RB command", 5434 "Timeout", 5435 "Invalid metadata entry", 5436 "RB overflow", 5437 "Invalid wptr address", 5438 "Interrupt cookie send failure", 5439 "Invalid RB address", 5440 "Invalid writeback address" /* error code 9 */ 5441 }; 5442 5443 if (status >= ARRAY_SIZE(fw_err_code_msg)) 5444 status = 0; 5445 5446 dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]); 5447 return; 5448 } 5449 5450 /** 5451 * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion 5452 * @remote_accel_id: remote accelator id to wait for reply 5453 * @adev: amdgpu device pointer 5454 * @wb_cpu: CPU virtual address of writeback buffer 5455 * @seq: Sequence number to wait for 5456 * 5457 * Polls the writeback buffer waiting for the sequence number to reach or 5458 * exceed the expected value. The writeback buffer contains status and data 5459 * fields updated by firmware to indicate completion and error conditions. 5460 * 5461 * Return: 0 on success 5462 * -ETIME on timeout 5463 * -ECOMM on firmware return error status 5464 */ 5465 static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev, 5466 u32 remote_accel_id, 5467 struct amdgpu_ualink_wb *wb_cpu, 5468 u32 seq) 5469 { 5470 /* 2 seconds timeout, long enough for FW to reply */ 5471 ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC); 5472 u32 status, data; 5473 u64 rptr; 5474 5475 while (true) { 5476 data = READ_ONCE(wb_cpu->data); 5477 if (data >= seq) { 5478 status = READ_ONCE(wb_cpu->status); 5479 rptr = READ_ONCE(wb_cpu->rptr); 5480 break; 5481 } 5482 5483 if (ktime_after(ktime_get(), timeout)) { 5484 dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id); 5485 return -ETIME; 5486 } 5487 5488 usleep_range(10, 50); 5489 } 5490 5491 if (status) { 5492 dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n", 5493 remote_accel_id, status, data, seq, rptr); 5494 5495 amdgpu_ualink_remote_error(adev, status); 5496 return -ECOMM; 5497 } 5498 5499 dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n", 5500 remote_accel_id, seq, data, rptr); 5501 return 0; 5502 } 5503 5504 #define AMDGPU_UALINK_REMOTE_OP_TLB_INV 0x1 5505 #define AMDGPU_UALINK_REMOTE_OP_INT 0x2 5506 #define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB 0x3 5507 5508 static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data, 5509 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5510 { 5511 u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT; 5512 u32 *cpu_addr = *cpu_addr_p; 5513 u32 flush_type = dw0; 5514 u32 size_in_pages = dw3; 5515 u32 cmd; 5516 5517 cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV; 5518 cmd |= 1 << 28; /* headptr update */ 5519 cmd |= 1 << 29; /* wb enable */ 5520 5521 *cpu_addr++ = cmd; 5522 *cpu_addr++ = wb_data; 5523 *cpu_addr++ = 0; 5524 *cpu_addr++ = 0; 5525 *cpu_addr++ = 0; 5526 *cpu_addr++ = 0; 5527 *cpu_addr++ = 0; /* metadata */ 5528 *cpu_addr++ = 0; /* metadata */ 5529 5530 pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages); 5531 5532 /* Always shootdown everything with full address size s-field coding */ 5533 addr = GENMASK_ULL(51, 11); 5534 5535 *cpu_addr++ = lower_32_bits(addr) | flush_type; 5536 *cpu_addr++ = upper_32_bits(addr); 5537 5538 *cpu_addr++ = 0; 5539 *cpu_addr++ = 0; 5540 *cpu_addr++ = 0; 5541 *cpu_addr++ = 0; 5542 *cpu_addr++ = 0; 5543 *cpu_addr++ = 0; 5544 *cpu_addr_p = cpu_addr; 5545 } 5546 5547 static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data, 5548 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5549 { 5550 u32 *cpu_addr = *cpu_addr_p; 5551 u32 cmd; 5552 5553 cmd = AMDGPU_UALINK_REMOTE_OP_INT; 5554 cmd |= 1 << 28; /* headptr_update */ 5555 cmd |= 1 << 29; /* wb enable */ 5556 5557 *cpu_addr++ = cmd; 5558 *cpu_addr++ = wb_data; 5559 *cpu_addr++ = 0; 5560 *cpu_addr++ = 0; 5561 *cpu_addr++ = 0; 5562 *cpu_addr++ = 0; 5563 *cpu_addr++ = 0; /* metadata */ 5564 *cpu_addr++ = 0; /* metadata */ 5565 /* updated by f/w, overwrite PASID with GPU ID */ 5566 *cpu_addr++ = 0; 5567 *cpu_addr++ = 0; 5568 *cpu_addr++ = 0; 5569 *cpu_addr++ = 0; 5570 /* IH ContextID 4 dwords */ 5571 *cpu_addr++ = dw0; 5572 *cpu_addr++ = dw1; 5573 *cpu_addr++ = dw2; 5574 *cpu_addr++ = dw3; 5575 *cpu_addr_p = cpu_addr; 5576 } 5577 5578 static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data, 5579 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5580 { 5581 u32 *cpu_addr = *cpu_addr_p; 5582 u32 cmd; 5583 5584 cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB; 5585 cmd |= 1 << 28; /* headptr update */ 5586 cmd |= 1 << 29; /* wb enable */ 5587 5588 *cpu_addr++ = cmd; 5589 *cpu_addr++ = wb_data; 5590 *cpu_addr++ = dw1 & 0xFFFFFFFC; /* lower32 wb_npa */ 5591 *cpu_addr++ = dw0 & 0xFFFFF; /* upper32 wb_npa */ 5592 *cpu_addr++ = 0; 5593 *cpu_addr++ = 0; 5594 *cpu_addr++ = 0; /* metadata */ 5595 *cpu_addr++ = 0; /* metadata */ 5596 *cpu_addr++ = 0; 5597 *cpu_addr++ = 0; 5598 *cpu_addr++ = 0; 5599 *cpu_addr++ = 0; 5600 *cpu_addr++ = 0; 5601 *cpu_addr++ = 0; 5602 *cpu_addr++ = 0; 5603 *cpu_addr++ = 0; 5604 *cpu_addr_p = cpu_addr; 5605 } 5606 5607 /* 5608 * Function proto to generate packet using above different emit_ functions 5609 */ 5610 typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0, 5611 u32 dw1, u32 dw2, u32 dw3); 5612 5613 /* 5614 * Refresh rptr from the writeback and check the ring has room for one more 5615 * command. The caller must hold the peer lock. 5616 * 5617 * Return 0 on success or a negative error code. 5618 */ 5619 static int amdgpu_ualink_ring_check_space(struct amdgpu_device *adev, 5620 u32 remote_accel_id, 5621 struct amdgpu_ualink_ring *ring, 5622 struct amdgpu_ualink_wb *wb_cpu) 5623 { 5624 ring->rptr = READ_ONCE(wb_cpu->rptr); 5625 5626 if (WARN_ON_ONCE(ring->rptr > ring->wptr)) { 5627 dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n", 5628 remote_accel_id, ring->wptr, ring->rptr); 5629 return -EFAULT; 5630 } 5631 5632 if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) { 5633 dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n", 5634 remote_accel_id, ring->wptr, ring->rptr); 5635 return -ENOSPC; 5636 } 5637 5638 return 0; 5639 } 5640 5641 static int amdgpu_ualink_send_command(struct amdgpu_device *adev, 5642 u32 remote_accel_id, 5643 struct amdgpu_ualink_ring *ring, 5644 struct amdgpu_ualink_wb *wb_cpu, 5645 ualink_emit_packet emit_func, 5646 u32 dw0, u32 dw1, u32 dw2, u32 dw3) 5647 { 5648 struct amdgpu_ualink_remote *remote = to_remote(adev); 5649 struct amdgpu_ualink_peer *peer; 5650 struct amdgpu_ring *sdma_ring; 5651 struct dma_fence *fence; 5652 struct amdgpu_job *job; 5653 struct amdgpu_ib *ib; 5654 u32 seq, ndw, ndw_copy_cmd, rem; 5655 u64 doorbell_npa_gart; 5656 u32 *src_cpu; 5657 u64 src; 5658 int r; 5659 5660 /* Refuse sends unless active */ 5661 if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 5662 return -ESHUTDOWN; 5663 5664 peer = &remote->peer[remote_accel_id]; 5665 5666 r = amdgpu_ualink_ring_check_space(adev, remote_accel_id, ring, wb_cpu); 5667 if (r) 5668 goto out; 5669 5670 /* 5671 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell 5672 * 5673 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to 5674 * ensure the ordering of 3 sdma copy commands execution. 5675 */ 5676 5677 /* buffer after sdma commands, starting at 8 dwords boundary */ 5678 ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8); 5679 5680 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */ 5681 ndw = ndw_copy_cmd + 16 + 2 + 1; 5682 r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM, 5683 ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, 5684 AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job); 5685 if (r) 5686 goto out; 5687 5688 ib = &job->ibs[0]; 5689 src = ib->gpu_addr + ndw_copy_cmd * 4; 5690 src_cpu = ib->ptr + ndw_copy_cmd; 5691 5692 seq = ++ring->seq; 5693 5694 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5695 dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n", 5696 src, ring->rb_npa_gart + rem * 64, 5697 ring->wptr_npa_gart, ring->doorbell_npa_gart, seq); 5698 5699 dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr); 5700 5701 /* remote command packet */ 5702 emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3); 5703 5704 /* wptr value */ 5705 *src_cpu++ = lower_32_bits(ring->wptr + 1); 5706 *src_cpu++ = upper_32_bits(ring->wptr + 1); 5707 5708 /* doorbell value */ 5709 *src_cpu++ = lower_32_bits(ring->wptr + 1); 5710 5711 /* 5712 * Add an offset to the doorbell address that cycles through different 5713 * values of bit 11:8 to use different DXS ports. 5714 */ 5715 peer->dxs_port = (peer->dxs_port + 1) & 0xF; 5716 doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8); 5717 dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n", 5718 peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart); 5719 5720 dev_dbg(adev->dev, "use %s\n", remote->use_lsdma ? "lsdma" : "sdma"); 5721 5722 if (remote->use_lsdma) { 5723 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5724 r = amdgpu_lsdma_copy_mem(adev, src, 5725 ring->rb_npa_gart + rem * 64, 5726 64); 5727 if (!r) 5728 r = amdgpu_lsdma_copy_mem(adev, src + 64, 5729 ring->wptr_npa_gart, 8); 5730 if (!r) 5731 r = amdgpu_lsdma_copy_mem(adev, src + 72, 5732 doorbell_npa_gart, 4); 5733 5734 amdgpu_job_free(job); 5735 5736 /* Copy failed: command never reached the remote, so no 5737 * writeback will arrive. Return now instead of polling the 5738 * completion for the full remote timeout. 5739 */ 5740 if (r) { 5741 dev_dbg(adev->dev, 5742 "remote %u lsdma copy failed (r %d), skip completion wait\n", 5743 remote_accel_id, r); 5744 return r; 5745 } 5746 5747 goto out_wait_complete; 5748 } 5749 5750 sdma_ring = &adev->sdma.instance[0].ring; 5751 5752 /* 5753 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands 5754 * execution doesn't overlap, doorbell update is after data and wptr update 5755 * finished. 5756 */ 5757 div_u64_rem(ring->wptr, ring->rb_size, &rem); 5758 amdgpu_emit_copy_buffer(adev, ib, src, 5759 ring->rb_npa_gart + rem * 64, 5760 64, 0); 5761 5762 ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop; 5763 5764 amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0); 5765 5766 ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop; 5767 5768 amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0); 5769 5770 amdgpu_ring_pad_ib(sdma_ring, ib); 5771 WARN_ON(ib->length_dw > ndw_copy_cmd); 5772 5773 fence = amdgpu_job_submit(job); 5774 r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT); 5775 dma_fence_put(fence); 5776 if (r <= 0) { 5777 dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n", 5778 remote_accel_id, r); 5779 if (r == 0) 5780 r = -ETIME; 5781 goto out; 5782 } 5783 5784 out_wait_complete: 5785 /* 5786 * poll remote completion writeback data 5787 */ 5788 r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq); 5789 5790 /* 5791 * Advance wptr unless the remote timed out, even if the FW returned an 5792 * error status. On timeout the caller resets the connection, which 5793 * resyncs wptr from the remote in amdgpu_ualink_setup_connection. 5794 */ 5795 if (r != -ETIME) 5796 ring->wptr++; 5797 5798 out: 5799 dev_dbg(adev->dev, "ret r = %d\n", r); 5800 return r; 5801 } 5802 5803 static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev, 5804 u32 remote_accel_id, 5805 struct amdgpu_ualink_wb **wb_cpu, 5806 u64 *wb_npa, u32 type) 5807 { 5808 struct amdgpu_ualink_remote *remote = to_remote(adev); 5809 u32 accel_id = ualink_accel_id(adev); 5810 u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE); 5811 uintptr_t wb; 5812 u64 npa; 5813 u32 offset = 0; 5814 5815 if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) { 5816 wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel; 5817 npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, 5818 remote_accel_id, accel_id); 5819 } else { 5820 wb = (uintptr_t)remote->rptr_cpu_addr; 5821 npa = remote->rptr_npa; 5822 npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT; 5823 } 5824 5825 if (type == RB_TYPE_TLB_INV) 5826 offset = ualink_tlb_wb_offset(adev, remote_accel_id); 5827 else if (type == RB_TYPE_REMOTE_INTERRUPT) 5828 offset = ualink_wb_offset(adev, remote_accel_id); 5829 5830 *wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset); 5831 if (wb_npa) 5832 *wb_npa = npa + offset; 5833 5834 dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id, 5835 remote_accel_id, npa + offset); 5836 } 5837 5838 struct amdgpu_ualink_wptr_fence_cb { 5839 struct dma_fence_cb base; 5840 u64 *wptr_cpu; 5841 u64 result; 5842 }; 5843 5844 static void amdgpu_ualink_read_wptr_fence_cb(struct dma_fence *fence, 5845 struct dma_fence_cb *cb) 5846 { 5847 struct amdgpu_ualink_wptr_fence_cb *wptr_cb; 5848 5849 wptr_cb = container_of(cb, typeof(*wptr_cb), base); 5850 wptr_cb->result = *wptr_cb->wptr_cpu; 5851 } 5852 5853 static int amdgpu_ualink_read_remote_wptr(struct amdgpu_device *adev, 5854 u32 remote_accel_id, 5855 struct amdgpu_ualink_ring *ring, 5856 u64 *wptr) 5857 { 5858 struct amdgpu_ualink_remote *remote = to_remote(adev); 5859 struct amdgpu_ualink_peer *peer = &remote->peer[remote_accel_id]; 5860 struct amdgpu_ualink_wptr_fence_cb wptr_cb = { }; 5861 struct amdgpu_ring *sdma_ring; 5862 struct dma_fence *fence; 5863 struct amdgpu_job *job; 5864 struct amdgpu_ib *ib; 5865 u32 ndw, ndw_copy_cmd; 5866 u64 wptr_gpu, *wptr_cpu; 5867 int r; 5868 5869 *wptr = 0; 5870 5871 /* Refuse remote access unless active */ 5872 if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) 5873 return -ESHUTDOWN; 5874 5875 /* 1 sdma copy command to read wptr */ 5876 ndw_copy_cmd = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); 5877 5878 /* wptr 2 dwords */ 5879 ndw = ndw_copy_cmd + 2; 5880 r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM, 5881 ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, 5882 AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job); 5883 if (r) 5884 return r; 5885 5886 ib = &job->ibs[0]; 5887 wptr_gpu = ib->gpu_addr + ndw_copy_cmd * 4; 5888 wptr_cpu = (u64 *)(ib->ptr + ndw_copy_cmd); 5889 5890 dev_dbg(adev->dev, "read wptr from remote %u npa gart wptr 0x%llx use %s\n", 5891 remote_accel_id, ring->wptr_npa_gart, 5892 remote->use_lsdma ? "lsdma" : "sdma"); 5893 5894 if (remote->use_lsdma) { 5895 r = amdgpu_lsdma_copy_mem(adev, ring->wptr_npa_gart, wptr_gpu, 8); 5896 if (!r) 5897 *wptr = *wptr_cpu; 5898 amdgpu_job_free(job); 5899 goto out; 5900 } 5901 5902 wptr_cb.wptr_cpu = wptr_cpu; 5903 5904 amdgpu_emit_copy_buffer(adev, ib, ring->wptr_npa_gart, wptr_gpu, 8, 0); 5905 5906 sdma_ring = &adev->sdma.instance[0].ring; 5907 amdgpu_ring_pad_ib(sdma_ring, ib); 5908 WARN_ON(ib->length_dw > ndw_copy_cmd); 5909 5910 fence = amdgpu_job_submit(job); 5911 5912 r = dma_fence_add_callback(fence, &wptr_cb.base, 5913 amdgpu_ualink_read_wptr_fence_cb); 5914 if (r == -ENOENT) { 5915 /* Fence already signaled; the callback won't run, do it inline. */ 5916 amdgpu_ualink_read_wptr_fence_cb(fence, &wptr_cb.base); 5917 r = 0; 5918 goto out_put_fence; 5919 } else if (r) { 5920 /* -EINVAL: NULL fence or func, not expected here. */ 5921 goto out_put_fence; 5922 } 5923 5924 r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT); 5925 if (r > 0) { 5926 r = 0; /* read wptr successfully */ 5927 goto out_put_fence; 5928 } 5929 5930 dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n", 5931 remote_accel_id, r); 5932 5933 if (r == 0) 5934 r = -ETIME; 5935 5936 /* 5937 * Timed out: the callback is still armed and wptr_cb is on our stack. 5938 * Cancel it so it can't fire after we return and write into the freed 5939 * stack frame. 5940 */ 5941 dma_fence_remove_callback(fence, &wptr_cb.base); 5942 5943 out_put_fence: 5944 if (!r) 5945 *wptr = wptr_cb.result; 5946 dma_fence_put(fence); 5947 5948 out: 5949 dev_dbg(adev->dev, "remote %u wptr 0x%llx return %d\n", 5950 remote_accel_id, *wptr, r); 5951 return r; 5952 } 5953 5954 /** 5955 * amdgpu_ualink_ring_resync_ptrs - sync local ring pointers from remote 5956 * @adev: amdgpu device pointer 5957 * @remote_accel_id: remote accelerator ID 5958 * 5959 * After a reboot on either side the two ends disagree on wptr. Read the remote 5960 * wptr of both peer rings and adopt it as the local wptr and rptr. 5961 * 5962 * Return 0 if both rings already match, 1 if they were resynced, or a negative 5963 * error code if a remote read failed, leaving the rings untouched. 5964 */ 5965 static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev, 5966 u32 remote_accel_id) 5967 { 5968 struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id]; 5969 struct amdgpu_ualink_wb *int_wb, *tlb_wb; 5970 u64 int_wptr, tlb_wptr; 5971 int r; 5972 5973 /* ring->wptr and ring->rptr are owned by the send path. */ 5974 guard(mutex)(&peer->lock); 5975 5976 /* Read both before updating either, so a failure changes nothing. */ 5977 r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id, 5978 &peer->interrupt, &int_wptr); 5979 if (!r) 5980 r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id, 5981 &peer->shootdown, &tlb_wptr); 5982 if (r) { 5983 dev_dbg(adev->dev, "accel_id %u read remote %u wptr failed %d\n", 5984 ualink_accel_id(adev), remote_accel_id, r); 5985 return r; 5986 } 5987 5988 if (peer->interrupt.wptr == int_wptr && peer->shootdown.wptr == tlb_wptr) { 5989 dev_dbg(adev->dev, "accel_id %u wptr 0x%llx 0x%llx is same as remote %u\n", 5990 ualink_accel_id(adev), int_wptr, tlb_wptr, remote_accel_id); 5991 return 0; 5992 } 5993 5994 dev_dbg(adev->dev, 5995 "accel_id %u wptr 0x%llx 0x%llx sync to remote %u wptr 0x%llx 0x%llx\n", 5996 ualink_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr, 5997 remote_accel_id, int_wptr, tlb_wptr); 5998 5999 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL, 6000 RB_TYPE_REMOTE_INTERRUPT); 6001 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL, 6002 RB_TYPE_TLB_INV); 6003 6004 peer->interrupt.wptr = int_wptr; 6005 peer->interrupt.rptr = int_wptr; 6006 peer->shootdown.wptr = tlb_wptr; 6007 peer->shootdown.rptr = tlb_wptr; 6008 6009 /* Drop the stale writeback rptr, it disagrees with the resynced wptr. */ 6010 WRITE_ONCE(int_wb->rptr, int_wptr); 6011 WRITE_ONCE(tlb_wb->rptr, tlb_wptr); 6012 return 1; 6013 } 6014 6015 static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev, 6016 u32 remote_accel_id) 6017 { 6018 struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id]; 6019 6020 dev_dbg(adev->dev, "remote accel_id %u\n", remote_accel_id); 6021 6022 guard(mutex)(&peer->lock); 6023 6024 peer->interrupt.ready = false; 6025 peer->shootdown.ready = false; 6026 } 6027 6028 static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev, 6029 u32 remote_accel_id, u32 ring_type) 6030 { 6031 struct amdgpu_ualink_remote *remote = to_remote(adev); 6032 struct amdgpu_ualink_peer *peer; 6033 struct amdgpu_ualink_ring *ring; 6034 struct amdgpu_ualink_wb *wb_cpu; 6035 u64 wb_npa; 6036 int r; 6037 6038 peer = &remote->peer[remote_accel_id]; 6039 if (ring_type == RB_TYPE_REMOTE_INTERRUPT) 6040 ring = &peer->interrupt; 6041 else if (ring_type == RB_TYPE_TLB_INV) 6042 ring = &peer->shootdown; 6043 else 6044 return -EINVAL; 6045 6046 amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa, 6047 ring_type); 6048 6049 r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu, 6050 amdgpu_ualink_emit_update_wb_addr, 6051 upper_32_bits(wb_npa), 6052 lower_32_bits(wb_npa), 6053 0, 0); 6054 if (!r) { 6055 ring->wb_cpu = wb_cpu; 6056 ring->ready = true; 6057 } 6058 return r; 6059 } 6060 6061 /** 6062 * amdgpu_ualink_remote_shootdown - Send a remote TLB invalidation request 6063 * @adev: amdgpu device pointer 6064 * @remote_accel_id: accelerator ID of the remote GPU to shootdown 6065 * @addr: page-aligned address to invalidate 6066 * @size_in_pages: number of pages to invalidate, 0 to invalidate entire TLB 6067 * @flush_type: type of TLB flush to perform 6068 * 6069 * Sends a remote TLB shootdown command to the specified peer GPU via the 6070 * UALink ring buffer. The address and size are translated using the S-field 6071 * encoding before being written to the ring. 6072 * 6073 * Return: 0 on success, negative error code on failure 6074 */ 6075 static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev, 6076 u32 remote_accel_id, u64 addr, 6077 u32 size_in_pages, u32 flush_type) 6078 { 6079 struct amdgpu_ualink_remote *remote = to_remote(adev); 6080 struct amdgpu_ualink_peer *peer; 6081 struct amdgpu_ualink_ring *ring; 6082 int r; 6083 6084 dev_dbg(adev->dev, "send remote shootdown to remote_accel_id %u\n", 6085 remote_accel_id); 6086 6087 peer = &remote->peer[remote_accel_id]; 6088 ring = &peer->shootdown; 6089 6090 guard(mutex)(&peer->lock); 6091 6092 if (!ring->ready) { 6093 dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id); 6094 r = amdgpu_ualink_update_wb_address(adev, remote_accel_id, 6095 RB_TYPE_TLB_INV); 6096 if (r) 6097 return r; 6098 } 6099 6100 return amdgpu_ualink_send_command(adev, remote_accel_id, ring, 6101 ring->wb_cpu, 6102 amdgpu_ualink_emit_shootdown, 6103 flush_type, upper_32_bits(addr), 6104 lower_32_bits(addr), size_in_pages); 6105 } 6106 6107 /** 6108 * amdgpu_ualink_remote_interrupt - Send a remote interrupt to a peer GPU 6109 * @adev: amdgpu device pointer 6110 * @remote_accel_id: accelerator ID of the remote GPU to interrupt 6111 * @dw0: IH context ID dword 0 6112 * @dw1: IH context ID dword 1 6113 * @dw2: IH context ID dword 2 6114 * @dw3: IH context ID dword 3 6115 * 6116 * Sends a remote interrupt command to the specified peer GPU via the 6117 * UALink ring buffer. The four context ID dwords are delivered to the 6118 * remote GPU's interrupt handler. 6119 * 6120 * Return: 0 on success, negative error code on failure 6121 */ 6122 static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev, 6123 u32 remote_accel_id, u32 dw0, u32 dw1, 6124 u32 dw2, u32 dw3) 6125 { 6126 struct amdgpu_ualink_remote *remote = to_remote(adev); 6127 struct amdgpu_ualink_peer *peer; 6128 struct amdgpu_ualink_ring *ring; 6129 int r; 6130 6131 peer = &remote->peer[remote_accel_id]; 6132 ring = &peer->interrupt; 6133 6134 guard(mutex)(&peer->lock); 6135 6136 if (!ring->ready) { 6137 dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id); 6138 r = amdgpu_ualink_update_wb_address(adev, remote_accel_id, 6139 RB_TYPE_REMOTE_INTERRUPT); 6140 if (r) 6141 return r; 6142 } 6143 6144 return amdgpu_ualink_send_command(adev, remote_accel_id, ring, 6145 ring->wb_cpu, 6146 amdgpu_ualink_emit_interrupt, 6147 dw0, dw1, dw2, dw3); 6148 } 6149 6150 /** 6151 * amdgpu_ualink_peer_remote_init - Initialize remote peer GPU connections 6152 * @adev: amdgpu device pointer 6153 * 6154 * Sets up ring buffers, GART mappings, and control structures for 6155 * communicating with remote GPUs in the fabric. 6156 * 6157 * Return: 0 on success, negative error code on failure 6158 */ 6159 static int amdgpu_ualink_peer_remote_init(struct amdgpu_device *adev) 6160 { 6161 u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT; 6162 struct amdgpu_ualink_remote *remote = to_remote(adev); 6163 struct amdgpu_bo *bo = remote->ring_bo; 6164 struct amdgpu_ualink_peer *peer; 6165 struct amdgpu_ualink_ring *interrupt; 6166 struct amdgpu_ualink_ring *shootdown; 6167 u32 src_accel_id = ualink_accel_id(adev); 6168 u32 dst_accel_id; 6169 u64 npa, flags; 6170 int r = 0; 6171 6172 /* NPA mapping PTE flags VSCT = 0011 */ 6173 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, bo->tbo.resource); 6174 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | AMDGPU_PTE_BUS_ATOMICS; 6175 flags &= ~AMDGPU_PTE_VALID; 6176 flags &= ~AMDGPU_PTE_SYSTEM; 6177 /* PTE.X=0 turn off RPC checks for RBs, wptr and doorbell NPA address */ 6178 flags &= ~AMDGPU_PTE_EXECUTABLE; 6179 6180 dev_dbg(adev->dev, "src_accel_id %u gart mapping flags 0x%llx\n", 6181 src_accel_id, flags); 6182 6183 for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 6184 if (dst_accel_id == src_accel_id) 6185 continue; 6186 6187 dev_dbg(adev->dev, "peer_remote to dst_accel_id %u\n", dst_accel_id); 6188 6189 peer = &remote->peer[dst_accel_id]; 6190 mutex_init(&peer->lock); 6191 6192 interrupt = &peer->interrupt; 6193 shootdown = &peer->shootdown; 6194 6195 /* ring buffer npa gart mapping */ 6196 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT, 6197 src_accel_id, dst_accel_id); 6198 r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &interrupt->mm_node_rb, flags); 6199 if (r) 6200 break; 6201 interrupt->rb_npa_gart = adev->gmc.gart_start + 6202 (interrupt->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT); 6203 6204 dev_dbg(adev->dev, "rb npa 0x%llx mapped on gart 0x%llx\n", 6205 npa, interrupt->rb_npa_gart); 6206 6207 /* tlb invalidate ring buffer npa gart mapping */ 6208 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TLB_INV, src_accel_id, 6209 dst_accel_id); 6210 r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &shootdown->mm_node_rb, flags); 6211 if (r) 6212 break; 6213 shootdown->rb_npa_gart = adev->gmc.gart_start + 6214 (shootdown->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT); 6215 6216 dev_dbg(adev->dev, "tlb rb npa 0x%llx mapped on gart 0x%llx\n", 6217 npa, shootdown->rb_npa_gart); 6218 6219 /* wptr npa gart mapping */ 6220 npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TAILPTR, src_accel_id, 6221 dst_accel_id); 6222 r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_wptr, flags); 6223 if (r) 6224 break; 6225 6226 interrupt->wptr_npa_gart = adev->gmc.gart_start + 6227 (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT) 6228 + ualink_wptr_offset(adev, src_accel_id); 6229 shootdown->wptr_npa_gart = adev->gmc.gart_start + 6230 (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT) 6231 + ualink_tlb_wptr_offset(adev, src_accel_id); 6232 6233 dev_dbg(adev->dev, "rb wptr npa 0x%llx on gart 0x%llx, tlb wptr on gart 0x%llx\n", 6234 npa, interrupt->wptr_npa_gart, shootdown->wptr_npa_gart); 6235 6236 /* 6237 * doorbell npa gart mapping, doorbell NPA address coding 6238 * 6239 * ReqAddr[51:40] = 0xFFF 6240 * ReqAddr[39:30] = GPU accel_id 6241 * ReqAddr[29:12] = 18'h00000 6242 * ReqAddr[11:8] = Random value 6243 * ReqAddr[7:6] = PMI Index 6244 * 0 - PMI0 Remote Interrupt 6245 * 1 - PMI1 Remote TLB Shootdown 6246 * ReqAddr[5:0] = 6'h0 6247 */ 6248 npa = 0xFFFULL << 40 | (u64)dst_accel_id << 30; 6249 r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_doorbell, flags); 6250 if (r) 6251 break; 6252 6253 interrupt->doorbell_npa_gart = adev->gmc.gart_start + 6254 (interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT); 6255 6256 dev_dbg(adev->dev, "rb doorbell npa 0x%llx mapped on gart 0x%llx\n", 6257 npa, interrupt->doorbell_npa_gart); 6258 6259 shootdown->doorbell_npa_gart = adev->gmc.gart_start+ 6260 (interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT) + (1 << 6); 6261 6262 dev_dbg(adev->dev, "tlb doorbell npa 0x%llx mapped on gart 0x%llx\n", 6263 npa, shootdown->doorbell_npa_gart); 6264 6265 interrupt->rb_size = AMDGPU_UALINK_RB_SIZE / 64; 6266 interrupt->wptr = 0; 6267 interrupt->rptr = 0; 6268 interrupt->seq = 0; 6269 6270 shootdown->rb_size = AMDGPU_UALINK_RB_SIZE / 64; 6271 shootdown->wptr = 0; 6272 shootdown->rptr = 0; 6273 shootdown->seq = 0; 6274 } 6275 6276 dev_dbg(adev->dev, "peer remote init done r=%d\n", r); 6277 return r; 6278 } 6279 6280 /** 6281 * amdgpu_ualink_peer_remote_fini - Clean up remote peer GPU connections 6282 * @adev: amdgpu device pointer 6283 * 6284 * Unmaps GART entries and frees resources for remote GPU communication. 6285 */ 6286 static void amdgpu_ualink_peer_remote_fini(struct amdgpu_device *adev) 6287 { 6288 u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT; 6289 struct amdgpu_ualink_remote *remote = to_remote(adev); 6290 u32 src_accel_id = ualink_accel_id(adev); 6291 u32 dst_accel_id; 6292 struct amdgpu_ualink_peer *peer; 6293 struct amdgpu_ualink_ring *ring; 6294 6295 dev_dbg(adev->dev, "src_accel_id %u\n", src_accel_id); 6296 6297 for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) { 6298 dev_dbg(adev->dev, "dst_accel_id %u\n", dst_accel_id); 6299 if (dst_accel_id == src_accel_id) 6300 continue; 6301 6302 peer = &remote->peer[dst_accel_id]; 6303 mutex_destroy(&peer->lock); 6304 6305 ring = &peer->interrupt; 6306 amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb); 6307 amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_wptr); 6308 amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_doorbell); 6309 6310 ring = &peer->shootdown; 6311 amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb); 6312 } 6313 } 6314 6315 /** 6316 * amdgpu_ualink_sw_init - Initialize UALink software resources 6317 * @adev: amdgpu device pointer 6318 * 6319 * Performs full UALink software initialization including metadata allocation, 6320 * NPA address mapping, SDMA scheduler entity setup, and remote peer GPU 6321 * connection initialization. On failure, all previously initialized resources 6322 * are cleaned up. 6323 * 6324 * Return: 0 on success, negative error code on failure 6325 */ 6326 int amdgpu_ualink_sw_init(struct amdgpu_device *adev) 6327 { 6328 struct amdgpu_ualink_remote *remote; 6329 int r; 6330 6331 remote = kzalloc_obj(*remote); 6332 if (!remote) 6333 return -ENOMEM; 6334 adev->ualink.remote = remote; 6335 6336 r = amdgpu_ualink_metadata_init(adev); 6337 if (r) 6338 goto out; 6339 6340 r = amdgpu_ualink_metadata_npa_mapping(adev); 6341 if (r) 6342 goto out_metadata_fini; 6343 6344 r = amdgpu_ualink_sdma_entities_init(adev); 6345 if (r) 6346 goto out_npa_unmap; 6347 6348 r = amdgpu_ualink_peer_remote_init(adev); 6349 if (r) 6350 goto out_sdma_entities_fini; 6351 6352 dev_dbg(adev->dev, "ualink sw init succeed\n"); 6353 return 0; 6354 6355 out_sdma_entities_fini: 6356 amdgpu_ualink_sdma_entities_fini(adev); 6357 out_npa_unmap: 6358 amdgpu_ualink_metadata_npa_unmapping(adev); 6359 out_metadata_fini: 6360 amdgpu_ualink_metadata_fini(adev); 6361 out: 6362 kfree(adev->ualink.remote); 6363 adev->ualink.remote = NULL; 6364 dev_dbg(adev->dev, "ualink sw init failed %d\n", r); 6365 return r; 6366 } 6367 6368 /** 6369 * amdgpu_ualink_sw_fini - Tear down UALink software resources 6370 * @adev: amdgpu device pointer 6371 * 6372 * Cleans up all UALink software resources in reverse order of initialization: 6373 * remote peer connections, SDMA entities, NPA mappings, and metadata. 6374 */ 6375 void amdgpu_ualink_sw_fini(struct amdgpu_device *adev) 6376 { 6377 dev_dbg(adev->dev, "halt accel_id %u addr_mode %d\n", ualink_accel_id(adev), 6378 ualink_addr_mode(adev)); 6379 6380 adev->ualink.msg_ctl->send_halt(adev); 6381 6382 amdgpu_ualink_peer_remote_fini(adev); 6383 amdgpu_ualink_sdma_entities_fini(adev); 6384 amdgpu_ualink_metadata_npa_unmapping(adev); 6385 amdgpu_ualink_metadata_fini(adev); 6386 kfree(adev->ualink.remote); 6387 adev->ualink.remote = NULL; 6388 } 6389 6390 int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id) 6391 { 6392 return amdgpu_ualink_send_hello_msg(adev, remote_accel_id); 6393 } 6394 6395 /* 6396 * The low bits of handle_lo carry the message-header field on the wire. 6397 * Strip those bits and splice in @acc_id to recover the 6398 * fully-qualified handle_lo value. 6399 */ 6400 static inline u64 amdgpu_ualink_reassemble_handle(u64 handle_lo, u32 acc_id) 6401 { 6402 return (handle_lo & ~AMDGPU_UALINK_MESSAGE_HEADER_MASK) | 6403 (acc_id & AMDGPU_UALINK_HANDLE_ACCID_MASK); 6404 } 6405 6406 static int amdgpu_ualink_process_irq(struct amdgpu_device *adev, 6407 struct amdgpu_irq_src *source, 6408 struct amdgpu_iv_entry *entry) 6409 { 6410 u32 sender_acc_id, receiver_acc_id, msg_type, src_acc_id; 6411 enum amdgpu_ualink_accel_state accel_state; 6412 struct amdgpu_ualink_handle handle; 6413 u32 size, npa_addr, fail_reason; 6414 u32 dw0, dw1, dw2, dw3; 6415 u32 local_acc_id; 6416 int handled = 1; 6417 6418 if (unlikely(adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)) { 6419 dev_dbg(adev->dev, 6420 "UALink manager not initialized, dropping irq\n"); 6421 return handled; 6422 } 6423 6424 /* Only handle interrupts while ACTIVE. Otherwise (e.g. during teardown) 6425 * consume stale ring entries but drop them, so they don't touch state 6426 * being freed. 6427 */ 6428 accel_state = adev->ualink.info->accel_state; 6429 if (accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) { 6430 dev_dbg(adev->dev, 6431 "Dropping stale UALink interrupt, accel_state %d\n", 6432 accel_state); 6433 return handled; 6434 } 6435 6436 dev_dbg(adev->dev, "%s client_id 0x%x src_id 0x%x ih\n", 6437 entry->ih == &adev->irq.ih ? "ring" : "ualink soft ring", 6438 entry->client_id, entry->src_id); 6439 6440 /* ContextID 4 dwords */ 6441 src_acc_id = entry->pasid; 6442 dw0 = entry->src_data[0]; 6443 dw1 = entry->src_data[1]; 6444 dw2 = entry->src_data[2]; 6445 dw3 = entry->src_data[3]; 6446 6447 /* Copy IH entry into ualink soft ring. */ 6448 if (entry->ih == &adev->irq.ih) { 6449 dev_dbg(adev->dev, "src accel_id %u context id 0x%x 0x%x 0x%x 0x%x\n", 6450 src_acc_id, dw0, dw1, dw2, dw3); 6451 dev_dbg(adev->dev, "delegate to ualink irq soft ring\n"); 6452 amdgpu_irq_ualink_delegate(adev, entry, 8); 6453 return handled; 6454 } 6455 6456 msg_type = dw0 & AMDGPU_UALINK_MESSAGE_HEADER_MASK; 6457 local_acc_id = adev->ualink.info->ppod.accel_id; 6458 dev_dbg(adev->dev, "Got MSG: remote acc_id %u msg_type %u\n", 6459 src_acc_id, msg_type); 6460 6461 /* Debug hook: if the bit corresponding to this msg_type is set in 6462 * drop_msg_bitmap, drop this single packet and clear the bit so that 6463 * any subsequent incoming messages are processed normally. Used to 6464 * exercise the connection reset/recovery paths via debugfs. 6465 */ 6466 if (msg_type < BITS_PER_LONG && 6467 test_and_clear_bit(msg_type, &adev->ualink.drop_msg_bitmap)) { 6468 dev_warn(adev->dev, 6469 "DROP MSG (debugfs): src acc_id %u msg_type %u dw[0-3] 0x%x 0x%x 0x%x 0x%x\n", 6470 src_acc_id, msg_type, dw0, dw1, dw2, dw3); 6471 return handled; 6472 } 6473 6474 switch (msg_type) { 6475 case AMDGPU_UALINK_HELLO_MSG: 6476 receiver_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT) & 6477 AMDGPU_UALINK_HELLO_MSG_ACCID_MASK; 6478 sender_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT) & 6479 AMDGPU_UALINK_HELLO_MSG_ACCID_MASK; 6480 dev_dbg(adev->dev, 6481 "Got HELLO MSG: src acc_id %u receiver_acc_id %u sender_acc_id %u\n", 6482 src_acc_id, receiver_acc_id, sender_acc_id); 6483 amdgpu_ualink_process_hello_msg(adev, receiver_acc_id, sender_acc_id, 6484 src_acc_id); 6485 break; 6486 case AMDGPU_UALINK_HELLO_ACK_MSG: 6487 dev_dbg(adev->dev, 6488 "Got HELLO-ACK MSG: remote acc_id %u\n", src_acc_id); 6489 amdgpu_ualink_process_hello_ack_msg(adev, src_acc_id); 6490 break; 6491 case AMDGPU_UALINK_NPA_REQ_MSG: 6492 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6493 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6494 local_acc_id); 6495 dev_dbg(adev->dev, 6496 "Got NPA-REQ MSG: remote acc_id %u handle %llx:%llx\n", 6497 src_acc_id, handle.handle_hi, handle.handle_lo); 6498 amdgpu_ualink_process_npa_req_msg(adev, src_acc_id, handle); 6499 break; 6500 case AMDGPU_UALINK_NPA_RSP_MSG: 6501 memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo)); 6502 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6503 src_acc_id); 6504 size = entry->src_data[2]; 6505 npa_addr = entry->src_data[3]; 6506 dev_dbg(adev->dev, 6507 "Got NPA-RSP MSG: remote acc_id %u handle_lo %llx size 0x%x npa_addr 0x%x\n", 6508 src_acc_id, handle.handle_lo, size, npa_addr); 6509 amdgpu_ualink_process_npa_rsp_msg(adev, src_acc_id, 6510 handle.handle_lo, 6511 npa_addr, size); 6512 break; 6513 case AMDGPU_UALINK_NPA_FAIL_MSG: 6514 memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo)); 6515 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6516 src_acc_id); 6517 fail_reason = entry->src_data[2] & 6518 AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK; 6519 dev_dbg(adev->dev, 6520 "Got NPA-FAIL MSG: remote acc_id %u handle_lo %llx fail_reason %u\n", 6521 src_acc_id, handle.handle_lo, fail_reason); 6522 amdgpu_ualink_process_npa_fail_msg(adev, src_acc_id, 6523 handle.handle_lo, 6524 fail_reason); 6525 break; 6526 case AMDGPU_UALINK_NPA_REVOKE_MSG: 6527 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6528 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6529 src_acc_id); 6530 dev_dbg(adev->dev, 6531 "Got NPA-REVOKE MSG: remote acc_id %u handle %llx:%llx\n", 6532 src_acc_id, handle.handle_hi, handle.handle_lo); 6533 amdgpu_ualink_process_npa_revoke_msg(adev, src_acc_id, handle); 6534 break; 6535 case AMDGPU_UALINK_NPA_RELEASE_MSG: 6536 memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle)); 6537 handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo, 6538 local_acc_id); 6539 dev_dbg(adev->dev, 6540 "Got NPA-RELEASE MSG: remote acc_id %u handle %llx:%llx\n", 6541 src_acc_id, handle.handle_hi, handle.handle_lo); 6542 amdgpu_ualink_process_npa_release_msg(adev, src_acc_id, handle); 6543 break; 6544 default: 6545 dev_err(adev->dev, "Unknown message type (%u)\n", msg_type); 6546 break; 6547 } 6548 6549 return handled; 6550 } 6551 6552 static int amdgpu_ualink_set_irq_state(struct amdgpu_device *adev, 6553 struct amdgpu_irq_src *source, 6554 u32 type, 6555 enum amdgpu_interrupt_state state) 6556 { 6557 /* 6558 * Don't set register to enable/disable nHT controller interrupt. 6559 * 6560 * F/W running on MP2, which can always send cookie to IH block to 6561 * interrupt driver. 6562 */ 6563 dev_dbg(adev->dev, "ualink interrupt %s\n", 6564 state == AMDGPU_IRQ_STATE_ENABLE ? "enable" : "disable"); 6565 return 0; 6566 } 6567 6568 static const struct amdgpu_irq_src_funcs ualink_irq_funcs = { 6569 .set = amdgpu_ualink_set_irq_state, 6570 .process = amdgpu_ualink_process_irq, 6571 }; 6572 6573 /** 6574 * amdgpu_ualink_init_interrupt - register the UALink IRQ source 6575 * @adev: amdgpu device pointer 6576 * @client_id: IH client ID for the interrupt source 6577 * @src_id: source ID for the interrupt source 6578 * 6579 * Registers the UALink interrupt source with the IH subsystem. 6580 * 6581 * Return: 0 on success, negative error code on failure 6582 */ 6583 int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev, 6584 unsigned int client_id, unsigned int src_id) 6585 { 6586 int r; 6587 6588 dev_dbg(adev->dev, "init ualink irq client_id 0x%x src_id 0x%x\n", 6589 client_id, src_id); 6590 6591 adev->ualink.irq.num_types = 1; 6592 adev->ualink.irq.funcs = &ualink_irq_funcs; 6593 6594 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->ualink.irq); 6595 return r; 6596 } 6597 6598