1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/types.h> 25 #include <linux/kernel.h> 26 #include <linux/pci.h> 27 #include <linux/errno.h> 28 #include <linux/acpi.h> 29 #include <linux/hash.h> 30 #include <linux/cpufreq.h> 31 #include <linux/log2.h> 32 #include <linux/dmi.h> 33 #include <linux/atomic.h> 34 35 #include "kfd_priv.h" 36 #include "kfd_crat.h" 37 #include "kfd_topology.h" 38 #include "kfd_device_queue_manager.h" 39 #include "kfd_iommu.h" 40 #include "kfd_svm.h" 41 #include "amdgpu_amdkfd.h" 42 #include "amdgpu_ras.h" 43 #include "amdgpu.h" 44 45 /* topology_device_list - Master list of all topology devices */ 46 static struct list_head topology_device_list; 47 static struct kfd_system_properties sys_props; 48 49 static DECLARE_RWSEM(topology_lock); 50 static uint32_t topology_crat_proximity_domain; 51 52 struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock( 53 uint32_t proximity_domain) 54 { 55 struct kfd_topology_device *top_dev; 56 struct kfd_topology_device *device = NULL; 57 58 list_for_each_entry(top_dev, &topology_device_list, list) 59 if (top_dev->proximity_domain == proximity_domain) { 60 device = top_dev; 61 break; 62 } 63 64 return device; 65 } 66 67 struct kfd_topology_device *kfd_topology_device_by_proximity_domain( 68 uint32_t proximity_domain) 69 { 70 struct kfd_topology_device *device = NULL; 71 72 down_read(&topology_lock); 73 74 device = kfd_topology_device_by_proximity_domain_no_lock( 75 proximity_domain); 76 up_read(&topology_lock); 77 78 return device; 79 } 80 81 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id) 82 { 83 struct kfd_topology_device *top_dev = NULL; 84 struct kfd_topology_device *ret = NULL; 85 86 down_read(&topology_lock); 87 88 list_for_each_entry(top_dev, &topology_device_list, list) 89 if (top_dev->gpu_id == gpu_id) { 90 ret = top_dev; 91 break; 92 } 93 94 up_read(&topology_lock); 95 96 return ret; 97 } 98 99 struct kfd_node *kfd_device_by_id(uint32_t gpu_id) 100 { 101 struct kfd_topology_device *top_dev; 102 103 top_dev = kfd_topology_device_by_id(gpu_id); 104 if (!top_dev) 105 return NULL; 106 107 return top_dev->gpu; 108 } 109 110 struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev) 111 { 112 struct kfd_topology_device *top_dev; 113 struct kfd_node *device = NULL; 114 115 down_read(&topology_lock); 116 117 list_for_each_entry(top_dev, &topology_device_list, list) 118 if (top_dev->gpu && top_dev->gpu->adev->pdev == pdev) { 119 device = top_dev->gpu; 120 break; 121 } 122 123 up_read(&topology_lock); 124 125 return device; 126 } 127 128 /* Called with write topology_lock acquired */ 129 static void kfd_release_topology_device(struct kfd_topology_device *dev) 130 { 131 struct kfd_mem_properties *mem; 132 struct kfd_cache_properties *cache; 133 struct kfd_iolink_properties *iolink; 134 struct kfd_iolink_properties *p2plink; 135 struct kfd_perf_properties *perf; 136 137 list_del(&dev->list); 138 139 while (dev->mem_props.next != &dev->mem_props) { 140 mem = container_of(dev->mem_props.next, 141 struct kfd_mem_properties, list); 142 list_del(&mem->list); 143 kfree(mem); 144 } 145 146 while (dev->cache_props.next != &dev->cache_props) { 147 cache = container_of(dev->cache_props.next, 148 struct kfd_cache_properties, list); 149 list_del(&cache->list); 150 kfree(cache); 151 } 152 153 while (dev->io_link_props.next != &dev->io_link_props) { 154 iolink = container_of(dev->io_link_props.next, 155 struct kfd_iolink_properties, list); 156 list_del(&iolink->list); 157 kfree(iolink); 158 } 159 160 while (dev->p2p_link_props.next != &dev->p2p_link_props) { 161 p2plink = container_of(dev->p2p_link_props.next, 162 struct kfd_iolink_properties, list); 163 list_del(&p2plink->list); 164 kfree(p2plink); 165 } 166 167 while (dev->perf_props.next != &dev->perf_props) { 168 perf = container_of(dev->perf_props.next, 169 struct kfd_perf_properties, list); 170 list_del(&perf->list); 171 kfree(perf); 172 } 173 174 kfree(dev); 175 } 176 177 void kfd_release_topology_device_list(struct list_head *device_list) 178 { 179 struct kfd_topology_device *dev; 180 181 while (!list_empty(device_list)) { 182 dev = list_first_entry(device_list, 183 struct kfd_topology_device, list); 184 kfd_release_topology_device(dev); 185 } 186 } 187 188 static void kfd_release_live_view(void) 189 { 190 kfd_release_topology_device_list(&topology_device_list); 191 memset(&sys_props, 0, sizeof(sys_props)); 192 } 193 194 struct kfd_topology_device *kfd_create_topology_device( 195 struct list_head *device_list) 196 { 197 struct kfd_topology_device *dev; 198 199 dev = kfd_alloc_struct(dev); 200 if (!dev) { 201 pr_err("No memory to allocate a topology device"); 202 return NULL; 203 } 204 205 INIT_LIST_HEAD(&dev->mem_props); 206 INIT_LIST_HEAD(&dev->cache_props); 207 INIT_LIST_HEAD(&dev->io_link_props); 208 INIT_LIST_HEAD(&dev->p2p_link_props); 209 INIT_LIST_HEAD(&dev->perf_props); 210 211 list_add_tail(&dev->list, device_list); 212 213 return dev; 214 } 215 216 217 #define sysfs_show_gen_prop(buffer, offs, fmt, ...) \ 218 (offs += snprintf(buffer+offs, PAGE_SIZE-offs, \ 219 fmt, __VA_ARGS__)) 220 #define sysfs_show_32bit_prop(buffer, offs, name, value) \ 221 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value) 222 #define sysfs_show_64bit_prop(buffer, offs, name, value) \ 223 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value) 224 #define sysfs_show_32bit_val(buffer, offs, value) \ 225 sysfs_show_gen_prop(buffer, offs, "%u\n", value) 226 #define sysfs_show_str_val(buffer, offs, value) \ 227 sysfs_show_gen_prop(buffer, offs, "%s\n", value) 228 229 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, 230 char *buffer) 231 { 232 int offs = 0; 233 234 /* Making sure that the buffer is an empty string */ 235 buffer[0] = 0; 236 237 if (attr == &sys_props.attr_genid) { 238 sysfs_show_32bit_val(buffer, offs, 239 sys_props.generation_count); 240 } else if (attr == &sys_props.attr_props) { 241 sysfs_show_64bit_prop(buffer, offs, "platform_oem", 242 sys_props.platform_oem); 243 sysfs_show_64bit_prop(buffer, offs, "platform_id", 244 sys_props.platform_id); 245 sysfs_show_64bit_prop(buffer, offs, "platform_rev", 246 sys_props.platform_rev); 247 } else { 248 offs = -EINVAL; 249 } 250 251 return offs; 252 } 253 254 static void kfd_topology_kobj_release(struct kobject *kobj) 255 { 256 kfree(kobj); 257 } 258 259 static const struct sysfs_ops sysprops_ops = { 260 .show = sysprops_show, 261 }; 262 263 static const struct kobj_type sysprops_type = { 264 .release = kfd_topology_kobj_release, 265 .sysfs_ops = &sysprops_ops, 266 }; 267 268 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, 269 char *buffer) 270 { 271 int offs = 0; 272 struct kfd_iolink_properties *iolink; 273 274 /* Making sure that the buffer is an empty string */ 275 buffer[0] = 0; 276 277 iolink = container_of(attr, struct kfd_iolink_properties, attr); 278 if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu)) 279 return -EPERM; 280 sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type); 281 sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj); 282 sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min); 283 sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from); 284 sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to); 285 sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight); 286 sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency); 287 sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency); 288 sysfs_show_32bit_prop(buffer, offs, "min_bandwidth", 289 iolink->min_bandwidth); 290 sysfs_show_32bit_prop(buffer, offs, "max_bandwidth", 291 iolink->max_bandwidth); 292 sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size", 293 iolink->rec_transfer_size); 294 sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags); 295 296 return offs; 297 } 298 299 static const struct sysfs_ops iolink_ops = { 300 .show = iolink_show, 301 }; 302 303 static const struct kobj_type iolink_type = { 304 .release = kfd_topology_kobj_release, 305 .sysfs_ops = &iolink_ops, 306 }; 307 308 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, 309 char *buffer) 310 { 311 int offs = 0; 312 struct kfd_mem_properties *mem; 313 314 /* Making sure that the buffer is an empty string */ 315 buffer[0] = 0; 316 317 mem = container_of(attr, struct kfd_mem_properties, attr); 318 if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu)) 319 return -EPERM; 320 sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type); 321 sysfs_show_64bit_prop(buffer, offs, "size_in_bytes", 322 mem->size_in_bytes); 323 sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags); 324 sysfs_show_32bit_prop(buffer, offs, "width", mem->width); 325 sysfs_show_32bit_prop(buffer, offs, "mem_clk_max", 326 mem->mem_clk_max); 327 328 return offs; 329 } 330 331 static const struct sysfs_ops mem_ops = { 332 .show = mem_show, 333 }; 334 335 static const struct kobj_type mem_type = { 336 .release = kfd_topology_kobj_release, 337 .sysfs_ops = &mem_ops, 338 }; 339 340 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, 341 char *buffer) 342 { 343 int offs = 0; 344 uint32_t i, j; 345 struct kfd_cache_properties *cache; 346 347 /* Making sure that the buffer is an empty string */ 348 buffer[0] = 0; 349 cache = container_of(attr, struct kfd_cache_properties, attr); 350 if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu)) 351 return -EPERM; 352 sysfs_show_32bit_prop(buffer, offs, "processor_id_low", 353 cache->processor_id_low); 354 sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level); 355 sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size); 356 sysfs_show_32bit_prop(buffer, offs, "cache_line_size", 357 cache->cacheline_size); 358 sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag", 359 cache->cachelines_per_tag); 360 sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc); 361 sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency); 362 sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type); 363 364 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map "); 365 for (i = 0; i < cache->sibling_map_size; i++) 366 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) 367 /* Check each bit */ 368 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,", 369 (cache->sibling_map[i] >> j) & 1); 370 371 /* Replace the last "," with end of line */ 372 buffer[offs-1] = '\n'; 373 return offs; 374 } 375 376 static const struct sysfs_ops cache_ops = { 377 .show = kfd_cache_show, 378 }; 379 380 static const struct kobj_type cache_type = { 381 .release = kfd_topology_kobj_release, 382 .sysfs_ops = &cache_ops, 383 }; 384 385 /****** Sysfs of Performance Counters ******/ 386 387 struct kfd_perf_attr { 388 struct kobj_attribute attr; 389 uint32_t data; 390 }; 391 392 static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs, 393 char *buf) 394 { 395 int offs = 0; 396 struct kfd_perf_attr *attr; 397 398 buf[0] = 0; 399 attr = container_of(attrs, struct kfd_perf_attr, attr); 400 if (!attr->data) /* invalid data for PMC */ 401 return 0; 402 else 403 return sysfs_show_32bit_val(buf, offs, attr->data); 404 } 405 406 #define KFD_PERF_DESC(_name, _data) \ 407 { \ 408 .attr = __ATTR(_name, 0444, perf_show, NULL), \ 409 .data = _data, \ 410 } 411 412 static struct kfd_perf_attr perf_attr_iommu[] = { 413 KFD_PERF_DESC(max_concurrent, 0), 414 KFD_PERF_DESC(num_counters, 0), 415 KFD_PERF_DESC(counter_ids, 0), 416 }; 417 /****************************************/ 418 419 static ssize_t node_show(struct kobject *kobj, struct attribute *attr, 420 char *buffer) 421 { 422 int offs = 0; 423 struct kfd_topology_device *dev; 424 uint32_t log_max_watch_addr; 425 426 /* Making sure that the buffer is an empty string */ 427 buffer[0] = 0; 428 429 if (strcmp(attr->name, "gpu_id") == 0) { 430 dev = container_of(attr, struct kfd_topology_device, 431 attr_gpuid); 432 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 433 return -EPERM; 434 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id); 435 } 436 437 if (strcmp(attr->name, "name") == 0) { 438 dev = container_of(attr, struct kfd_topology_device, 439 attr_name); 440 441 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 442 return -EPERM; 443 return sysfs_show_str_val(buffer, offs, dev->node_props.name); 444 } 445 446 dev = container_of(attr, struct kfd_topology_device, 447 attr_props); 448 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 449 return -EPERM; 450 sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count", 451 dev->node_props.cpu_cores_count); 452 sysfs_show_32bit_prop(buffer, offs, "simd_count", 453 dev->gpu ? (dev->node_props.simd_count * 454 NUM_XCC(dev->gpu->xcc_mask)) : 0); 455 sysfs_show_32bit_prop(buffer, offs, "mem_banks_count", 456 dev->node_props.mem_banks_count); 457 sysfs_show_32bit_prop(buffer, offs, "caches_count", 458 dev->node_props.caches_count); 459 sysfs_show_32bit_prop(buffer, offs, "io_links_count", 460 dev->node_props.io_links_count); 461 sysfs_show_32bit_prop(buffer, offs, "p2p_links_count", 462 dev->node_props.p2p_links_count); 463 sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base", 464 dev->node_props.cpu_core_id_base); 465 sysfs_show_32bit_prop(buffer, offs, "simd_id_base", 466 dev->node_props.simd_id_base); 467 sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd", 468 dev->node_props.max_waves_per_simd); 469 sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb", 470 dev->node_props.lds_size_in_kb); 471 sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb", 472 dev->node_props.gds_size_in_kb); 473 sysfs_show_32bit_prop(buffer, offs, "num_gws", 474 dev->node_props.num_gws); 475 sysfs_show_32bit_prop(buffer, offs, "wave_front_size", 476 dev->node_props.wave_front_size); 477 sysfs_show_32bit_prop(buffer, offs, "array_count", 478 dev->gpu ? (dev->node_props.array_count * 479 NUM_XCC(dev->gpu->xcc_mask)) : 0); 480 sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine", 481 dev->node_props.simd_arrays_per_engine); 482 sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array", 483 dev->node_props.cu_per_simd_array); 484 sysfs_show_32bit_prop(buffer, offs, "simd_per_cu", 485 dev->node_props.simd_per_cu); 486 sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu", 487 dev->node_props.max_slots_scratch_cu); 488 sysfs_show_32bit_prop(buffer, offs, "gfx_target_version", 489 dev->node_props.gfx_target_version); 490 sysfs_show_32bit_prop(buffer, offs, "vendor_id", 491 dev->node_props.vendor_id); 492 sysfs_show_32bit_prop(buffer, offs, "device_id", 493 dev->node_props.device_id); 494 sysfs_show_32bit_prop(buffer, offs, "location_id", 495 dev->node_props.location_id); 496 sysfs_show_32bit_prop(buffer, offs, "domain", 497 dev->node_props.domain); 498 sysfs_show_32bit_prop(buffer, offs, "drm_render_minor", 499 dev->node_props.drm_render_minor); 500 sysfs_show_64bit_prop(buffer, offs, "hive_id", 501 dev->node_props.hive_id); 502 sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines", 503 dev->node_props.num_sdma_engines); 504 sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines", 505 dev->node_props.num_sdma_xgmi_engines); 506 sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine", 507 dev->node_props.num_sdma_queues_per_engine); 508 sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", 509 dev->node_props.num_cp_queues); 510 511 if (dev->gpu) { 512 log_max_watch_addr = 513 __ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points); 514 515 if (log_max_watch_addr) { 516 dev->node_props.capability |= 517 HSA_CAP_WATCH_POINTS_SUPPORTED; 518 519 dev->node_props.capability |= 520 ((log_max_watch_addr << 521 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & 522 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); 523 } 524 525 if (dev->gpu->adev->asic_type == CHIP_TONGA) 526 dev->node_props.capability |= 527 HSA_CAP_AQL_QUEUE_DOUBLE_MAP; 528 529 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute", 530 dev->node_props.max_engine_clk_fcompute); 531 532 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL); 533 534 sysfs_show_32bit_prop(buffer, offs, "fw_version", 535 dev->gpu->kfd->mec_fw_version); 536 sysfs_show_32bit_prop(buffer, offs, "capability", 537 dev->node_props.capability); 538 sysfs_show_64bit_prop(buffer, offs, "debug_prop", 539 dev->node_props.debug_prop); 540 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", 541 dev->gpu->kfd->sdma_fw_version); 542 sysfs_show_64bit_prop(buffer, offs, "unique_id", 543 dev->gpu->adev->unique_id); 544 sysfs_show_32bit_prop(buffer, offs, "num_xcc", 545 NUM_XCC(dev->gpu->xcc_mask)); 546 } 547 548 return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", 549 cpufreq_quick_get_max(0)/1000); 550 } 551 552 static const struct sysfs_ops node_ops = { 553 .show = node_show, 554 }; 555 556 static const struct kobj_type node_type = { 557 .release = kfd_topology_kobj_release, 558 .sysfs_ops = &node_ops, 559 }; 560 561 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) 562 { 563 sysfs_remove_file(kobj, attr); 564 kobject_del(kobj); 565 kobject_put(kobj); 566 } 567 568 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) 569 { 570 struct kfd_iolink_properties *p2plink; 571 struct kfd_iolink_properties *iolink; 572 struct kfd_cache_properties *cache; 573 struct kfd_mem_properties *mem; 574 struct kfd_perf_properties *perf; 575 576 if (dev->kobj_iolink) { 577 list_for_each_entry(iolink, &dev->io_link_props, list) 578 if (iolink->kobj) { 579 kfd_remove_sysfs_file(iolink->kobj, 580 &iolink->attr); 581 iolink->kobj = NULL; 582 } 583 kobject_del(dev->kobj_iolink); 584 kobject_put(dev->kobj_iolink); 585 dev->kobj_iolink = NULL; 586 } 587 588 if (dev->kobj_p2plink) { 589 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 590 if (p2plink->kobj) { 591 kfd_remove_sysfs_file(p2plink->kobj, 592 &p2plink->attr); 593 p2plink->kobj = NULL; 594 } 595 kobject_del(dev->kobj_p2plink); 596 kobject_put(dev->kobj_p2plink); 597 dev->kobj_p2plink = NULL; 598 } 599 600 if (dev->kobj_cache) { 601 list_for_each_entry(cache, &dev->cache_props, list) 602 if (cache->kobj) { 603 kfd_remove_sysfs_file(cache->kobj, 604 &cache->attr); 605 cache->kobj = NULL; 606 } 607 kobject_del(dev->kobj_cache); 608 kobject_put(dev->kobj_cache); 609 dev->kobj_cache = NULL; 610 } 611 612 if (dev->kobj_mem) { 613 list_for_each_entry(mem, &dev->mem_props, list) 614 if (mem->kobj) { 615 kfd_remove_sysfs_file(mem->kobj, &mem->attr); 616 mem->kobj = NULL; 617 } 618 kobject_del(dev->kobj_mem); 619 kobject_put(dev->kobj_mem); 620 dev->kobj_mem = NULL; 621 } 622 623 if (dev->kobj_perf) { 624 list_for_each_entry(perf, &dev->perf_props, list) { 625 kfree(perf->attr_group); 626 perf->attr_group = NULL; 627 } 628 kobject_del(dev->kobj_perf); 629 kobject_put(dev->kobj_perf); 630 dev->kobj_perf = NULL; 631 } 632 633 if (dev->kobj_node) { 634 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); 635 sysfs_remove_file(dev->kobj_node, &dev->attr_name); 636 sysfs_remove_file(dev->kobj_node, &dev->attr_props); 637 kobject_del(dev->kobj_node); 638 kobject_put(dev->kobj_node); 639 dev->kobj_node = NULL; 640 } 641 } 642 643 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, 644 uint32_t id) 645 { 646 struct kfd_iolink_properties *p2plink; 647 struct kfd_iolink_properties *iolink; 648 struct kfd_cache_properties *cache; 649 struct kfd_mem_properties *mem; 650 struct kfd_perf_properties *perf; 651 int ret; 652 uint32_t i, num_attrs; 653 struct attribute **attrs; 654 655 if (WARN_ON(dev->kobj_node)) 656 return -EEXIST; 657 658 /* 659 * Creating the sysfs folders 660 */ 661 dev->kobj_node = kfd_alloc_struct(dev->kobj_node); 662 if (!dev->kobj_node) 663 return -ENOMEM; 664 665 ret = kobject_init_and_add(dev->kobj_node, &node_type, 666 sys_props.kobj_nodes, "%d", id); 667 if (ret < 0) { 668 kobject_put(dev->kobj_node); 669 return ret; 670 } 671 672 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); 673 if (!dev->kobj_mem) 674 return -ENOMEM; 675 676 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); 677 if (!dev->kobj_cache) 678 return -ENOMEM; 679 680 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); 681 if (!dev->kobj_iolink) 682 return -ENOMEM; 683 684 dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node); 685 if (!dev->kobj_p2plink) 686 return -ENOMEM; 687 688 dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node); 689 if (!dev->kobj_perf) 690 return -ENOMEM; 691 692 /* 693 * Creating sysfs files for node properties 694 */ 695 dev->attr_gpuid.name = "gpu_id"; 696 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; 697 sysfs_attr_init(&dev->attr_gpuid); 698 dev->attr_name.name = "name"; 699 dev->attr_name.mode = KFD_SYSFS_FILE_MODE; 700 sysfs_attr_init(&dev->attr_name); 701 dev->attr_props.name = "properties"; 702 dev->attr_props.mode = KFD_SYSFS_FILE_MODE; 703 sysfs_attr_init(&dev->attr_props); 704 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); 705 if (ret < 0) 706 return ret; 707 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); 708 if (ret < 0) 709 return ret; 710 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); 711 if (ret < 0) 712 return ret; 713 714 i = 0; 715 list_for_each_entry(mem, &dev->mem_props, list) { 716 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 717 if (!mem->kobj) 718 return -ENOMEM; 719 ret = kobject_init_and_add(mem->kobj, &mem_type, 720 dev->kobj_mem, "%d", i); 721 if (ret < 0) { 722 kobject_put(mem->kobj); 723 return ret; 724 } 725 726 mem->attr.name = "properties"; 727 mem->attr.mode = KFD_SYSFS_FILE_MODE; 728 sysfs_attr_init(&mem->attr); 729 ret = sysfs_create_file(mem->kobj, &mem->attr); 730 if (ret < 0) 731 return ret; 732 i++; 733 } 734 735 i = 0; 736 list_for_each_entry(cache, &dev->cache_props, list) { 737 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 738 if (!cache->kobj) 739 return -ENOMEM; 740 ret = kobject_init_and_add(cache->kobj, &cache_type, 741 dev->kobj_cache, "%d", i); 742 if (ret < 0) { 743 kobject_put(cache->kobj); 744 return ret; 745 } 746 747 cache->attr.name = "properties"; 748 cache->attr.mode = KFD_SYSFS_FILE_MODE; 749 sysfs_attr_init(&cache->attr); 750 ret = sysfs_create_file(cache->kobj, &cache->attr); 751 if (ret < 0) 752 return ret; 753 i++; 754 } 755 756 i = 0; 757 list_for_each_entry(iolink, &dev->io_link_props, list) { 758 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 759 if (!iolink->kobj) 760 return -ENOMEM; 761 ret = kobject_init_and_add(iolink->kobj, &iolink_type, 762 dev->kobj_iolink, "%d", i); 763 if (ret < 0) { 764 kobject_put(iolink->kobj); 765 return ret; 766 } 767 768 iolink->attr.name = "properties"; 769 iolink->attr.mode = KFD_SYSFS_FILE_MODE; 770 sysfs_attr_init(&iolink->attr); 771 ret = sysfs_create_file(iolink->kobj, &iolink->attr); 772 if (ret < 0) 773 return ret; 774 i++; 775 } 776 777 i = 0; 778 list_for_each_entry(p2plink, &dev->p2p_link_props, list) { 779 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 780 if (!p2plink->kobj) 781 return -ENOMEM; 782 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 783 dev->kobj_p2plink, "%d", i); 784 if (ret < 0) { 785 kobject_put(p2plink->kobj); 786 return ret; 787 } 788 789 p2plink->attr.name = "properties"; 790 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 791 sysfs_attr_init(&p2plink->attr); 792 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 793 if (ret < 0) 794 return ret; 795 i++; 796 } 797 798 /* All hardware blocks have the same number of attributes. */ 799 num_attrs = ARRAY_SIZE(perf_attr_iommu); 800 list_for_each_entry(perf, &dev->perf_props, list) { 801 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr) 802 * num_attrs + sizeof(struct attribute_group), 803 GFP_KERNEL); 804 if (!perf->attr_group) 805 return -ENOMEM; 806 807 attrs = (struct attribute **)(perf->attr_group + 1); 808 if (!strcmp(perf->block_name, "iommu")) { 809 /* Information of IOMMU's num_counters and counter_ids is shown 810 * under /sys/bus/event_source/devices/amd_iommu. We don't 811 * duplicate here. 812 */ 813 perf_attr_iommu[0].data = perf->max_concurrent; 814 for (i = 0; i < num_attrs; i++) 815 attrs[i] = &perf_attr_iommu[i].attr.attr; 816 } 817 perf->attr_group->name = perf->block_name; 818 perf->attr_group->attrs = attrs; 819 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group); 820 if (ret < 0) 821 return ret; 822 } 823 824 return 0; 825 } 826 827 /* Called with write topology lock acquired */ 828 static int kfd_build_sysfs_node_tree(void) 829 { 830 struct kfd_topology_device *dev; 831 int ret; 832 uint32_t i = 0; 833 834 list_for_each_entry(dev, &topology_device_list, list) { 835 ret = kfd_build_sysfs_node_entry(dev, i); 836 if (ret < 0) 837 return ret; 838 i++; 839 } 840 841 return 0; 842 } 843 844 /* Called with write topology lock acquired */ 845 static void kfd_remove_sysfs_node_tree(void) 846 { 847 struct kfd_topology_device *dev; 848 849 list_for_each_entry(dev, &topology_device_list, list) 850 kfd_remove_sysfs_node_entry(dev); 851 } 852 853 static int kfd_topology_update_sysfs(void) 854 { 855 int ret; 856 857 if (!sys_props.kobj_topology) { 858 sys_props.kobj_topology = 859 kfd_alloc_struct(sys_props.kobj_topology); 860 if (!sys_props.kobj_topology) 861 return -ENOMEM; 862 863 ret = kobject_init_and_add(sys_props.kobj_topology, 864 &sysprops_type, &kfd_device->kobj, 865 "topology"); 866 if (ret < 0) { 867 kobject_put(sys_props.kobj_topology); 868 return ret; 869 } 870 871 sys_props.kobj_nodes = kobject_create_and_add("nodes", 872 sys_props.kobj_topology); 873 if (!sys_props.kobj_nodes) 874 return -ENOMEM; 875 876 sys_props.attr_genid.name = "generation_id"; 877 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; 878 sysfs_attr_init(&sys_props.attr_genid); 879 ret = sysfs_create_file(sys_props.kobj_topology, 880 &sys_props.attr_genid); 881 if (ret < 0) 882 return ret; 883 884 sys_props.attr_props.name = "system_properties"; 885 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; 886 sysfs_attr_init(&sys_props.attr_props); 887 ret = sysfs_create_file(sys_props.kobj_topology, 888 &sys_props.attr_props); 889 if (ret < 0) 890 return ret; 891 } 892 893 kfd_remove_sysfs_node_tree(); 894 895 return kfd_build_sysfs_node_tree(); 896 } 897 898 static void kfd_topology_release_sysfs(void) 899 { 900 kfd_remove_sysfs_node_tree(); 901 if (sys_props.kobj_topology) { 902 sysfs_remove_file(sys_props.kobj_topology, 903 &sys_props.attr_genid); 904 sysfs_remove_file(sys_props.kobj_topology, 905 &sys_props.attr_props); 906 if (sys_props.kobj_nodes) { 907 kobject_del(sys_props.kobj_nodes); 908 kobject_put(sys_props.kobj_nodes); 909 sys_props.kobj_nodes = NULL; 910 } 911 kobject_del(sys_props.kobj_topology); 912 kobject_put(sys_props.kobj_topology); 913 sys_props.kobj_topology = NULL; 914 } 915 } 916 917 /* Called with write topology_lock acquired */ 918 static void kfd_topology_update_device_list(struct list_head *temp_list, 919 struct list_head *master_list) 920 { 921 while (!list_empty(temp_list)) { 922 list_move_tail(temp_list->next, master_list); 923 sys_props.num_devices++; 924 } 925 } 926 927 static void kfd_debug_print_topology(void) 928 { 929 struct kfd_topology_device *dev; 930 931 down_read(&topology_lock); 932 933 dev = list_last_entry(&topology_device_list, 934 struct kfd_topology_device, list); 935 if (dev) { 936 if (dev->node_props.cpu_cores_count && 937 dev->node_props.simd_count) { 938 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n", 939 dev->node_props.device_id, 940 dev->node_props.vendor_id); 941 } else if (dev->node_props.cpu_cores_count) 942 pr_info("Topology: Add CPU node\n"); 943 else if (dev->node_props.simd_count) 944 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n", 945 dev->node_props.device_id, 946 dev->node_props.vendor_id); 947 } 948 up_read(&topology_lock); 949 } 950 951 /* Helper function for intializing platform_xx members of 952 * kfd_system_properties. Uses OEM info from the last CPU/APU node. 953 */ 954 static void kfd_update_system_properties(void) 955 { 956 struct kfd_topology_device *dev; 957 958 down_read(&topology_lock); 959 dev = list_last_entry(&topology_device_list, 960 struct kfd_topology_device, list); 961 if (dev) { 962 sys_props.platform_id = 963 (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK; 964 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id); 965 sys_props.platform_rev = dev->oem_revision; 966 } 967 up_read(&topology_lock); 968 } 969 970 static void find_system_memory(const struct dmi_header *dm, 971 void *private) 972 { 973 struct kfd_mem_properties *mem; 974 u16 mem_width, mem_clock; 975 struct kfd_topology_device *kdev = 976 (struct kfd_topology_device *)private; 977 const u8 *dmi_data = (const u8 *)(dm + 1); 978 979 if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) { 980 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6)); 981 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11)); 982 list_for_each_entry(mem, &kdev->mem_props, list) { 983 if (mem_width != 0xFFFF && mem_width != 0) 984 mem->width = mem_width; 985 if (mem_clock != 0) 986 mem->mem_clk_max = mem_clock; 987 } 988 } 989 } 990 991 /* 992 * Performance counters information is not part of CRAT but we would like to 993 * put them in the sysfs under topology directory for Thunk to get the data. 994 * This function is called before updating the sysfs. 995 */ 996 static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev) 997 { 998 /* These are the only counters supported so far */ 999 return kfd_iommu_add_perf_counters(kdev); 1000 } 1001 1002 /* kfd_add_non_crat_information - Add information that is not currently 1003 * defined in CRAT but is necessary for KFD topology 1004 * @dev - topology device to which addition info is added 1005 */ 1006 static void kfd_add_non_crat_information(struct kfd_topology_device *kdev) 1007 { 1008 /* Check if CPU only node. */ 1009 if (!kdev->gpu) { 1010 /* Add system memory information */ 1011 dmi_walk(find_system_memory, kdev); 1012 } 1013 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */ 1014 } 1015 1016 /* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices. 1017 * Ignore CRAT for all other devices. AMD APU is identified if both CPU 1018 * and GPU cores are present. 1019 * @device_list - topology device list created by parsing ACPI CRAT table. 1020 * @return - TRUE if invalid, FALSE is valid. 1021 */ 1022 static bool kfd_is_acpi_crat_invalid(struct list_head *device_list) 1023 { 1024 struct kfd_topology_device *dev; 1025 1026 list_for_each_entry(dev, device_list, list) { 1027 if (dev->node_props.cpu_cores_count && 1028 dev->node_props.simd_count) 1029 return false; 1030 } 1031 pr_info("Ignoring ACPI CRAT on non-APU system\n"); 1032 return true; 1033 } 1034 1035 int kfd_topology_init(void) 1036 { 1037 void *crat_image = NULL; 1038 size_t image_size = 0; 1039 int ret; 1040 struct list_head temp_topology_device_list; 1041 int cpu_only_node = 0; 1042 struct kfd_topology_device *kdev; 1043 int proximity_domain; 1044 1045 /* topology_device_list - Master list of all topology devices 1046 * temp_topology_device_list - temporary list created while parsing CRAT 1047 * or VCRAT. Once parsing is complete the contents of list is moved to 1048 * topology_device_list 1049 */ 1050 1051 /* Initialize the head for the both the lists */ 1052 INIT_LIST_HEAD(&topology_device_list); 1053 INIT_LIST_HEAD(&temp_topology_device_list); 1054 init_rwsem(&topology_lock); 1055 1056 memset(&sys_props, 0, sizeof(sys_props)); 1057 1058 /* Proximity domains in ACPI CRAT tables start counting at 1059 * 0. The same should be true for virtual CRAT tables created 1060 * at this stage. GPUs added later in kfd_topology_add_device 1061 * use a counter. 1062 */ 1063 proximity_domain = 0; 1064 1065 /* 1066 * Get the CRAT image from the ACPI. If ACPI doesn't have one 1067 * or if ACPI CRAT is invalid create a virtual CRAT. 1068 * NOTE: The current implementation expects all AMD APUs to have 1069 * CRAT. If no CRAT is available, it is assumed to be a CPU 1070 */ 1071 ret = kfd_create_crat_image_acpi(&crat_image, &image_size); 1072 if (!ret) { 1073 ret = kfd_parse_crat_table(crat_image, 1074 &temp_topology_device_list, 1075 proximity_domain); 1076 if (ret || 1077 kfd_is_acpi_crat_invalid(&temp_topology_device_list)) { 1078 kfd_release_topology_device_list( 1079 &temp_topology_device_list); 1080 kfd_destroy_crat_image(crat_image); 1081 crat_image = NULL; 1082 } 1083 } 1084 1085 if (!crat_image) { 1086 ret = kfd_create_crat_image_virtual(&crat_image, &image_size, 1087 COMPUTE_UNIT_CPU, NULL, 1088 proximity_domain); 1089 cpu_only_node = 1; 1090 if (ret) { 1091 pr_err("Error creating VCRAT table for CPU\n"); 1092 return ret; 1093 } 1094 1095 ret = kfd_parse_crat_table(crat_image, 1096 &temp_topology_device_list, 1097 proximity_domain); 1098 if (ret) { 1099 pr_err("Error parsing VCRAT table for CPU\n"); 1100 goto err; 1101 } 1102 } 1103 1104 kdev = list_first_entry(&temp_topology_device_list, 1105 struct kfd_topology_device, list); 1106 kfd_add_perf_to_topology(kdev); 1107 1108 down_write(&topology_lock); 1109 kfd_topology_update_device_list(&temp_topology_device_list, 1110 &topology_device_list); 1111 topology_crat_proximity_domain = sys_props.num_devices-1; 1112 ret = kfd_topology_update_sysfs(); 1113 up_write(&topology_lock); 1114 1115 if (!ret) { 1116 sys_props.generation_count++; 1117 kfd_update_system_properties(); 1118 kfd_debug_print_topology(); 1119 } else 1120 pr_err("Failed to update topology in sysfs ret=%d\n", ret); 1121 1122 /* For nodes with GPU, this information gets added 1123 * when GPU is detected (kfd_topology_add_device). 1124 */ 1125 if (cpu_only_node) { 1126 /* Add additional information to CPU only node created above */ 1127 down_write(&topology_lock); 1128 kdev = list_first_entry(&topology_device_list, 1129 struct kfd_topology_device, list); 1130 up_write(&topology_lock); 1131 kfd_add_non_crat_information(kdev); 1132 } 1133 1134 err: 1135 kfd_destroy_crat_image(crat_image); 1136 return ret; 1137 } 1138 1139 void kfd_topology_shutdown(void) 1140 { 1141 down_write(&topology_lock); 1142 kfd_topology_release_sysfs(); 1143 kfd_release_live_view(); 1144 up_write(&topology_lock); 1145 } 1146 1147 static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu) 1148 { 1149 uint32_t hashout; 1150 uint32_t buf[8]; 1151 uint64_t local_mem_size; 1152 int i; 1153 1154 if (!gpu) 1155 return 0; 1156 1157 local_mem_size = gpu->local_mem_info.local_mem_size_private + 1158 gpu->local_mem_info.local_mem_size_public; 1159 buf[0] = gpu->adev->pdev->devfn; 1160 buf[1] = gpu->adev->pdev->subsystem_vendor | 1161 (gpu->adev->pdev->subsystem_device << 16); 1162 buf[2] = pci_domain_nr(gpu->adev->pdev->bus); 1163 buf[3] = gpu->adev->pdev->device; 1164 buf[4] = gpu->adev->pdev->bus->number; 1165 buf[5] = lower_32_bits(local_mem_size); 1166 buf[6] = upper_32_bits(local_mem_size); 1167 buf[7] = (ffs(gpu->xcc_mask) - 1) | (NUM_XCC(gpu->xcc_mask) << 16); 1168 1169 for (i = 0, hashout = 0; i < 8; i++) 1170 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); 1171 1172 return hashout; 1173 } 1174 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If 1175 * the GPU device is not already present in the topology device 1176 * list then return NULL. This means a new topology device has to 1177 * be created for this GPU. 1178 */ 1179 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu) 1180 { 1181 struct kfd_topology_device *dev; 1182 struct kfd_topology_device *out_dev = NULL; 1183 struct kfd_mem_properties *mem; 1184 struct kfd_cache_properties *cache; 1185 struct kfd_iolink_properties *iolink; 1186 struct kfd_iolink_properties *p2plink; 1187 1188 list_for_each_entry(dev, &topology_device_list, list) { 1189 /* Discrete GPUs need their own topology device list 1190 * entries. Don't assign them to CPU/APU nodes. 1191 */ 1192 if (!gpu->kfd->use_iommu_v2 && 1193 dev->node_props.cpu_cores_count) 1194 continue; 1195 1196 if (!dev->gpu && (dev->node_props.simd_count > 0)) { 1197 dev->gpu = gpu; 1198 out_dev = dev; 1199 1200 list_for_each_entry(mem, &dev->mem_props, list) 1201 mem->gpu = dev->gpu; 1202 list_for_each_entry(cache, &dev->cache_props, list) 1203 cache->gpu = dev->gpu; 1204 list_for_each_entry(iolink, &dev->io_link_props, list) 1205 iolink->gpu = dev->gpu; 1206 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 1207 p2plink->gpu = dev->gpu; 1208 break; 1209 } 1210 } 1211 return out_dev; 1212 } 1213 1214 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) 1215 { 1216 /* 1217 * TODO: Generate an event for thunk about the arrival/removal 1218 * of the GPU 1219 */ 1220 } 1221 1222 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info, 1223 * patch this after CRAT parsing. 1224 */ 1225 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) 1226 { 1227 struct kfd_mem_properties *mem; 1228 struct kfd_local_mem_info local_mem_info; 1229 1230 if (!dev) 1231 return; 1232 1233 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with 1234 * single bank of VRAM local memory. 1235 * for dGPUs - VCRAT reports only one bank of Local Memory 1236 * for APUs - If CRAT from ACPI reports more than one bank, then 1237 * all the banks will report the same mem_clk_max information 1238 */ 1239 amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info, 1240 dev->gpu->xcp); 1241 1242 list_for_each_entry(mem, &dev->mem_props, list) 1243 mem->mem_clk_max = local_mem_info.mem_clk_max; 1244 } 1245 1246 static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev, 1247 struct kfd_topology_device *target_gpu_dev, 1248 struct kfd_iolink_properties *link) 1249 { 1250 /* xgmi always supports atomics between links. */ 1251 if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI) 1252 return; 1253 1254 /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */ 1255 if (target_gpu_dev) { 1256 uint32_t cap; 1257 1258 pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev, 1259 PCI_EXP_DEVCAP2, &cap); 1260 1261 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 1262 PCI_EXP_DEVCAP2_ATOMIC_COMP64))) 1263 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1264 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1265 /* set gpu (dev) flags. */ 1266 } else { 1267 if (!dev->gpu->kfd->pci_atomic_requested || 1268 dev->gpu->adev->asic_type == CHIP_HAWAII) 1269 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1270 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1271 } 1272 } 1273 1274 static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev, 1275 struct kfd_iolink_properties *outbound_link, 1276 struct kfd_iolink_properties *inbound_link) 1277 { 1278 /* CPU -> GPU with PCIe */ 1279 if (!to_dev->gpu && 1280 inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) 1281 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1282 1283 if (to_dev->gpu) { 1284 /* GPU <-> GPU with PCIe and 1285 * Vega20 with XGMI 1286 */ 1287 if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS || 1288 (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI && 1289 KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) { 1290 outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1291 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1292 } 1293 } 1294 } 1295 1296 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev) 1297 { 1298 struct kfd_iolink_properties *link, *inbound_link; 1299 struct kfd_topology_device *peer_dev; 1300 1301 if (!dev || !dev->gpu) 1302 return; 1303 1304 /* GPU only creates direct links so apply flags setting to all */ 1305 list_for_each_entry(link, &dev->io_link_props, list) { 1306 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1307 kfd_set_iolink_no_atomics(dev, NULL, link); 1308 peer_dev = kfd_topology_device_by_proximity_domain( 1309 link->node_to); 1310 1311 if (!peer_dev) 1312 continue; 1313 1314 /* Include the CPU peer in GPU hive if connected over xGMI. */ 1315 if (!peer_dev->gpu && 1316 link->iolink_type == CRAT_IOLINK_TYPE_XGMI) { 1317 /* 1318 * If the GPU is not part of a GPU hive, use its pci 1319 * device location as the hive ID to bind with the CPU. 1320 */ 1321 if (!dev->node_props.hive_id) 1322 dev->node_props.hive_id = pci_dev_id(dev->gpu->adev->pdev); 1323 peer_dev->node_props.hive_id = dev->node_props.hive_id; 1324 } 1325 1326 list_for_each_entry(inbound_link, &peer_dev->io_link_props, 1327 list) { 1328 if (inbound_link->node_to != link->node_from) 1329 continue; 1330 1331 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1332 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1333 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1334 } 1335 } 1336 1337 /* Create indirect links so apply flags setting to all */ 1338 list_for_each_entry(link, &dev->p2p_link_props, list) { 1339 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1340 kfd_set_iolink_no_atomics(dev, NULL, link); 1341 peer_dev = kfd_topology_device_by_proximity_domain( 1342 link->node_to); 1343 1344 if (!peer_dev) 1345 continue; 1346 1347 list_for_each_entry(inbound_link, &peer_dev->p2p_link_props, 1348 list) { 1349 if (inbound_link->node_to != link->node_from) 1350 continue; 1351 1352 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1353 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1354 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1355 } 1356 } 1357 } 1358 1359 static int kfd_build_p2p_node_entry(struct kfd_topology_device *dev, 1360 struct kfd_iolink_properties *p2plink) 1361 { 1362 int ret; 1363 1364 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 1365 if (!p2plink->kobj) 1366 return -ENOMEM; 1367 1368 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 1369 dev->kobj_p2plink, "%d", dev->node_props.p2p_links_count - 1); 1370 if (ret < 0) { 1371 kobject_put(p2plink->kobj); 1372 return ret; 1373 } 1374 1375 p2plink->attr.name = "properties"; 1376 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 1377 sysfs_attr_init(&p2plink->attr); 1378 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 1379 if (ret < 0) 1380 return ret; 1381 1382 return 0; 1383 } 1384 1385 static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int gpu_node) 1386 { 1387 struct kfd_iolink_properties *gpu_link, *tmp_link, *cpu_link; 1388 struct kfd_iolink_properties *props = NULL, *props2 = NULL; 1389 struct kfd_topology_device *cpu_dev; 1390 int ret = 0; 1391 int i, num_cpu; 1392 1393 num_cpu = 0; 1394 list_for_each_entry(cpu_dev, &topology_device_list, list) { 1395 if (cpu_dev->gpu) 1396 break; 1397 num_cpu++; 1398 } 1399 1400 gpu_link = list_first_entry(&kdev->io_link_props, 1401 struct kfd_iolink_properties, list); 1402 if (!gpu_link) 1403 return -ENOMEM; 1404 1405 for (i = 0; i < num_cpu; i++) { 1406 /* CPU <--> GPU */ 1407 if (gpu_link->node_to == i) 1408 continue; 1409 1410 /* find CPU <--> CPU links */ 1411 cpu_link = NULL; 1412 cpu_dev = kfd_topology_device_by_proximity_domain(i); 1413 if (cpu_dev) { 1414 list_for_each_entry(tmp_link, 1415 &cpu_dev->io_link_props, list) { 1416 if (tmp_link->node_to == gpu_link->node_to) { 1417 cpu_link = tmp_link; 1418 break; 1419 } 1420 } 1421 } 1422 1423 if (!cpu_link) 1424 return -ENOMEM; 1425 1426 /* CPU <--> CPU <--> GPU, GPU node*/ 1427 props = kfd_alloc_struct(props); 1428 if (!props) 1429 return -ENOMEM; 1430 1431 memcpy(props, gpu_link, sizeof(struct kfd_iolink_properties)); 1432 props->weight = gpu_link->weight + cpu_link->weight; 1433 props->min_latency = gpu_link->min_latency + cpu_link->min_latency; 1434 props->max_latency = gpu_link->max_latency + cpu_link->max_latency; 1435 props->min_bandwidth = min(gpu_link->min_bandwidth, cpu_link->min_bandwidth); 1436 props->max_bandwidth = min(gpu_link->max_bandwidth, cpu_link->max_bandwidth); 1437 1438 props->node_from = gpu_node; 1439 props->node_to = i; 1440 kdev->node_props.p2p_links_count++; 1441 list_add_tail(&props->list, &kdev->p2p_link_props); 1442 ret = kfd_build_p2p_node_entry(kdev, props); 1443 if (ret < 0) 1444 return ret; 1445 1446 /* for small Bar, no CPU --> GPU in-direct links */ 1447 if (kfd_dev_is_large_bar(kdev->gpu)) { 1448 /* CPU <--> CPU <--> GPU, CPU node*/ 1449 props2 = kfd_alloc_struct(props2); 1450 if (!props2) 1451 return -ENOMEM; 1452 1453 memcpy(props2, props, sizeof(struct kfd_iolink_properties)); 1454 props2->node_from = i; 1455 props2->node_to = gpu_node; 1456 props2->kobj = NULL; 1457 cpu_dev->node_props.p2p_links_count++; 1458 list_add_tail(&props2->list, &cpu_dev->p2p_link_props); 1459 ret = kfd_build_p2p_node_entry(cpu_dev, props2); 1460 if (ret < 0) 1461 return ret; 1462 } 1463 } 1464 return ret; 1465 } 1466 1467 #if defined(CONFIG_HSA_AMD_P2P) 1468 static int kfd_add_peer_prop(struct kfd_topology_device *kdev, 1469 struct kfd_topology_device *peer, int from, int to) 1470 { 1471 struct kfd_iolink_properties *props = NULL; 1472 struct kfd_iolink_properties *iolink1, *iolink2, *iolink3; 1473 struct kfd_topology_device *cpu_dev; 1474 int ret = 0; 1475 1476 if (!amdgpu_device_is_peer_accessible( 1477 kdev->gpu->adev, 1478 peer->gpu->adev)) 1479 return ret; 1480 1481 iolink1 = list_first_entry(&kdev->io_link_props, 1482 struct kfd_iolink_properties, list); 1483 if (!iolink1) 1484 return -ENOMEM; 1485 1486 iolink2 = list_first_entry(&peer->io_link_props, 1487 struct kfd_iolink_properties, list); 1488 if (!iolink2) 1489 return -ENOMEM; 1490 1491 props = kfd_alloc_struct(props); 1492 if (!props) 1493 return -ENOMEM; 1494 1495 memcpy(props, iolink1, sizeof(struct kfd_iolink_properties)); 1496 1497 props->weight = iolink1->weight + iolink2->weight; 1498 props->min_latency = iolink1->min_latency + iolink2->min_latency; 1499 props->max_latency = iolink1->max_latency + iolink2->max_latency; 1500 props->min_bandwidth = min(iolink1->min_bandwidth, iolink2->min_bandwidth); 1501 props->max_bandwidth = min(iolink2->max_bandwidth, iolink2->max_bandwidth); 1502 1503 if (iolink1->node_to != iolink2->node_to) { 1504 /* CPU->CPU link*/ 1505 cpu_dev = kfd_topology_device_by_proximity_domain(iolink1->node_to); 1506 if (cpu_dev) { 1507 list_for_each_entry(iolink3, &cpu_dev->io_link_props, list) 1508 if (iolink3->node_to == iolink2->node_to) 1509 break; 1510 1511 props->weight += iolink3->weight; 1512 props->min_latency += iolink3->min_latency; 1513 props->max_latency += iolink3->max_latency; 1514 props->min_bandwidth = min(props->min_bandwidth, 1515 iolink3->min_bandwidth); 1516 props->max_bandwidth = min(props->max_bandwidth, 1517 iolink3->max_bandwidth); 1518 } else { 1519 WARN(1, "CPU node not found"); 1520 } 1521 } 1522 1523 props->node_from = from; 1524 props->node_to = to; 1525 peer->node_props.p2p_links_count++; 1526 list_add_tail(&props->list, &peer->p2p_link_props); 1527 ret = kfd_build_p2p_node_entry(peer, props); 1528 1529 return ret; 1530 } 1531 #endif 1532 1533 static int kfd_dev_create_p2p_links(void) 1534 { 1535 struct kfd_topology_device *dev; 1536 struct kfd_topology_device *new_dev; 1537 #if defined(CONFIG_HSA_AMD_P2P) 1538 uint32_t i; 1539 #endif 1540 uint32_t k; 1541 int ret = 0; 1542 1543 k = 0; 1544 list_for_each_entry(dev, &topology_device_list, list) 1545 k++; 1546 if (k < 2) 1547 return 0; 1548 1549 new_dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list); 1550 if (WARN_ON(!new_dev->gpu)) 1551 return 0; 1552 1553 k--; 1554 1555 /* create in-direct links */ 1556 ret = kfd_create_indirect_link_prop(new_dev, k); 1557 if (ret < 0) 1558 goto out; 1559 1560 /* create p2p links */ 1561 #if defined(CONFIG_HSA_AMD_P2P) 1562 i = 0; 1563 list_for_each_entry(dev, &topology_device_list, list) { 1564 if (dev == new_dev) 1565 break; 1566 if (!dev->gpu || !dev->gpu->adev || 1567 (dev->gpu->kfd->hive_id && 1568 dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id)) 1569 goto next; 1570 1571 /* check if node(s) is/are peer accessible in one direction or bi-direction */ 1572 ret = kfd_add_peer_prop(new_dev, dev, i, k); 1573 if (ret < 0) 1574 goto out; 1575 1576 ret = kfd_add_peer_prop(dev, new_dev, k, i); 1577 if (ret < 0) 1578 goto out; 1579 next: 1580 i++; 1581 } 1582 #endif 1583 1584 out: 1585 return ret; 1586 } 1587 1588 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1589 static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext, 1590 struct kfd_gpu_cache_info *pcache_info, 1591 struct kfd_cu_info *cu_info, 1592 int cu_bitmask, 1593 int cache_type, unsigned int cu_processor_id, 1594 int cu_block) 1595 { 1596 unsigned int cu_sibling_map_mask; 1597 int first_active_cu; 1598 struct kfd_cache_properties *pcache = NULL; 1599 1600 cu_sibling_map_mask = cu_bitmask; 1601 cu_sibling_map_mask >>= cu_block; 1602 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1603 first_active_cu = ffs(cu_sibling_map_mask); 1604 1605 /* CU could be inactive. In case of shared cache find the first active 1606 * CU. and incase of non-shared cache check if the CU is inactive. If 1607 * inactive active skip it 1608 */ 1609 if (first_active_cu) { 1610 pcache = kfd_alloc_struct(pcache); 1611 if (!pcache) 1612 return -ENOMEM; 1613 1614 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1615 pcache->processor_id_low = cu_processor_id + (first_active_cu - 1); 1616 pcache->cache_level = pcache_info[cache_type].cache_level; 1617 pcache->cache_size = pcache_info[cache_type].cache_size; 1618 1619 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1620 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1621 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1622 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1623 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1624 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1625 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1626 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1627 1628 /* Sibling map is w.r.t processor_id_low, so shift out 1629 * inactive CU 1630 */ 1631 cu_sibling_map_mask = 1632 cu_sibling_map_mask >> (first_active_cu - 1); 1633 1634 pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1635 pcache->sibling_map[1] = 1636 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1637 pcache->sibling_map[2] = 1638 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1639 pcache->sibling_map[3] = 1640 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1641 1642 pcache->sibling_map_size = 4; 1643 *props_ext = pcache; 1644 1645 return 0; 1646 } 1647 return 1; 1648 } 1649 1650 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1651 static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, 1652 struct kfd_gpu_cache_info *pcache_info, 1653 struct kfd_cu_info *cu_info, 1654 int cache_type, unsigned int cu_processor_id) 1655 { 1656 unsigned int cu_sibling_map_mask; 1657 int first_active_cu; 1658 int i, j, k; 1659 struct kfd_cache_properties *pcache = NULL; 1660 1661 cu_sibling_map_mask = cu_info->cu_bitmap[0][0]; 1662 cu_sibling_map_mask &= 1663 ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1664 first_active_cu = ffs(cu_sibling_map_mask); 1665 1666 /* CU could be inactive. In case of shared cache find the first active 1667 * CU. and incase of non-shared cache check if the CU is inactive. If 1668 * inactive active skip it 1669 */ 1670 if (first_active_cu) { 1671 pcache = kfd_alloc_struct(pcache); 1672 if (!pcache) 1673 return -ENOMEM; 1674 1675 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1676 pcache->processor_id_low = cu_processor_id 1677 + (first_active_cu - 1); 1678 pcache->cache_level = pcache_info[cache_type].cache_level; 1679 pcache->cache_size = pcache_info[cache_type].cache_size; 1680 1681 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1682 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1683 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1684 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1685 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1686 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1687 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1688 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1689 1690 /* Sibling map is w.r.t processor_id_low, so shift out 1691 * inactive CU 1692 */ 1693 cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1); 1694 k = 0; 1695 1696 for (i = 0; i < cu_info->num_shader_engines; i++) { 1697 for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) { 1698 pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1699 pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1700 pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1701 pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1702 k += 4; 1703 1704 cu_sibling_map_mask = cu_info->cu_bitmap[i % 4][j + i / 4]; 1705 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1706 } 1707 } 1708 pcache->sibling_map_size = k; 1709 *props_ext = pcache; 1710 return 0; 1711 } 1712 return 1; 1713 } 1714 1715 #define KFD_MAX_CACHE_TYPES 6 1716 1717 /* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info 1718 * tables 1719 */ 1720 static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev) 1721 { 1722 struct kfd_gpu_cache_info *pcache_info = NULL; 1723 int i, j, k; 1724 int ct = 0; 1725 unsigned int cu_processor_id; 1726 int ret; 1727 unsigned int num_cu_shared; 1728 struct kfd_cu_info cu_info; 1729 struct kfd_cu_info *pcu_info; 1730 int gpu_processor_id; 1731 struct kfd_cache_properties *props_ext; 1732 int num_of_entries = 0; 1733 int num_of_cache_types = 0; 1734 struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; 1735 1736 amdgpu_amdkfd_get_cu_info(kdev->adev, &cu_info); 1737 pcu_info = &cu_info; 1738 1739 gpu_processor_id = dev->node_props.simd_id_base; 1740 1741 pcache_info = cache_info; 1742 num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info); 1743 if (!num_of_cache_types) { 1744 pr_warn("no cache info found\n"); 1745 return; 1746 } 1747 1748 /* For each type of cache listed in the kfd_gpu_cache_info table, 1749 * go through all available Compute Units. 1750 * The [i,j,k] loop will 1751 * if kfd_gpu_cache_info.num_cu_shared = 1 1752 * will parse through all available CU 1753 * If (kfd_gpu_cache_info.num_cu_shared != 1) 1754 * then it will consider only one CU from 1755 * the shared unit 1756 */ 1757 for (ct = 0; ct < num_of_cache_types; ct++) { 1758 cu_processor_id = gpu_processor_id; 1759 if (pcache_info[ct].cache_level == 1) { 1760 for (i = 0; i < pcu_info->num_shader_engines; i++) { 1761 for (j = 0; j < pcu_info->num_shader_arrays_per_engine; j++) { 1762 for (k = 0; k < pcu_info->num_cu_per_sh; k += pcache_info[ct].num_cu_shared) { 1763 1764 ret = fill_in_l1_pcache(&props_ext, pcache_info, pcu_info, 1765 pcu_info->cu_bitmap[i % 4][j + i / 4], ct, 1766 cu_processor_id, k); 1767 1768 if (ret < 0) 1769 break; 1770 1771 if (!ret) { 1772 num_of_entries++; 1773 list_add_tail(&props_ext->list, &dev->cache_props); 1774 } 1775 1776 /* Move to next CU block */ 1777 num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= 1778 pcu_info->num_cu_per_sh) ? 1779 pcache_info[ct].num_cu_shared : 1780 (pcu_info->num_cu_per_sh - k); 1781 cu_processor_id += num_cu_shared; 1782 } 1783 } 1784 } 1785 } else { 1786 ret = fill_in_l2_l3_pcache(&props_ext, pcache_info, 1787 pcu_info, ct, cu_processor_id); 1788 1789 if (ret < 0) 1790 break; 1791 1792 if (!ret) { 1793 num_of_entries++; 1794 list_add_tail(&props_ext->list, &dev->cache_props); 1795 } 1796 } 1797 } 1798 dev->node_props.caches_count += num_of_entries; 1799 pr_debug("Added [%d] GPU cache entries\n", num_of_entries); 1800 } 1801 1802 static int kfd_topology_add_device_locked(struct kfd_node *gpu, uint32_t gpu_id, 1803 struct kfd_topology_device **dev) 1804 { 1805 int proximity_domain = ++topology_crat_proximity_domain; 1806 struct list_head temp_topology_device_list; 1807 void *crat_image = NULL; 1808 size_t image_size = 0; 1809 int res; 1810 1811 res = kfd_create_crat_image_virtual(&crat_image, &image_size, 1812 COMPUTE_UNIT_GPU, gpu, 1813 proximity_domain); 1814 if (res) { 1815 pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n", 1816 gpu_id); 1817 topology_crat_proximity_domain--; 1818 goto err; 1819 } 1820 1821 INIT_LIST_HEAD(&temp_topology_device_list); 1822 1823 res = kfd_parse_crat_table(crat_image, 1824 &temp_topology_device_list, 1825 proximity_domain); 1826 if (res) { 1827 pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n", 1828 gpu_id); 1829 topology_crat_proximity_domain--; 1830 goto err; 1831 } 1832 1833 kfd_topology_update_device_list(&temp_topology_device_list, 1834 &topology_device_list); 1835 1836 *dev = kfd_assign_gpu(gpu); 1837 if (WARN_ON(!*dev)) { 1838 res = -ENODEV; 1839 goto err; 1840 } 1841 1842 /* Fill the cache affinity information here for the GPUs 1843 * using VCRAT 1844 */ 1845 kfd_fill_cache_non_crat_info(*dev, gpu); 1846 1847 /* Update the SYSFS tree, since we added another topology 1848 * device 1849 */ 1850 res = kfd_topology_update_sysfs(); 1851 if (!res) 1852 sys_props.generation_count++; 1853 else 1854 pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n", 1855 gpu_id, res); 1856 1857 err: 1858 kfd_destroy_crat_image(crat_image); 1859 return res; 1860 } 1861 1862 static void kfd_topology_set_dbg_firmware_support(struct kfd_topology_device *dev) 1863 { 1864 bool firmware_supported = true; 1865 1866 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0) && 1867 KFD_GC_VERSION(dev->gpu) < IP_VERSION(12, 0, 0)) { 1868 uint32_t mes_api_rev = (dev->gpu->adev->mes.sched_version & 1869 AMDGPU_MES_API_VERSION_MASK) >> 1870 AMDGPU_MES_API_VERSION_SHIFT; 1871 uint32_t mes_rev = dev->gpu->adev->mes.sched_version & 1872 AMDGPU_MES_VERSION_MASK; 1873 1874 firmware_supported = (mes_api_rev >= 14) && (mes_rev >= 64); 1875 goto out; 1876 } 1877 1878 /* 1879 * Note: Any unlisted devices here are assumed to support exception handling. 1880 * Add additional checks here as needed. 1881 */ 1882 switch (KFD_GC_VERSION(dev->gpu)) { 1883 case IP_VERSION(9, 0, 1): 1884 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459 + 32768; 1885 break; 1886 case IP_VERSION(9, 1, 0): 1887 case IP_VERSION(9, 2, 1): 1888 case IP_VERSION(9, 2, 2): 1889 case IP_VERSION(9, 3, 0): 1890 case IP_VERSION(9, 4, 0): 1891 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459; 1892 break; 1893 case IP_VERSION(9, 4, 1): 1894 firmware_supported = dev->gpu->kfd->mec_fw_version >= 60; 1895 break; 1896 case IP_VERSION(9, 4, 2): 1897 firmware_supported = dev->gpu->kfd->mec_fw_version >= 51; 1898 break; 1899 case IP_VERSION(10, 1, 10): 1900 case IP_VERSION(10, 1, 2): 1901 case IP_VERSION(10, 1, 1): 1902 firmware_supported = dev->gpu->kfd->mec_fw_version >= 144; 1903 break; 1904 case IP_VERSION(10, 3, 0): 1905 case IP_VERSION(10, 3, 2): 1906 case IP_VERSION(10, 3, 1): 1907 case IP_VERSION(10, 3, 4): 1908 case IP_VERSION(10, 3, 5): 1909 firmware_supported = dev->gpu->kfd->mec_fw_version >= 89; 1910 break; 1911 case IP_VERSION(10, 1, 3): 1912 case IP_VERSION(10, 3, 3): 1913 firmware_supported = false; 1914 break; 1915 default: 1916 break; 1917 } 1918 1919 out: 1920 if (firmware_supported) 1921 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED; 1922 } 1923 1924 static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) 1925 { 1926 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 << 1927 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 1928 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 1929 1930 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_SUPPORT | 1931 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_TRAP_OVERRIDE_SUPPORTED | 1932 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_MODE_SUPPORTED; 1933 1934 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0)) { 1935 dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9 | 1936 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1937 1938 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 4, 2)) 1939 dev->node_props.debug_prop |= 1940 HSA_DBG_DISPATCH_INFO_ALWAYS_VALID; 1941 else 1942 dev->node_props.capability |= 1943 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1944 } else { 1945 dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | 1946 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1947 1948 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(11, 0, 0)) 1949 dev->node_props.debug_prop |= HSA_DBG_DISPATCH_INFO_ALWAYS_VALID; 1950 else 1951 dev->node_props.capability |= 1952 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1953 } 1954 1955 kfd_topology_set_dbg_firmware_support(dev); 1956 } 1957 1958 int kfd_topology_add_device(struct kfd_node *gpu) 1959 { 1960 uint32_t gpu_id; 1961 struct kfd_topology_device *dev; 1962 struct kfd_cu_info cu_info; 1963 int res = 0; 1964 int i; 1965 const char *asic_name = amdgpu_asic_name[gpu->adev->asic_type]; 1966 1967 gpu_id = kfd_generate_gpu_id(gpu); 1968 pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id); 1969 1970 /* Check to see if this gpu device exists in the topology_device_list. 1971 * If so, assign the gpu to that device, 1972 * else create a Virtual CRAT for this gpu device and then parse that 1973 * CRAT to create a new topology device. Once created assign the gpu to 1974 * that topology device 1975 */ 1976 down_write(&topology_lock); 1977 dev = kfd_assign_gpu(gpu); 1978 if (!dev) 1979 res = kfd_topology_add_device_locked(gpu, gpu_id, &dev); 1980 up_write(&topology_lock); 1981 if (res) 1982 return res; 1983 1984 dev->gpu_id = gpu_id; 1985 gpu->id = gpu_id; 1986 1987 kfd_dev_create_p2p_links(); 1988 1989 /* TODO: Move the following lines to function 1990 * kfd_add_non_crat_information 1991 */ 1992 1993 /* Fill-in additional information that is not available in CRAT but 1994 * needed for the topology 1995 */ 1996 1997 amdgpu_amdkfd_get_cu_info(dev->gpu->adev, &cu_info); 1998 1999 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1; i++) { 2000 dev->node_props.name[i] = __tolower(asic_name[i]); 2001 if (asic_name[i] == '\0') 2002 break; 2003 } 2004 dev->node_props.name[i] = '\0'; 2005 2006 dev->node_props.simd_arrays_per_engine = 2007 cu_info.num_shader_arrays_per_engine; 2008 2009 dev->node_props.gfx_target_version = 2010 gpu->kfd->device_info.gfx_target_version; 2011 dev->node_props.vendor_id = gpu->adev->pdev->vendor; 2012 dev->node_props.device_id = gpu->adev->pdev->device; 2013 dev->node_props.capability |= 2014 ((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) & 2015 HSA_CAP_ASIC_REVISION_MASK); 2016 2017 dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); 2018 if (KFD_GC_VERSION(dev->gpu->kfd) == IP_VERSION(9, 4, 3)) 2019 dev->node_props.location_id |= dev->gpu->node_id; 2020 2021 dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); 2022 dev->node_props.max_engine_clk_fcompute = 2023 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev); 2024 dev->node_props.max_engine_clk_ccompute = 2025 cpufreq_quick_get_max(0) / 1000; 2026 2027 if (gpu->xcp) 2028 dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index; 2029 else 2030 dev->node_props.drm_render_minor = 2031 gpu->kfd->shared_resources.drm_render_minor; 2032 2033 dev->node_props.hive_id = gpu->kfd->hive_id; 2034 dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu); 2035 dev->node_props.num_sdma_xgmi_engines = 2036 kfd_get_num_xgmi_sdma_engines(gpu); 2037 dev->node_props.num_sdma_queues_per_engine = 2038 gpu->kfd->device_info.num_sdma_queues_per_engine - 2039 gpu->kfd->device_info.num_reserved_sdma_queues_per_engine; 2040 dev->node_props.num_gws = (dev->gpu->gws && 2041 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? 2042 dev->gpu->adev->gds.gws_size : 0; 2043 dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); 2044 2045 kfd_fill_mem_clk_max_info(dev); 2046 kfd_fill_iolink_non_crat_info(dev); 2047 2048 switch (dev->gpu->adev->asic_type) { 2049 case CHIP_KAVERI: 2050 case CHIP_HAWAII: 2051 case CHIP_TONGA: 2052 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 << 2053 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2054 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2055 break; 2056 case CHIP_CARRIZO: 2057 case CHIP_FIJI: 2058 case CHIP_POLARIS10: 2059 case CHIP_POLARIS11: 2060 case CHIP_POLARIS12: 2061 case CHIP_VEGAM: 2062 pr_debug("Adding doorbell packet type capability\n"); 2063 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 << 2064 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2065 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2066 break; 2067 default: 2068 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 0, 1)) 2069 WARN(1, "Unexpected ASIC family %u", 2070 dev->gpu->adev->asic_type); 2071 else 2072 kfd_topology_set_capabilities(dev); 2073 } 2074 2075 /* 2076 * Overwrite ATS capability according to needs_iommu_device to fix 2077 * potential missing corresponding bit in CRAT of BIOS. 2078 */ 2079 if (dev->gpu->kfd->use_iommu_v2) 2080 dev->node_props.capability |= HSA_CAP_ATS_PRESENT; 2081 else 2082 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT; 2083 2084 /* Fix errors in CZ CRAT. 2085 * simd_count: Carrizo CRAT reports wrong simd_count, probably 2086 * because it doesn't consider masked out CUs 2087 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd 2088 */ 2089 if (dev->gpu->adev->asic_type == CHIP_CARRIZO) { 2090 dev->node_props.simd_count = 2091 cu_info.simd_per_cu * cu_info.cu_active_number; 2092 dev->node_props.max_waves_per_simd = 10; 2093 } 2094 2095 /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */ 2096 dev->node_props.capability |= 2097 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ? 2098 HSA_CAP_SRAM_EDCSUPPORTED : 0; 2099 dev->node_props.capability |= 2100 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? 2101 HSA_CAP_MEM_EDCSUPPORTED : 0; 2102 2103 if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1)) 2104 dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ? 2105 HSA_CAP_RASEVENTNOTIFY : 0; 2106 2107 if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev)) 2108 dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED; 2109 2110 kfd_debug_print_topology(); 2111 2112 kfd_notify_gpu_change(gpu_id, 1); 2113 2114 return 0; 2115 } 2116 2117 /** 2118 * kfd_topology_update_io_links() - Update IO links after device removal. 2119 * @proximity_domain: Proximity domain value of the dev being removed. 2120 * 2121 * The topology list currently is arranged in increasing order of 2122 * proximity domain. 2123 * 2124 * Two things need to be done when a device is removed: 2125 * 1. All the IO links to this device need to be removed. 2126 * 2. All nodes after the current device node need to move 2127 * up once this device node is removed from the topology 2128 * list. As a result, the proximity domain values for 2129 * all nodes after the node being deleted reduce by 1. 2130 * This would also cause the proximity domain values for 2131 * io links to be updated based on new proximity domain 2132 * values. 2133 * 2134 * Context: The caller must hold write topology_lock. 2135 */ 2136 static void kfd_topology_update_io_links(int proximity_domain) 2137 { 2138 struct kfd_topology_device *dev; 2139 struct kfd_iolink_properties *iolink, *p2plink, *tmp; 2140 2141 list_for_each_entry(dev, &topology_device_list, list) { 2142 if (dev->proximity_domain > proximity_domain) 2143 dev->proximity_domain--; 2144 2145 list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) { 2146 /* 2147 * If there is an io link to the dev being deleted 2148 * then remove that IO link also. 2149 */ 2150 if (iolink->node_to == proximity_domain) { 2151 list_del(&iolink->list); 2152 dev->node_props.io_links_count--; 2153 } else { 2154 if (iolink->node_from > proximity_domain) 2155 iolink->node_from--; 2156 if (iolink->node_to > proximity_domain) 2157 iolink->node_to--; 2158 } 2159 } 2160 2161 list_for_each_entry_safe(p2plink, tmp, &dev->p2p_link_props, list) { 2162 /* 2163 * If there is a p2p link to the dev being deleted 2164 * then remove that p2p link also. 2165 */ 2166 if (p2plink->node_to == proximity_domain) { 2167 list_del(&p2plink->list); 2168 dev->node_props.p2p_links_count--; 2169 } else { 2170 if (p2plink->node_from > proximity_domain) 2171 p2plink->node_from--; 2172 if (p2plink->node_to > proximity_domain) 2173 p2plink->node_to--; 2174 } 2175 } 2176 } 2177 } 2178 2179 int kfd_topology_remove_device(struct kfd_node *gpu) 2180 { 2181 struct kfd_topology_device *dev, *tmp; 2182 uint32_t gpu_id; 2183 int res = -ENODEV; 2184 int i = 0; 2185 2186 down_write(&topology_lock); 2187 2188 list_for_each_entry_safe(dev, tmp, &topology_device_list, list) { 2189 if (dev->gpu == gpu) { 2190 gpu_id = dev->gpu_id; 2191 kfd_remove_sysfs_node_entry(dev); 2192 kfd_release_topology_device(dev); 2193 sys_props.num_devices--; 2194 kfd_topology_update_io_links(i); 2195 topology_crat_proximity_domain = sys_props.num_devices-1; 2196 sys_props.generation_count++; 2197 res = 0; 2198 if (kfd_topology_update_sysfs() < 0) 2199 kfd_topology_release_sysfs(); 2200 break; 2201 } 2202 i++; 2203 } 2204 2205 up_write(&topology_lock); 2206 2207 if (!res) 2208 kfd_notify_gpu_change(gpu_id, 0); 2209 2210 return res; 2211 } 2212 2213 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD 2214 * topology. If GPU device is found @idx, then valid kfd_dev pointer is 2215 * returned through @kdev 2216 * Return - 0: On success (@kdev will be NULL for non GPU nodes) 2217 * -1: If end of list 2218 */ 2219 int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev) 2220 { 2221 2222 struct kfd_topology_device *top_dev; 2223 uint8_t device_idx = 0; 2224 2225 *kdev = NULL; 2226 down_read(&topology_lock); 2227 2228 list_for_each_entry(top_dev, &topology_device_list, list) { 2229 if (device_idx == idx) { 2230 *kdev = top_dev->gpu; 2231 up_read(&topology_lock); 2232 return 0; 2233 } 2234 2235 device_idx++; 2236 } 2237 2238 up_read(&topology_lock); 2239 2240 return -1; 2241 2242 } 2243 2244 static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask) 2245 { 2246 int first_cpu_of_numa_node; 2247 2248 if (!cpumask || cpumask == cpu_none_mask) 2249 return -1; 2250 first_cpu_of_numa_node = cpumask_first(cpumask); 2251 if (first_cpu_of_numa_node >= nr_cpu_ids) 2252 return -1; 2253 #ifdef CONFIG_X86_64 2254 return cpu_data(first_cpu_of_numa_node).apicid; 2255 #else 2256 return first_cpu_of_numa_node; 2257 #endif 2258 } 2259 2260 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor 2261 * of the given NUMA node (numa_node_id) 2262 * Return -1 on failure 2263 */ 2264 int kfd_numa_node_to_apic_id(int numa_node_id) 2265 { 2266 if (numa_node_id == -1) { 2267 pr_warn("Invalid NUMA Node. Use online CPU mask\n"); 2268 return kfd_cpumask_to_apic_id(cpu_online_mask); 2269 } 2270 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id)); 2271 } 2272 2273 void kfd_double_confirm_iommu_support(struct kfd_dev *gpu) 2274 { 2275 struct kfd_topology_device *dev; 2276 2277 gpu->use_iommu_v2 = false; 2278 2279 if (!gpu->device_info.needs_iommu_device) 2280 return; 2281 2282 down_read(&topology_lock); 2283 2284 /* Only use IOMMUv2 if there is an APU topology node with no GPU 2285 * assigned yet. This GPU will be assigned to it. 2286 */ 2287 list_for_each_entry(dev, &topology_device_list, list) 2288 if (dev->node_props.cpu_cores_count && 2289 dev->node_props.simd_count && 2290 !dev->gpu) 2291 gpu->use_iommu_v2 = true; 2292 2293 up_read(&topology_lock); 2294 } 2295 2296 #if defined(CONFIG_DEBUG_FS) 2297 2298 int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) 2299 { 2300 struct kfd_topology_device *dev; 2301 unsigned int i = 0; 2302 int r = 0; 2303 2304 down_read(&topology_lock); 2305 2306 list_for_each_entry(dev, &topology_device_list, list) { 2307 if (!dev->gpu) { 2308 i++; 2309 continue; 2310 } 2311 2312 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2313 r = dqm_debugfs_hqds(m, dev->gpu->dqm); 2314 if (r) 2315 break; 2316 } 2317 2318 up_read(&topology_lock); 2319 2320 return r; 2321 } 2322 2323 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) 2324 { 2325 struct kfd_topology_device *dev; 2326 unsigned int i = 0; 2327 int r = 0; 2328 2329 down_read(&topology_lock); 2330 2331 list_for_each_entry(dev, &topology_device_list, list) { 2332 if (!dev->gpu) { 2333 i++; 2334 continue; 2335 } 2336 2337 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2338 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr); 2339 if (r) 2340 break; 2341 } 2342 2343 up_read(&topology_lock); 2344 2345 return r; 2346 } 2347 2348 #endif 2349