1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/types.h> 25 #include <linux/kernel.h> 26 #include <linux/pci.h> 27 #include <linux/errno.h> 28 #include <linux/acpi.h> 29 #include <linux/hash.h> 30 #include <linux/cpufreq.h> 31 #include <linux/log2.h> 32 #include <linux/dmi.h> 33 #include <linux/atomic.h> 34 #include <linux/crc16.h> 35 36 #include "kfd_priv.h" 37 #include "kfd_crat.h" 38 #include "kfd_topology.h" 39 #include "kfd_device_queue_manager.h" 40 #include "kfd_svm.h" 41 #include "kfd_debug.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_ras.h" 44 #include "amdgpu.h" 45 46 /* topology_device_list - Master list of all topology devices */ 47 static struct list_head topology_device_list; 48 static struct kfd_system_properties sys_props; 49 50 static DECLARE_RWSEM(topology_lock); 51 static uint32_t topology_crat_proximity_domain; 52 53 struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock( 54 uint32_t proximity_domain) 55 { 56 struct kfd_topology_device *top_dev; 57 struct kfd_topology_device *device = NULL; 58 59 list_for_each_entry(top_dev, &topology_device_list, list) 60 if (top_dev->proximity_domain == proximity_domain) { 61 device = top_dev; 62 break; 63 } 64 65 return device; 66 } 67 68 struct kfd_topology_device *kfd_topology_device_by_proximity_domain( 69 uint32_t proximity_domain) 70 { 71 struct kfd_topology_device *device = NULL; 72 73 down_read(&topology_lock); 74 75 device = kfd_topology_device_by_proximity_domain_no_lock( 76 proximity_domain); 77 up_read(&topology_lock); 78 79 return device; 80 } 81 82 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id) 83 { 84 struct kfd_topology_device *top_dev = NULL; 85 struct kfd_topology_device *ret = NULL; 86 87 down_read(&topology_lock); 88 89 list_for_each_entry(top_dev, &topology_device_list, list) 90 if (top_dev->gpu_id == gpu_id) { 91 ret = top_dev; 92 break; 93 } 94 95 up_read(&topology_lock); 96 97 return ret; 98 } 99 100 struct kfd_node *kfd_device_by_id(uint32_t gpu_id) 101 { 102 struct kfd_topology_device *top_dev; 103 104 top_dev = kfd_topology_device_by_id(gpu_id); 105 if (!top_dev) 106 return NULL; 107 108 return top_dev->gpu; 109 } 110 111 struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev) 112 { 113 struct kfd_topology_device *top_dev; 114 struct kfd_node *device = NULL; 115 116 down_read(&topology_lock); 117 118 list_for_each_entry(top_dev, &topology_device_list, list) 119 if (top_dev->gpu && top_dev->gpu->adev->pdev == pdev) { 120 device = top_dev->gpu; 121 break; 122 } 123 124 up_read(&topology_lock); 125 126 return device; 127 } 128 129 /* Called with write topology_lock acquired */ 130 static void kfd_release_topology_device(struct kfd_topology_device *dev) 131 { 132 struct kfd_mem_properties *mem; 133 struct kfd_cache_properties *cache; 134 struct kfd_iolink_properties *iolink; 135 struct kfd_iolink_properties *p2plink; 136 struct kfd_perf_properties *perf; 137 138 list_del(&dev->list); 139 140 while (dev->mem_props.next != &dev->mem_props) { 141 mem = container_of(dev->mem_props.next, 142 struct kfd_mem_properties, list); 143 list_del(&mem->list); 144 kfree(mem); 145 } 146 147 while (dev->cache_props.next != &dev->cache_props) { 148 cache = container_of(dev->cache_props.next, 149 struct kfd_cache_properties, list); 150 list_del(&cache->list); 151 kfree(cache); 152 } 153 154 while (dev->io_link_props.next != &dev->io_link_props) { 155 iolink = container_of(dev->io_link_props.next, 156 struct kfd_iolink_properties, list); 157 list_del(&iolink->list); 158 kfree(iolink); 159 } 160 161 while (dev->p2p_link_props.next != &dev->p2p_link_props) { 162 p2plink = container_of(dev->p2p_link_props.next, 163 struct kfd_iolink_properties, list); 164 list_del(&p2plink->list); 165 kfree(p2plink); 166 } 167 168 while (dev->perf_props.next != &dev->perf_props) { 169 perf = container_of(dev->perf_props.next, 170 struct kfd_perf_properties, list); 171 list_del(&perf->list); 172 kfree(perf); 173 } 174 175 kfree(dev); 176 } 177 178 void kfd_release_topology_device_list(struct list_head *device_list) 179 { 180 struct kfd_topology_device *dev; 181 182 while (!list_empty(device_list)) { 183 dev = list_first_entry(device_list, 184 struct kfd_topology_device, list); 185 kfd_release_topology_device(dev); 186 } 187 } 188 189 static void kfd_release_live_view(void) 190 { 191 kfd_release_topology_device_list(&topology_device_list); 192 memset(&sys_props, 0, sizeof(sys_props)); 193 } 194 195 struct kfd_topology_device *kfd_create_topology_device( 196 struct list_head *device_list) 197 { 198 struct kfd_topology_device *dev; 199 200 dev = kfd_alloc_struct(dev); 201 if (!dev) { 202 pr_err("No memory to allocate a topology device"); 203 return NULL; 204 } 205 206 INIT_LIST_HEAD(&dev->mem_props); 207 INIT_LIST_HEAD(&dev->cache_props); 208 INIT_LIST_HEAD(&dev->io_link_props); 209 INIT_LIST_HEAD(&dev->p2p_link_props); 210 INIT_LIST_HEAD(&dev->perf_props); 211 212 list_add_tail(&dev->list, device_list); 213 214 return dev; 215 } 216 217 218 #define sysfs_show_gen_prop(buffer, offs, fmt, ...) \ 219 (offs += snprintf(buffer+offs, PAGE_SIZE-offs, \ 220 fmt, __VA_ARGS__)) 221 #define sysfs_show_32bit_prop(buffer, offs, name, value) \ 222 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value) 223 #define sysfs_show_64bit_prop(buffer, offs, name, value) \ 224 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value) 225 #define sysfs_show_32bit_val(buffer, offs, value) \ 226 sysfs_show_gen_prop(buffer, offs, "%u\n", value) 227 #define sysfs_show_str_val(buffer, offs, value) \ 228 sysfs_show_gen_prop(buffer, offs, "%s\n", value) 229 230 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, 231 char *buffer) 232 { 233 int offs = 0; 234 235 /* Making sure that the buffer is an empty string */ 236 buffer[0] = 0; 237 238 if (attr == &sys_props.attr_genid) { 239 sysfs_show_32bit_val(buffer, offs, 240 sys_props.generation_count); 241 } else if (attr == &sys_props.attr_props) { 242 sysfs_show_64bit_prop(buffer, offs, "platform_oem", 243 sys_props.platform_oem); 244 sysfs_show_64bit_prop(buffer, offs, "platform_id", 245 sys_props.platform_id); 246 sysfs_show_64bit_prop(buffer, offs, "platform_rev", 247 sys_props.platform_rev); 248 } else { 249 offs = -EINVAL; 250 } 251 252 return offs; 253 } 254 255 static void kfd_topology_kobj_release(struct kobject *kobj) 256 { 257 kfree(kobj); 258 } 259 260 static const struct sysfs_ops sysprops_ops = { 261 .show = sysprops_show, 262 }; 263 264 static const struct kobj_type sysprops_type = { 265 .release = kfd_topology_kobj_release, 266 .sysfs_ops = &sysprops_ops, 267 }; 268 269 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, 270 char *buffer) 271 { 272 int offs = 0; 273 struct kfd_iolink_properties *iolink; 274 275 /* Making sure that the buffer is an empty string */ 276 buffer[0] = 0; 277 278 iolink = container_of(attr, struct kfd_iolink_properties, attr); 279 if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu)) 280 return -EPERM; 281 sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type); 282 sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj); 283 sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min); 284 sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from); 285 sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to); 286 sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight); 287 sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency); 288 sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency); 289 sysfs_show_32bit_prop(buffer, offs, "min_bandwidth", 290 iolink->min_bandwidth); 291 sysfs_show_32bit_prop(buffer, offs, "max_bandwidth", 292 iolink->max_bandwidth); 293 sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size", 294 iolink->rec_transfer_size); 295 sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags); 296 297 return offs; 298 } 299 300 static const struct sysfs_ops iolink_ops = { 301 .show = iolink_show, 302 }; 303 304 static const struct kobj_type iolink_type = { 305 .release = kfd_topology_kobj_release, 306 .sysfs_ops = &iolink_ops, 307 }; 308 309 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, 310 char *buffer) 311 { 312 int offs = 0; 313 struct kfd_mem_properties *mem; 314 315 /* Making sure that the buffer is an empty string */ 316 buffer[0] = 0; 317 318 mem = container_of(attr, struct kfd_mem_properties, attr); 319 if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu)) 320 return -EPERM; 321 sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type); 322 sysfs_show_64bit_prop(buffer, offs, "size_in_bytes", 323 mem->size_in_bytes); 324 sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags); 325 sysfs_show_32bit_prop(buffer, offs, "width", mem->width); 326 sysfs_show_32bit_prop(buffer, offs, "mem_clk_max", 327 mem->mem_clk_max); 328 329 return offs; 330 } 331 332 static const struct sysfs_ops mem_ops = { 333 .show = mem_show, 334 }; 335 336 static const struct kobj_type mem_type = { 337 .release = kfd_topology_kobj_release, 338 .sysfs_ops = &mem_ops, 339 }; 340 341 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, 342 char *buffer) 343 { 344 int offs = 0; 345 uint32_t i, j; 346 struct kfd_cache_properties *cache; 347 348 /* Making sure that the buffer is an empty string */ 349 buffer[0] = 0; 350 cache = container_of(attr, struct kfd_cache_properties, attr); 351 if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu)) 352 return -EPERM; 353 sysfs_show_32bit_prop(buffer, offs, "processor_id_low", 354 cache->processor_id_low); 355 sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level); 356 sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size); 357 sysfs_show_32bit_prop(buffer, offs, "cache_line_size", 358 cache->cacheline_size); 359 sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag", 360 cache->cachelines_per_tag); 361 sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc); 362 sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency); 363 sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type); 364 365 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map "); 366 for (i = 0; i < cache->sibling_map_size; i++) 367 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) 368 /* Check each bit */ 369 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,", 370 (cache->sibling_map[i] >> j) & 1); 371 372 /* Replace the last "," with end of line */ 373 buffer[offs-1] = '\n'; 374 return offs; 375 } 376 377 static const struct sysfs_ops cache_ops = { 378 .show = kfd_cache_show, 379 }; 380 381 static const struct kobj_type cache_type = { 382 .release = kfd_topology_kobj_release, 383 .sysfs_ops = &cache_ops, 384 }; 385 386 /****** Sysfs of Performance Counters ******/ 387 388 struct kfd_perf_attr { 389 struct kobj_attribute attr; 390 uint32_t data; 391 }; 392 393 static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs, 394 char *buf) 395 { 396 int offs = 0; 397 struct kfd_perf_attr *attr; 398 399 buf[0] = 0; 400 attr = container_of(attrs, struct kfd_perf_attr, attr); 401 if (!attr->data) /* invalid data for PMC */ 402 return 0; 403 else 404 return sysfs_show_32bit_val(buf, offs, attr->data); 405 } 406 407 #define KFD_PERF_DESC(_name, _data) \ 408 { \ 409 .attr = __ATTR(_name, 0444, perf_show, NULL), \ 410 .data = _data, \ 411 } 412 413 static struct kfd_perf_attr perf_attr_iommu[] = { 414 KFD_PERF_DESC(max_concurrent, 0), 415 KFD_PERF_DESC(num_counters, 0), 416 KFD_PERF_DESC(counter_ids, 0), 417 }; 418 /****************************************/ 419 420 static ssize_t node_show(struct kobject *kobj, struct attribute *attr, 421 char *buffer) 422 { 423 int offs = 0; 424 struct kfd_topology_device *dev; 425 uint32_t log_max_watch_addr; 426 427 /* Making sure that the buffer is an empty string */ 428 buffer[0] = 0; 429 430 if (strcmp(attr->name, "gpu_id") == 0) { 431 dev = container_of(attr, struct kfd_topology_device, 432 attr_gpuid); 433 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 434 return -EPERM; 435 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id); 436 } 437 438 if (strcmp(attr->name, "name") == 0) { 439 dev = container_of(attr, struct kfd_topology_device, 440 attr_name); 441 442 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 443 return -EPERM; 444 return sysfs_show_str_val(buffer, offs, dev->node_props.name); 445 } 446 447 dev = container_of(attr, struct kfd_topology_device, 448 attr_props); 449 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 450 return -EPERM; 451 sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count", 452 dev->node_props.cpu_cores_count); 453 sysfs_show_32bit_prop(buffer, offs, "simd_count", 454 dev->gpu ? dev->node_props.simd_count : 0); 455 sysfs_show_32bit_prop(buffer, offs, "mem_banks_count", 456 dev->node_props.mem_banks_count); 457 sysfs_show_32bit_prop(buffer, offs, "caches_count", 458 dev->node_props.caches_count); 459 sysfs_show_32bit_prop(buffer, offs, "io_links_count", 460 dev->node_props.io_links_count); 461 sysfs_show_32bit_prop(buffer, offs, "p2p_links_count", 462 dev->node_props.p2p_links_count); 463 sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base", 464 dev->node_props.cpu_core_id_base); 465 sysfs_show_32bit_prop(buffer, offs, "simd_id_base", 466 dev->node_props.simd_id_base); 467 sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd", 468 dev->node_props.max_waves_per_simd); 469 sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb", 470 dev->node_props.lds_size_in_kb); 471 sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb", 472 dev->node_props.gds_size_in_kb); 473 sysfs_show_32bit_prop(buffer, offs, "num_gws", 474 dev->node_props.num_gws); 475 sysfs_show_32bit_prop(buffer, offs, "wave_front_size", 476 dev->node_props.wave_front_size); 477 sysfs_show_32bit_prop(buffer, offs, "array_count", 478 dev->gpu ? (dev->node_props.array_count * 479 NUM_XCC(dev->gpu->xcc_mask)) : 0); 480 sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine", 481 dev->node_props.simd_arrays_per_engine); 482 sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array", 483 dev->node_props.cu_per_simd_array); 484 sysfs_show_32bit_prop(buffer, offs, "simd_per_cu", 485 dev->node_props.simd_per_cu); 486 sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu", 487 dev->node_props.max_slots_scratch_cu); 488 sysfs_show_32bit_prop(buffer, offs, "gfx_target_version", 489 dev->node_props.gfx_target_version); 490 sysfs_show_32bit_prop(buffer, offs, "vendor_id", 491 dev->node_props.vendor_id); 492 sysfs_show_32bit_prop(buffer, offs, "device_id", 493 dev->node_props.device_id); 494 sysfs_show_32bit_prop(buffer, offs, "location_id", 495 dev->node_props.location_id); 496 sysfs_show_32bit_prop(buffer, offs, "domain", 497 dev->node_props.domain); 498 sysfs_show_32bit_prop(buffer, offs, "drm_render_minor", 499 dev->node_props.drm_render_minor); 500 sysfs_show_64bit_prop(buffer, offs, "hive_id", 501 dev->node_props.hive_id); 502 sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines", 503 dev->node_props.num_sdma_engines); 504 sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines", 505 dev->node_props.num_sdma_xgmi_engines); 506 sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine", 507 dev->node_props.num_sdma_queues_per_engine); 508 sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", 509 dev->node_props.num_cp_queues); 510 511 if (dev->gpu) { 512 log_max_watch_addr = 513 __ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points); 514 515 if (log_max_watch_addr) { 516 dev->node_props.capability |= 517 HSA_CAP_WATCH_POINTS_SUPPORTED; 518 519 dev->node_props.capability |= 520 ((log_max_watch_addr << 521 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & 522 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); 523 } 524 525 if (dev->gpu->adev->asic_type == CHIP_TONGA) 526 dev->node_props.capability |= 527 HSA_CAP_AQL_QUEUE_DOUBLE_MAP; 528 529 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute", 530 dev->node_props.max_engine_clk_fcompute); 531 532 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL); 533 534 sysfs_show_32bit_prop(buffer, offs, "fw_version", 535 dev->gpu->kfd->mec_fw_version); 536 sysfs_show_32bit_prop(buffer, offs, "capability", 537 dev->node_props.capability); 538 sysfs_show_64bit_prop(buffer, offs, "debug_prop", 539 dev->node_props.debug_prop); 540 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", 541 dev->gpu->kfd->sdma_fw_version); 542 sysfs_show_64bit_prop(buffer, offs, "unique_id", 543 dev->gpu->adev->unique_id); 544 sysfs_show_32bit_prop(buffer, offs, "num_xcc", 545 NUM_XCC(dev->gpu->xcc_mask)); 546 } 547 548 return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", 549 cpufreq_quick_get_max(0)/1000); 550 } 551 552 static const struct sysfs_ops node_ops = { 553 .show = node_show, 554 }; 555 556 static const struct kobj_type node_type = { 557 .release = kfd_topology_kobj_release, 558 .sysfs_ops = &node_ops, 559 }; 560 561 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) 562 { 563 sysfs_remove_file(kobj, attr); 564 kobject_del(kobj); 565 kobject_put(kobj); 566 } 567 568 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) 569 { 570 struct kfd_iolink_properties *p2plink; 571 struct kfd_iolink_properties *iolink; 572 struct kfd_cache_properties *cache; 573 struct kfd_mem_properties *mem; 574 struct kfd_perf_properties *perf; 575 576 if (dev->kobj_iolink) { 577 list_for_each_entry(iolink, &dev->io_link_props, list) 578 if (iolink->kobj) { 579 kfd_remove_sysfs_file(iolink->kobj, 580 &iolink->attr); 581 iolink->kobj = NULL; 582 } 583 kobject_del(dev->kobj_iolink); 584 kobject_put(dev->kobj_iolink); 585 dev->kobj_iolink = NULL; 586 } 587 588 if (dev->kobj_p2plink) { 589 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 590 if (p2plink->kobj) { 591 kfd_remove_sysfs_file(p2plink->kobj, 592 &p2plink->attr); 593 p2plink->kobj = NULL; 594 } 595 kobject_del(dev->kobj_p2plink); 596 kobject_put(dev->kobj_p2plink); 597 dev->kobj_p2plink = NULL; 598 } 599 600 if (dev->kobj_cache) { 601 list_for_each_entry(cache, &dev->cache_props, list) 602 if (cache->kobj) { 603 kfd_remove_sysfs_file(cache->kobj, 604 &cache->attr); 605 cache->kobj = NULL; 606 } 607 kobject_del(dev->kobj_cache); 608 kobject_put(dev->kobj_cache); 609 dev->kobj_cache = NULL; 610 } 611 612 if (dev->kobj_mem) { 613 list_for_each_entry(mem, &dev->mem_props, list) 614 if (mem->kobj) { 615 kfd_remove_sysfs_file(mem->kobj, &mem->attr); 616 mem->kobj = NULL; 617 } 618 kobject_del(dev->kobj_mem); 619 kobject_put(dev->kobj_mem); 620 dev->kobj_mem = NULL; 621 } 622 623 if (dev->kobj_perf) { 624 list_for_each_entry(perf, &dev->perf_props, list) { 625 kfree(perf->attr_group); 626 perf->attr_group = NULL; 627 } 628 kobject_del(dev->kobj_perf); 629 kobject_put(dev->kobj_perf); 630 dev->kobj_perf = NULL; 631 } 632 633 if (dev->kobj_node) { 634 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); 635 sysfs_remove_file(dev->kobj_node, &dev->attr_name); 636 sysfs_remove_file(dev->kobj_node, &dev->attr_props); 637 kobject_del(dev->kobj_node); 638 kobject_put(dev->kobj_node); 639 dev->kobj_node = NULL; 640 } 641 } 642 643 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, 644 uint32_t id) 645 { 646 struct kfd_iolink_properties *p2plink; 647 struct kfd_iolink_properties *iolink; 648 struct kfd_cache_properties *cache; 649 struct kfd_mem_properties *mem; 650 struct kfd_perf_properties *perf; 651 int ret; 652 uint32_t i, num_attrs; 653 struct attribute **attrs; 654 655 if (WARN_ON(dev->kobj_node)) 656 return -EEXIST; 657 658 /* 659 * Creating the sysfs folders 660 */ 661 dev->kobj_node = kfd_alloc_struct(dev->kobj_node); 662 if (!dev->kobj_node) 663 return -ENOMEM; 664 665 ret = kobject_init_and_add(dev->kobj_node, &node_type, 666 sys_props.kobj_nodes, "%d", id); 667 if (ret < 0) { 668 kobject_put(dev->kobj_node); 669 return ret; 670 } 671 672 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); 673 if (!dev->kobj_mem) 674 return -ENOMEM; 675 676 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); 677 if (!dev->kobj_cache) 678 return -ENOMEM; 679 680 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); 681 if (!dev->kobj_iolink) 682 return -ENOMEM; 683 684 dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node); 685 if (!dev->kobj_p2plink) 686 return -ENOMEM; 687 688 dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node); 689 if (!dev->kobj_perf) 690 return -ENOMEM; 691 692 /* 693 * Creating sysfs files for node properties 694 */ 695 dev->attr_gpuid.name = "gpu_id"; 696 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; 697 sysfs_attr_init(&dev->attr_gpuid); 698 dev->attr_name.name = "name"; 699 dev->attr_name.mode = KFD_SYSFS_FILE_MODE; 700 sysfs_attr_init(&dev->attr_name); 701 dev->attr_props.name = "properties"; 702 dev->attr_props.mode = KFD_SYSFS_FILE_MODE; 703 sysfs_attr_init(&dev->attr_props); 704 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); 705 if (ret < 0) 706 return ret; 707 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); 708 if (ret < 0) 709 return ret; 710 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); 711 if (ret < 0) 712 return ret; 713 714 i = 0; 715 list_for_each_entry(mem, &dev->mem_props, list) { 716 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 717 if (!mem->kobj) 718 return -ENOMEM; 719 ret = kobject_init_and_add(mem->kobj, &mem_type, 720 dev->kobj_mem, "%d", i); 721 if (ret < 0) { 722 kobject_put(mem->kobj); 723 return ret; 724 } 725 726 mem->attr.name = "properties"; 727 mem->attr.mode = KFD_SYSFS_FILE_MODE; 728 sysfs_attr_init(&mem->attr); 729 ret = sysfs_create_file(mem->kobj, &mem->attr); 730 if (ret < 0) 731 return ret; 732 i++; 733 } 734 735 i = 0; 736 list_for_each_entry(cache, &dev->cache_props, list) { 737 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 738 if (!cache->kobj) 739 return -ENOMEM; 740 ret = kobject_init_and_add(cache->kobj, &cache_type, 741 dev->kobj_cache, "%d", i); 742 if (ret < 0) { 743 kobject_put(cache->kobj); 744 return ret; 745 } 746 747 cache->attr.name = "properties"; 748 cache->attr.mode = KFD_SYSFS_FILE_MODE; 749 sysfs_attr_init(&cache->attr); 750 ret = sysfs_create_file(cache->kobj, &cache->attr); 751 if (ret < 0) 752 return ret; 753 i++; 754 } 755 756 i = 0; 757 list_for_each_entry(iolink, &dev->io_link_props, list) { 758 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 759 if (!iolink->kobj) 760 return -ENOMEM; 761 ret = kobject_init_and_add(iolink->kobj, &iolink_type, 762 dev->kobj_iolink, "%d", i); 763 if (ret < 0) { 764 kobject_put(iolink->kobj); 765 return ret; 766 } 767 768 iolink->attr.name = "properties"; 769 iolink->attr.mode = KFD_SYSFS_FILE_MODE; 770 sysfs_attr_init(&iolink->attr); 771 ret = sysfs_create_file(iolink->kobj, &iolink->attr); 772 if (ret < 0) 773 return ret; 774 i++; 775 } 776 777 i = 0; 778 list_for_each_entry(p2plink, &dev->p2p_link_props, list) { 779 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 780 if (!p2plink->kobj) 781 return -ENOMEM; 782 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 783 dev->kobj_p2plink, "%d", i); 784 if (ret < 0) { 785 kobject_put(p2plink->kobj); 786 return ret; 787 } 788 789 p2plink->attr.name = "properties"; 790 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 791 sysfs_attr_init(&p2plink->attr); 792 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 793 if (ret < 0) 794 return ret; 795 i++; 796 } 797 798 /* All hardware blocks have the same number of attributes. */ 799 num_attrs = ARRAY_SIZE(perf_attr_iommu); 800 list_for_each_entry(perf, &dev->perf_props, list) { 801 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr) 802 * num_attrs + sizeof(struct attribute_group), 803 GFP_KERNEL); 804 if (!perf->attr_group) 805 return -ENOMEM; 806 807 attrs = (struct attribute **)(perf->attr_group + 1); 808 if (!strcmp(perf->block_name, "iommu")) { 809 /* Information of IOMMU's num_counters and counter_ids is shown 810 * under /sys/bus/event_source/devices/amd_iommu. We don't 811 * duplicate here. 812 */ 813 perf_attr_iommu[0].data = perf->max_concurrent; 814 for (i = 0; i < num_attrs; i++) 815 attrs[i] = &perf_attr_iommu[i].attr.attr; 816 } 817 perf->attr_group->name = perf->block_name; 818 perf->attr_group->attrs = attrs; 819 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group); 820 if (ret < 0) 821 return ret; 822 } 823 824 return 0; 825 } 826 827 /* Called with write topology lock acquired */ 828 static int kfd_build_sysfs_node_tree(void) 829 { 830 struct kfd_topology_device *dev; 831 int ret; 832 uint32_t i = 0; 833 834 list_for_each_entry(dev, &topology_device_list, list) { 835 ret = kfd_build_sysfs_node_entry(dev, i); 836 if (ret < 0) 837 return ret; 838 i++; 839 } 840 841 return 0; 842 } 843 844 /* Called with write topology lock acquired */ 845 static void kfd_remove_sysfs_node_tree(void) 846 { 847 struct kfd_topology_device *dev; 848 849 list_for_each_entry(dev, &topology_device_list, list) 850 kfd_remove_sysfs_node_entry(dev); 851 } 852 853 static int kfd_topology_update_sysfs(void) 854 { 855 int ret; 856 857 if (!sys_props.kobj_topology) { 858 sys_props.kobj_topology = 859 kfd_alloc_struct(sys_props.kobj_topology); 860 if (!sys_props.kobj_topology) 861 return -ENOMEM; 862 863 ret = kobject_init_and_add(sys_props.kobj_topology, 864 &sysprops_type, &kfd_device->kobj, 865 "topology"); 866 if (ret < 0) { 867 kobject_put(sys_props.kobj_topology); 868 return ret; 869 } 870 871 sys_props.kobj_nodes = kobject_create_and_add("nodes", 872 sys_props.kobj_topology); 873 if (!sys_props.kobj_nodes) 874 return -ENOMEM; 875 876 sys_props.attr_genid.name = "generation_id"; 877 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; 878 sysfs_attr_init(&sys_props.attr_genid); 879 ret = sysfs_create_file(sys_props.kobj_topology, 880 &sys_props.attr_genid); 881 if (ret < 0) 882 return ret; 883 884 sys_props.attr_props.name = "system_properties"; 885 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; 886 sysfs_attr_init(&sys_props.attr_props); 887 ret = sysfs_create_file(sys_props.kobj_topology, 888 &sys_props.attr_props); 889 if (ret < 0) 890 return ret; 891 } 892 893 kfd_remove_sysfs_node_tree(); 894 895 return kfd_build_sysfs_node_tree(); 896 } 897 898 static void kfd_topology_release_sysfs(void) 899 { 900 kfd_remove_sysfs_node_tree(); 901 if (sys_props.kobj_topology) { 902 sysfs_remove_file(sys_props.kobj_topology, 903 &sys_props.attr_genid); 904 sysfs_remove_file(sys_props.kobj_topology, 905 &sys_props.attr_props); 906 if (sys_props.kobj_nodes) { 907 kobject_del(sys_props.kobj_nodes); 908 kobject_put(sys_props.kobj_nodes); 909 sys_props.kobj_nodes = NULL; 910 } 911 kobject_del(sys_props.kobj_topology); 912 kobject_put(sys_props.kobj_topology); 913 sys_props.kobj_topology = NULL; 914 } 915 } 916 917 /* Called with write topology_lock acquired */ 918 static void kfd_topology_update_device_list(struct list_head *temp_list, 919 struct list_head *master_list) 920 { 921 while (!list_empty(temp_list)) { 922 list_move_tail(temp_list->next, master_list); 923 sys_props.num_devices++; 924 } 925 } 926 927 static void kfd_debug_print_topology(void) 928 { 929 struct kfd_topology_device *dev; 930 931 down_read(&topology_lock); 932 933 dev = list_last_entry(&topology_device_list, 934 struct kfd_topology_device, list); 935 if (dev) { 936 if (dev->node_props.cpu_cores_count && 937 dev->node_props.simd_count) { 938 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n", 939 dev->node_props.device_id, 940 dev->node_props.vendor_id); 941 } else if (dev->node_props.cpu_cores_count) 942 pr_info("Topology: Add CPU node\n"); 943 else if (dev->node_props.simd_count) 944 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n", 945 dev->node_props.device_id, 946 dev->node_props.vendor_id); 947 } 948 up_read(&topology_lock); 949 } 950 951 /* Helper function for intializing platform_xx members of 952 * kfd_system_properties. Uses OEM info from the last CPU/APU node. 953 */ 954 static void kfd_update_system_properties(void) 955 { 956 struct kfd_topology_device *dev; 957 958 down_read(&topology_lock); 959 dev = list_last_entry(&topology_device_list, 960 struct kfd_topology_device, list); 961 if (dev) { 962 sys_props.platform_id = dev->oem_id64; 963 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id); 964 sys_props.platform_rev = dev->oem_revision; 965 } 966 up_read(&topology_lock); 967 } 968 969 static void find_system_memory(const struct dmi_header *dm, 970 void *private) 971 { 972 struct kfd_mem_properties *mem; 973 u16 mem_width, mem_clock; 974 struct kfd_topology_device *kdev = 975 (struct kfd_topology_device *)private; 976 const u8 *dmi_data = (const u8 *)(dm + 1); 977 978 if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) { 979 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6)); 980 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11)); 981 list_for_each_entry(mem, &kdev->mem_props, list) { 982 if (mem_width != 0xFFFF && mem_width != 0) 983 mem->width = mem_width; 984 if (mem_clock != 0) 985 mem->mem_clk_max = mem_clock; 986 } 987 } 988 } 989 990 /* kfd_add_non_crat_information - Add information that is not currently 991 * defined in CRAT but is necessary for KFD topology 992 * @dev - topology device to which addition info is added 993 */ 994 static void kfd_add_non_crat_information(struct kfd_topology_device *kdev) 995 { 996 /* Check if CPU only node. */ 997 if (!kdev->gpu) { 998 /* Add system memory information */ 999 dmi_walk(find_system_memory, kdev); 1000 } 1001 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */ 1002 } 1003 1004 int kfd_topology_init(void) 1005 { 1006 void *crat_image = NULL; 1007 size_t image_size = 0; 1008 int ret; 1009 struct list_head temp_topology_device_list; 1010 int cpu_only_node = 0; 1011 struct kfd_topology_device *kdev; 1012 int proximity_domain; 1013 1014 /* topology_device_list - Master list of all topology devices 1015 * temp_topology_device_list - temporary list created while parsing CRAT 1016 * or VCRAT. Once parsing is complete the contents of list is moved to 1017 * topology_device_list 1018 */ 1019 1020 /* Initialize the head for the both the lists */ 1021 INIT_LIST_HEAD(&topology_device_list); 1022 INIT_LIST_HEAD(&temp_topology_device_list); 1023 init_rwsem(&topology_lock); 1024 1025 memset(&sys_props, 0, sizeof(sys_props)); 1026 1027 /* Proximity domains in ACPI CRAT tables start counting at 1028 * 0. The same should be true for virtual CRAT tables created 1029 * at this stage. GPUs added later in kfd_topology_add_device 1030 * use a counter. 1031 */ 1032 proximity_domain = 0; 1033 1034 ret = kfd_create_crat_image_virtual(&crat_image, &image_size, 1035 COMPUTE_UNIT_CPU, NULL, 1036 proximity_domain); 1037 cpu_only_node = 1; 1038 if (ret) { 1039 pr_err("Error creating VCRAT table for CPU\n"); 1040 return ret; 1041 } 1042 1043 ret = kfd_parse_crat_table(crat_image, 1044 &temp_topology_device_list, 1045 proximity_domain); 1046 if (ret) { 1047 pr_err("Error parsing VCRAT table for CPU\n"); 1048 goto err; 1049 } 1050 1051 kdev = list_first_entry(&temp_topology_device_list, 1052 struct kfd_topology_device, list); 1053 1054 down_write(&topology_lock); 1055 kfd_topology_update_device_list(&temp_topology_device_list, 1056 &topology_device_list); 1057 topology_crat_proximity_domain = sys_props.num_devices-1; 1058 ret = kfd_topology_update_sysfs(); 1059 up_write(&topology_lock); 1060 1061 if (!ret) { 1062 sys_props.generation_count++; 1063 kfd_update_system_properties(); 1064 kfd_debug_print_topology(); 1065 } else 1066 pr_err("Failed to update topology in sysfs ret=%d\n", ret); 1067 1068 /* For nodes with GPU, this information gets added 1069 * when GPU is detected (kfd_topology_add_device). 1070 */ 1071 if (cpu_only_node) { 1072 /* Add additional information to CPU only node created above */ 1073 down_write(&topology_lock); 1074 kdev = list_first_entry(&topology_device_list, 1075 struct kfd_topology_device, list); 1076 up_write(&topology_lock); 1077 kfd_add_non_crat_information(kdev); 1078 } 1079 1080 err: 1081 kfd_destroy_crat_image(crat_image); 1082 return ret; 1083 } 1084 1085 void kfd_topology_shutdown(void) 1086 { 1087 down_write(&topology_lock); 1088 kfd_topology_release_sysfs(); 1089 kfd_release_live_view(); 1090 up_write(&topology_lock); 1091 } 1092 1093 static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu) 1094 { 1095 uint32_t gpu_id; 1096 uint32_t buf[8]; 1097 uint64_t local_mem_size; 1098 struct kfd_topology_device *dev; 1099 bool is_unique; 1100 uint8_t *crc_buf; 1101 1102 if (!gpu) 1103 return 0; 1104 1105 crc_buf = (uint8_t *)&buf; 1106 local_mem_size = gpu->local_mem_info.local_mem_size_private + 1107 gpu->local_mem_info.local_mem_size_public; 1108 buf[0] = gpu->adev->pdev->devfn; 1109 buf[1] = gpu->adev->pdev->subsystem_vendor | 1110 (gpu->adev->pdev->subsystem_device << 16); 1111 buf[2] = pci_domain_nr(gpu->adev->pdev->bus); 1112 buf[3] = gpu->adev->pdev->device; 1113 buf[4] = gpu->adev->pdev->bus->number; 1114 buf[5] = lower_32_bits(local_mem_size); 1115 buf[6] = upper_32_bits(local_mem_size); 1116 buf[7] = (ffs(gpu->xcc_mask) - 1) | (NUM_XCC(gpu->xcc_mask) << 16); 1117 1118 gpu_id = crc16(0, crc_buf, sizeof(buf)) & 1119 ((1 << KFD_GPU_ID_HASH_WIDTH) - 1); 1120 1121 /* There is a very small possibility when generating a 1122 * 16 (KFD_GPU_ID_HASH_WIDTH) bit value from 8 word buffer 1123 * that the value could be 0 or non-unique. So, check if 1124 * it is unique and non-zero. If not unique increment till 1125 * unique one is found. In case of overflow, restart from 1 1126 */ 1127 1128 down_read(&topology_lock); 1129 do { 1130 is_unique = true; 1131 if (!gpu_id) 1132 gpu_id = 1; 1133 list_for_each_entry(dev, &topology_device_list, list) { 1134 if (dev->gpu && dev->gpu_id == gpu_id) { 1135 is_unique = false; 1136 break; 1137 } 1138 } 1139 if (unlikely(!is_unique)) 1140 gpu_id = (gpu_id + 1) & 1141 ((1 << KFD_GPU_ID_HASH_WIDTH) - 1); 1142 } while (!is_unique); 1143 up_read(&topology_lock); 1144 1145 return gpu_id; 1146 } 1147 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If 1148 * the GPU device is not already present in the topology device 1149 * list then return NULL. This means a new topology device has to 1150 * be created for this GPU. 1151 */ 1152 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu) 1153 { 1154 struct kfd_topology_device *dev; 1155 struct kfd_topology_device *out_dev = NULL; 1156 struct kfd_mem_properties *mem; 1157 struct kfd_cache_properties *cache; 1158 struct kfd_iolink_properties *iolink; 1159 struct kfd_iolink_properties *p2plink; 1160 1161 list_for_each_entry(dev, &topology_device_list, list) { 1162 /* Discrete GPUs need their own topology device list 1163 * entries. Don't assign them to CPU/APU nodes. 1164 */ 1165 if (dev->node_props.cpu_cores_count) 1166 continue; 1167 1168 if (!dev->gpu && (dev->node_props.simd_count > 0)) { 1169 dev->gpu = gpu; 1170 out_dev = dev; 1171 1172 list_for_each_entry(mem, &dev->mem_props, list) 1173 mem->gpu = dev->gpu; 1174 list_for_each_entry(cache, &dev->cache_props, list) 1175 cache->gpu = dev->gpu; 1176 list_for_each_entry(iolink, &dev->io_link_props, list) 1177 iolink->gpu = dev->gpu; 1178 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 1179 p2plink->gpu = dev->gpu; 1180 break; 1181 } 1182 } 1183 return out_dev; 1184 } 1185 1186 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) 1187 { 1188 /* 1189 * TODO: Generate an event for thunk about the arrival/removal 1190 * of the GPU 1191 */ 1192 } 1193 1194 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info, 1195 * patch this after CRAT parsing. 1196 */ 1197 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) 1198 { 1199 struct kfd_mem_properties *mem; 1200 struct kfd_local_mem_info local_mem_info; 1201 1202 if (!dev) 1203 return; 1204 1205 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with 1206 * single bank of VRAM local memory. 1207 * for dGPUs - VCRAT reports only one bank of Local Memory 1208 * for APUs - If CRAT from ACPI reports more than one bank, then 1209 * all the banks will report the same mem_clk_max information 1210 */ 1211 amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info, 1212 dev->gpu->xcp); 1213 1214 list_for_each_entry(mem, &dev->mem_props, list) 1215 mem->mem_clk_max = local_mem_info.mem_clk_max; 1216 } 1217 1218 static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev, 1219 struct kfd_topology_device *target_gpu_dev, 1220 struct kfd_iolink_properties *link) 1221 { 1222 /* xgmi always supports atomics between links. */ 1223 if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI) 1224 return; 1225 1226 /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */ 1227 if (target_gpu_dev) { 1228 uint32_t cap; 1229 1230 pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev, 1231 PCI_EXP_DEVCAP2, &cap); 1232 1233 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 1234 PCI_EXP_DEVCAP2_ATOMIC_COMP64))) 1235 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1236 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1237 /* set gpu (dev) flags. */ 1238 } else { 1239 if (!dev->gpu->kfd->pci_atomic_requested || 1240 dev->gpu->adev->asic_type == CHIP_HAWAII) 1241 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1242 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1243 } 1244 } 1245 1246 static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev, 1247 struct kfd_iolink_properties *outbound_link, 1248 struct kfd_iolink_properties *inbound_link) 1249 { 1250 /* CPU -> GPU with PCIe */ 1251 if (!to_dev->gpu && 1252 inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) 1253 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1254 1255 if (to_dev->gpu) { 1256 /* GPU <-> GPU with PCIe and 1257 * Vega20 with XGMI 1258 */ 1259 if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS || 1260 (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI && 1261 KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) { 1262 outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1263 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1264 } 1265 } 1266 } 1267 1268 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev) 1269 { 1270 struct kfd_iolink_properties *link, *inbound_link; 1271 struct kfd_topology_device *peer_dev; 1272 1273 if (!dev || !dev->gpu) 1274 return; 1275 1276 /* GPU only creates direct links so apply flags setting to all */ 1277 list_for_each_entry(link, &dev->io_link_props, list) { 1278 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1279 kfd_set_iolink_no_atomics(dev, NULL, link); 1280 peer_dev = kfd_topology_device_by_proximity_domain( 1281 link->node_to); 1282 1283 if (!peer_dev) 1284 continue; 1285 1286 /* Include the CPU peer in GPU hive if connected over xGMI. */ 1287 if (!peer_dev->gpu && 1288 link->iolink_type == CRAT_IOLINK_TYPE_XGMI) { 1289 /* 1290 * If the GPU is not part of a GPU hive, use its pci 1291 * device location as the hive ID to bind with the CPU. 1292 */ 1293 if (!dev->node_props.hive_id) 1294 dev->node_props.hive_id = pci_dev_id(dev->gpu->adev->pdev); 1295 peer_dev->node_props.hive_id = dev->node_props.hive_id; 1296 } 1297 1298 list_for_each_entry(inbound_link, &peer_dev->io_link_props, 1299 list) { 1300 if (inbound_link->node_to != link->node_from) 1301 continue; 1302 1303 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1304 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1305 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1306 } 1307 } 1308 1309 /* Create indirect links so apply flags setting to all */ 1310 list_for_each_entry(link, &dev->p2p_link_props, list) { 1311 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1312 kfd_set_iolink_no_atomics(dev, NULL, link); 1313 peer_dev = kfd_topology_device_by_proximity_domain( 1314 link->node_to); 1315 1316 if (!peer_dev) 1317 continue; 1318 1319 list_for_each_entry(inbound_link, &peer_dev->p2p_link_props, 1320 list) { 1321 if (inbound_link->node_to != link->node_from) 1322 continue; 1323 1324 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1325 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1326 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1327 } 1328 } 1329 } 1330 1331 static int kfd_build_p2p_node_entry(struct kfd_topology_device *dev, 1332 struct kfd_iolink_properties *p2plink) 1333 { 1334 int ret; 1335 1336 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 1337 if (!p2plink->kobj) 1338 return -ENOMEM; 1339 1340 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 1341 dev->kobj_p2plink, "%d", dev->node_props.p2p_links_count - 1); 1342 if (ret < 0) { 1343 kobject_put(p2plink->kobj); 1344 return ret; 1345 } 1346 1347 p2plink->attr.name = "properties"; 1348 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 1349 sysfs_attr_init(&p2plink->attr); 1350 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 1351 if (ret < 0) 1352 return ret; 1353 1354 return 0; 1355 } 1356 1357 static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int gpu_node) 1358 { 1359 struct kfd_iolink_properties *gpu_link, *tmp_link, *cpu_link; 1360 struct kfd_iolink_properties *props = NULL, *props2 = NULL; 1361 struct kfd_topology_device *cpu_dev; 1362 int ret = 0; 1363 int i, num_cpu; 1364 1365 num_cpu = 0; 1366 list_for_each_entry(cpu_dev, &topology_device_list, list) { 1367 if (cpu_dev->gpu) 1368 break; 1369 num_cpu++; 1370 } 1371 1372 if (list_empty(&kdev->io_link_props)) 1373 return -ENODATA; 1374 1375 gpu_link = list_first_entry(&kdev->io_link_props, 1376 struct kfd_iolink_properties, list); 1377 1378 for (i = 0; i < num_cpu; i++) { 1379 /* CPU <--> GPU */ 1380 if (gpu_link->node_to == i) 1381 continue; 1382 1383 /* find CPU <--> CPU links */ 1384 cpu_link = NULL; 1385 cpu_dev = kfd_topology_device_by_proximity_domain(i); 1386 if (cpu_dev) { 1387 list_for_each_entry(tmp_link, 1388 &cpu_dev->io_link_props, list) { 1389 if (tmp_link->node_to == gpu_link->node_to) { 1390 cpu_link = tmp_link; 1391 break; 1392 } 1393 } 1394 } 1395 1396 if (!cpu_link) 1397 return -ENOMEM; 1398 1399 /* CPU <--> CPU <--> GPU, GPU node*/ 1400 props = kfd_alloc_struct(props); 1401 if (!props) 1402 return -ENOMEM; 1403 1404 memcpy(props, gpu_link, sizeof(struct kfd_iolink_properties)); 1405 props->weight = gpu_link->weight + cpu_link->weight; 1406 props->min_latency = gpu_link->min_latency + cpu_link->min_latency; 1407 props->max_latency = gpu_link->max_latency + cpu_link->max_latency; 1408 props->min_bandwidth = min(gpu_link->min_bandwidth, cpu_link->min_bandwidth); 1409 props->max_bandwidth = min(gpu_link->max_bandwidth, cpu_link->max_bandwidth); 1410 1411 props->node_from = gpu_node; 1412 props->node_to = i; 1413 kdev->node_props.p2p_links_count++; 1414 list_add_tail(&props->list, &kdev->p2p_link_props); 1415 ret = kfd_build_p2p_node_entry(kdev, props); 1416 if (ret < 0) 1417 return ret; 1418 1419 /* for small Bar, no CPU --> GPU in-direct links */ 1420 if (kfd_dev_is_large_bar(kdev->gpu)) { 1421 /* CPU <--> CPU <--> GPU, CPU node*/ 1422 props2 = kfd_alloc_struct(props2); 1423 if (!props2) 1424 return -ENOMEM; 1425 1426 memcpy(props2, props, sizeof(struct kfd_iolink_properties)); 1427 props2->node_from = i; 1428 props2->node_to = gpu_node; 1429 props2->kobj = NULL; 1430 cpu_dev->node_props.p2p_links_count++; 1431 list_add_tail(&props2->list, &cpu_dev->p2p_link_props); 1432 ret = kfd_build_p2p_node_entry(cpu_dev, props2); 1433 if (ret < 0) 1434 return ret; 1435 } 1436 } 1437 return ret; 1438 } 1439 1440 #if defined(CONFIG_HSA_AMD_P2P) 1441 static int kfd_add_peer_prop(struct kfd_topology_device *kdev, 1442 struct kfd_topology_device *peer, int from, int to) 1443 { 1444 struct kfd_iolink_properties *props = NULL; 1445 struct kfd_iolink_properties *iolink1, *iolink2, *iolink3; 1446 struct kfd_topology_device *cpu_dev; 1447 int ret = 0; 1448 1449 if (!amdgpu_device_is_peer_accessible( 1450 kdev->gpu->adev, 1451 peer->gpu->adev)) 1452 return ret; 1453 1454 if (list_empty(&kdev->io_link_props)) 1455 return -ENODATA; 1456 1457 iolink1 = list_first_entry(&kdev->io_link_props, 1458 struct kfd_iolink_properties, list); 1459 1460 if (list_empty(&peer->io_link_props)) 1461 return -ENODATA; 1462 1463 iolink2 = list_first_entry(&peer->io_link_props, 1464 struct kfd_iolink_properties, list); 1465 1466 props = kfd_alloc_struct(props); 1467 if (!props) 1468 return -ENOMEM; 1469 1470 memcpy(props, iolink1, sizeof(struct kfd_iolink_properties)); 1471 1472 props->weight = iolink1->weight + iolink2->weight; 1473 props->min_latency = iolink1->min_latency + iolink2->min_latency; 1474 props->max_latency = iolink1->max_latency + iolink2->max_latency; 1475 props->min_bandwidth = min(iolink1->min_bandwidth, iolink2->min_bandwidth); 1476 props->max_bandwidth = min(iolink2->max_bandwidth, iolink2->max_bandwidth); 1477 1478 if (iolink1->node_to != iolink2->node_to) { 1479 /* CPU->CPU link*/ 1480 cpu_dev = kfd_topology_device_by_proximity_domain(iolink1->node_to); 1481 if (cpu_dev) { 1482 list_for_each_entry(iolink3, &cpu_dev->io_link_props, list) { 1483 if (iolink3->node_to != iolink2->node_to) 1484 continue; 1485 1486 props->weight += iolink3->weight; 1487 props->min_latency += iolink3->min_latency; 1488 props->max_latency += iolink3->max_latency; 1489 props->min_bandwidth = min(props->min_bandwidth, 1490 iolink3->min_bandwidth); 1491 props->max_bandwidth = min(props->max_bandwidth, 1492 iolink3->max_bandwidth); 1493 break; 1494 } 1495 } else { 1496 WARN(1, "CPU node not found"); 1497 } 1498 } 1499 1500 props->node_from = from; 1501 props->node_to = to; 1502 peer->node_props.p2p_links_count++; 1503 list_add_tail(&props->list, &peer->p2p_link_props); 1504 ret = kfd_build_p2p_node_entry(peer, props); 1505 1506 return ret; 1507 } 1508 #endif 1509 1510 static int kfd_dev_create_p2p_links(void) 1511 { 1512 struct kfd_topology_device *dev; 1513 struct kfd_topology_device *new_dev; 1514 #if defined(CONFIG_HSA_AMD_P2P) 1515 uint32_t i; 1516 #endif 1517 uint32_t k; 1518 int ret = 0; 1519 1520 k = 0; 1521 list_for_each_entry(dev, &topology_device_list, list) 1522 k++; 1523 if (k < 2) 1524 return 0; 1525 1526 new_dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list); 1527 if (WARN_ON(!new_dev->gpu)) 1528 return 0; 1529 1530 k--; 1531 1532 /* create in-direct links */ 1533 ret = kfd_create_indirect_link_prop(new_dev, k); 1534 if (ret < 0) 1535 goto out; 1536 1537 /* create p2p links */ 1538 #if defined(CONFIG_HSA_AMD_P2P) 1539 i = 0; 1540 list_for_each_entry(dev, &topology_device_list, list) { 1541 if (dev == new_dev) 1542 break; 1543 if (!dev->gpu || !dev->gpu->adev || 1544 (dev->gpu->kfd->hive_id && 1545 dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id)) 1546 goto next; 1547 1548 /* check if node(s) is/are peer accessible in one direction or bi-direction */ 1549 ret = kfd_add_peer_prop(new_dev, dev, i, k); 1550 if (ret < 0) 1551 goto out; 1552 1553 ret = kfd_add_peer_prop(dev, new_dev, k, i); 1554 if (ret < 0) 1555 goto out; 1556 next: 1557 i++; 1558 } 1559 #endif 1560 1561 out: 1562 return ret; 1563 } 1564 1565 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1566 static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext, 1567 struct kfd_gpu_cache_info *pcache_info, 1568 int cu_bitmask, 1569 int cache_type, unsigned int cu_processor_id, 1570 int cu_block) 1571 { 1572 unsigned int cu_sibling_map_mask; 1573 int first_active_cu; 1574 struct kfd_cache_properties *pcache = NULL; 1575 1576 cu_sibling_map_mask = cu_bitmask; 1577 cu_sibling_map_mask >>= cu_block; 1578 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1579 first_active_cu = ffs(cu_sibling_map_mask); 1580 1581 /* CU could be inactive. In case of shared cache find the first active 1582 * CU. and incase of non-shared cache check if the CU is inactive. If 1583 * inactive active skip it 1584 */ 1585 if (first_active_cu) { 1586 pcache = kfd_alloc_struct(pcache); 1587 if (!pcache) 1588 return -ENOMEM; 1589 1590 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1591 pcache->processor_id_low = cu_processor_id + (first_active_cu - 1); 1592 pcache->cache_level = pcache_info[cache_type].cache_level; 1593 pcache->cache_size = pcache_info[cache_type].cache_size; 1594 pcache->cacheline_size = pcache_info[cache_type].cache_line_size; 1595 1596 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1597 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1598 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1599 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1600 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1601 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1602 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1603 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1604 1605 /* Sibling map is w.r.t processor_id_low, so shift out 1606 * inactive CU 1607 */ 1608 cu_sibling_map_mask = 1609 cu_sibling_map_mask >> (first_active_cu - 1); 1610 1611 pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1612 pcache->sibling_map[1] = 1613 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1614 pcache->sibling_map[2] = 1615 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1616 pcache->sibling_map[3] = 1617 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1618 1619 pcache->sibling_map_size = 4; 1620 *props_ext = pcache; 1621 1622 return 0; 1623 } 1624 return 1; 1625 } 1626 1627 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1628 static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, 1629 struct kfd_gpu_cache_info *pcache_info, 1630 struct amdgpu_cu_info *cu_info, 1631 struct amdgpu_gfx_config *gfx_info, 1632 int cache_type, unsigned int cu_processor_id, 1633 struct kfd_node *knode) 1634 { 1635 unsigned int cu_sibling_map_mask; 1636 int first_active_cu; 1637 int i, j, k, xcc, start, end; 1638 int num_xcc = NUM_XCC(knode->xcc_mask); 1639 struct kfd_cache_properties *pcache = NULL; 1640 enum amdgpu_memory_partition mode; 1641 struct amdgpu_device *adev = knode->adev; 1642 1643 start = ffs(knode->xcc_mask) - 1; 1644 end = start + num_xcc; 1645 cu_sibling_map_mask = cu_info->bitmap[start][0][0]; 1646 cu_sibling_map_mask &= 1647 ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1648 first_active_cu = ffs(cu_sibling_map_mask); 1649 1650 /* CU could be inactive. In case of shared cache find the first active 1651 * CU. and incase of non-shared cache check if the CU is inactive. If 1652 * inactive active skip it 1653 */ 1654 if (first_active_cu) { 1655 pcache = kfd_alloc_struct(pcache); 1656 if (!pcache) 1657 return -ENOMEM; 1658 1659 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1660 pcache->processor_id_low = cu_processor_id 1661 + (first_active_cu - 1); 1662 pcache->cache_level = pcache_info[cache_type].cache_level; 1663 pcache->cacheline_size = pcache_info[cache_type].cache_line_size; 1664 1665 if (KFD_GC_VERSION(knode) == IP_VERSION(9, 4, 3) || 1666 KFD_GC_VERSION(knode) == IP_VERSION(9, 4, 4)) 1667 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1668 else 1669 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1670 1671 pcache->cache_size = pcache_info[cache_type].cache_size; 1672 /* Partition mode only affects L3 cache size */ 1673 if (mode && pcache->cache_level == 3) 1674 pcache->cache_size /= mode; 1675 1676 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1677 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1678 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1679 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1680 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1681 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1682 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1683 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1684 1685 /* Sibling map is w.r.t processor_id_low, so shift out 1686 * inactive CU 1687 */ 1688 cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1); 1689 k = 0; 1690 1691 for (xcc = start; xcc < end; xcc++) { 1692 for (i = 0; i < gfx_info->max_shader_engines; i++) { 1693 for (j = 0; j < gfx_info->max_sh_per_se; j++) { 1694 pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1695 pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1696 pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1697 pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1698 k += 4; 1699 1700 cu_sibling_map_mask = cu_info->bitmap[xcc][i % 4][j + i / 4]; 1701 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1702 } 1703 } 1704 } 1705 pcache->sibling_map_size = k; 1706 *props_ext = pcache; 1707 return 0; 1708 } 1709 return 1; 1710 } 1711 1712 #define KFD_MAX_CACHE_TYPES 6 1713 1714 /* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info 1715 * tables 1716 */ 1717 static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev) 1718 { 1719 struct kfd_gpu_cache_info *pcache_info = NULL; 1720 int i, j, k, xcc, start, end; 1721 int ct = 0; 1722 unsigned int cu_processor_id; 1723 int ret; 1724 unsigned int num_cu_shared; 1725 struct amdgpu_cu_info *cu_info = &kdev->adev->gfx.cu_info; 1726 struct amdgpu_gfx_config *gfx_info = &kdev->adev->gfx.config; 1727 int gpu_processor_id; 1728 struct kfd_cache_properties *props_ext; 1729 int num_of_entries = 0; 1730 int num_of_cache_types = 0; 1731 struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; 1732 1733 1734 gpu_processor_id = dev->node_props.simd_id_base; 1735 1736 memset(cache_info, 0, sizeof(cache_info)); 1737 pcache_info = cache_info; 1738 num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info); 1739 if (!num_of_cache_types) { 1740 pr_warn("no cache info found\n"); 1741 return; 1742 } 1743 1744 /* For each type of cache listed in the kfd_gpu_cache_info table, 1745 * go through all available Compute Units. 1746 * The [i,j,k] loop will 1747 * if kfd_gpu_cache_info.num_cu_shared = 1 1748 * will parse through all available CU 1749 * If (kfd_gpu_cache_info.num_cu_shared != 1) 1750 * then it will consider only one CU from 1751 * the shared unit 1752 */ 1753 start = ffs(kdev->xcc_mask) - 1; 1754 end = start + NUM_XCC(kdev->xcc_mask); 1755 1756 for (ct = 0; ct < num_of_cache_types; ct++) { 1757 cu_processor_id = gpu_processor_id; 1758 if (pcache_info[ct].cache_level == 1) { 1759 for (xcc = start; xcc < end; xcc++) { 1760 for (i = 0; i < gfx_info->max_shader_engines; i++) { 1761 for (j = 0; j < gfx_info->max_sh_per_se; j++) { 1762 for (k = 0; k < gfx_info->max_cu_per_sh; k += pcache_info[ct].num_cu_shared) { 1763 1764 ret = fill_in_l1_pcache(&props_ext, pcache_info, 1765 cu_info->bitmap[xcc][i % 4][j + i / 4], ct, 1766 cu_processor_id, k); 1767 1768 if (ret < 0) 1769 break; 1770 1771 if (!ret) { 1772 num_of_entries++; 1773 list_add_tail(&props_ext->list, &dev->cache_props); 1774 } 1775 1776 /* Move to next CU block */ 1777 num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= 1778 gfx_info->max_cu_per_sh) ? 1779 pcache_info[ct].num_cu_shared : 1780 (gfx_info->max_cu_per_sh - k); 1781 cu_processor_id += num_cu_shared; 1782 } 1783 } 1784 } 1785 } 1786 } else { 1787 ret = fill_in_l2_l3_pcache(&props_ext, pcache_info, 1788 cu_info, gfx_info, ct, cu_processor_id, kdev); 1789 1790 if (ret < 0) 1791 break; 1792 1793 if (!ret) { 1794 num_of_entries++; 1795 list_add_tail(&props_ext->list, &dev->cache_props); 1796 } 1797 } 1798 } 1799 dev->node_props.caches_count += num_of_entries; 1800 pr_debug("Added [%d] GPU cache entries\n", num_of_entries); 1801 } 1802 1803 static int kfd_topology_add_device_locked(struct kfd_node *gpu, 1804 struct kfd_topology_device **dev) 1805 { 1806 int proximity_domain = ++topology_crat_proximity_domain; 1807 struct list_head temp_topology_device_list; 1808 void *crat_image = NULL; 1809 size_t image_size = 0; 1810 int res; 1811 1812 res = kfd_create_crat_image_virtual(&crat_image, &image_size, 1813 COMPUTE_UNIT_GPU, gpu, 1814 proximity_domain); 1815 if (res) { 1816 dev_err(gpu->adev->dev, "Error creating VCRAT\n"); 1817 topology_crat_proximity_domain--; 1818 goto err; 1819 } 1820 1821 INIT_LIST_HEAD(&temp_topology_device_list); 1822 1823 res = kfd_parse_crat_table(crat_image, 1824 &temp_topology_device_list, 1825 proximity_domain); 1826 if (res) { 1827 dev_err(gpu->adev->dev, "Error parsing VCRAT\n"); 1828 topology_crat_proximity_domain--; 1829 goto err; 1830 } 1831 1832 kfd_topology_update_device_list(&temp_topology_device_list, 1833 &topology_device_list); 1834 1835 *dev = kfd_assign_gpu(gpu); 1836 if (WARN_ON(!*dev)) { 1837 res = -ENODEV; 1838 goto err; 1839 } 1840 1841 /* Fill the cache affinity information here for the GPUs 1842 * using VCRAT 1843 */ 1844 kfd_fill_cache_non_crat_info(*dev, gpu); 1845 1846 /* Update the SYSFS tree, since we added another topology 1847 * device 1848 */ 1849 res = kfd_topology_update_sysfs(); 1850 if (!res) 1851 sys_props.generation_count++; 1852 else 1853 dev_err(gpu->adev->dev, "Failed to update GPU to sysfs topology. res=%d\n", 1854 res); 1855 1856 err: 1857 kfd_destroy_crat_image(crat_image); 1858 return res; 1859 } 1860 1861 static void kfd_topology_set_dbg_firmware_support(struct kfd_topology_device *dev) 1862 { 1863 bool firmware_supported = true; 1864 1865 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0) && 1866 KFD_GC_VERSION(dev->gpu) < IP_VERSION(12, 0, 0)) { 1867 uint32_t mes_api_rev = (dev->gpu->adev->mes.sched_version & 1868 AMDGPU_MES_API_VERSION_MASK) >> 1869 AMDGPU_MES_API_VERSION_SHIFT; 1870 uint32_t mes_rev = dev->gpu->adev->mes.sched_version & 1871 AMDGPU_MES_VERSION_MASK; 1872 1873 firmware_supported = (mes_api_rev >= 14) && (mes_rev >= 64); 1874 goto out; 1875 } 1876 1877 /* 1878 * Note: Any unlisted devices here are assumed to support exception handling. 1879 * Add additional checks here as needed. 1880 */ 1881 switch (KFD_GC_VERSION(dev->gpu)) { 1882 case IP_VERSION(9, 0, 1): 1883 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459 + 32768; 1884 break; 1885 case IP_VERSION(9, 1, 0): 1886 case IP_VERSION(9, 2, 1): 1887 case IP_VERSION(9, 2, 2): 1888 case IP_VERSION(9, 3, 0): 1889 case IP_VERSION(9, 4, 0): 1890 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459; 1891 break; 1892 case IP_VERSION(9, 4, 1): 1893 firmware_supported = dev->gpu->kfd->mec_fw_version >= 60; 1894 break; 1895 case IP_VERSION(9, 4, 2): 1896 firmware_supported = dev->gpu->kfd->mec_fw_version >= 51; 1897 break; 1898 case IP_VERSION(10, 1, 10): 1899 case IP_VERSION(10, 1, 2): 1900 case IP_VERSION(10, 1, 1): 1901 firmware_supported = dev->gpu->kfd->mec_fw_version >= 144; 1902 break; 1903 case IP_VERSION(10, 3, 0): 1904 case IP_VERSION(10, 3, 2): 1905 case IP_VERSION(10, 3, 1): 1906 case IP_VERSION(10, 3, 4): 1907 case IP_VERSION(10, 3, 5): 1908 firmware_supported = dev->gpu->kfd->mec_fw_version >= 89; 1909 break; 1910 case IP_VERSION(10, 1, 3): 1911 case IP_VERSION(10, 3, 3): 1912 firmware_supported = false; 1913 break; 1914 default: 1915 break; 1916 } 1917 1918 out: 1919 if (firmware_supported) 1920 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED; 1921 } 1922 1923 static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) 1924 { 1925 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 << 1926 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 1927 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 1928 1929 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_SUPPORT | 1930 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_TRAP_OVERRIDE_SUPPORTED | 1931 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_MODE_SUPPORTED; 1932 1933 if (kfd_dbg_has_ttmps_always_setup(dev->gpu)) 1934 dev->node_props.debug_prop |= HSA_DBG_DISPATCH_INFO_ALWAYS_VALID; 1935 1936 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0)) { 1937 if (KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 3) || 1938 KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 4)) 1939 dev->node_props.debug_prop |= 1940 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9_4_3 | 1941 HSA_DBG_WATCH_ADDR_MASK_HI_BIT_GFX9_4_3; 1942 else 1943 dev->node_props.debug_prop |= 1944 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9 | 1945 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1946 1947 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 2)) 1948 dev->node_props.capability |= 1949 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1950 } else { 1951 dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | 1952 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1953 1954 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0)) 1955 dev->node_props.capability |= 1956 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1957 1958 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0)) 1959 dev->node_props.capability |= 1960 HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED; 1961 } 1962 1963 kfd_topology_set_dbg_firmware_support(dev); 1964 } 1965 1966 int kfd_topology_add_device(struct kfd_node *gpu) 1967 { 1968 uint32_t gpu_id; 1969 struct kfd_topology_device *dev; 1970 int res = 0; 1971 int i; 1972 const char *asic_name = amdgpu_asic_name[gpu->adev->asic_type]; 1973 struct amdgpu_gfx_config *gfx_info = &gpu->adev->gfx.config; 1974 struct amdgpu_cu_info *cu_info = &gpu->adev->gfx.cu_info; 1975 1976 if (gpu->xcp && !gpu->xcp->ddev) { 1977 dev_warn(gpu->adev->dev, 1978 "Won't add GPU to topology since it has no drm node assigned."); 1979 return 0; 1980 } else { 1981 dev_dbg(gpu->adev->dev, "Adding new GPU to topology\n"); 1982 } 1983 1984 /* Check to see if this gpu device exists in the topology_device_list. 1985 * If so, assign the gpu to that device, 1986 * else create a Virtual CRAT for this gpu device and then parse that 1987 * CRAT to create a new topology device. Once created assign the gpu to 1988 * that topology device 1989 */ 1990 down_write(&topology_lock); 1991 dev = kfd_assign_gpu(gpu); 1992 if (!dev) 1993 res = kfd_topology_add_device_locked(gpu, &dev); 1994 up_write(&topology_lock); 1995 if (res) 1996 return res; 1997 1998 gpu_id = kfd_generate_gpu_id(gpu); 1999 dev->gpu_id = gpu_id; 2000 gpu->id = gpu_id; 2001 2002 kfd_dev_create_p2p_links(); 2003 2004 /* TODO: Move the following lines to function 2005 * kfd_add_non_crat_information 2006 */ 2007 2008 /* Fill-in additional information that is not available in CRAT but 2009 * needed for the topology 2010 */ 2011 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1; i++) { 2012 dev->node_props.name[i] = __tolower(asic_name[i]); 2013 if (asic_name[i] == '\0') 2014 break; 2015 } 2016 dev->node_props.name[i] = '\0'; 2017 2018 dev->node_props.simd_arrays_per_engine = 2019 gfx_info->max_sh_per_se; 2020 2021 dev->node_props.gfx_target_version = 2022 gpu->kfd->device_info.gfx_target_version; 2023 dev->node_props.vendor_id = gpu->adev->pdev->vendor; 2024 dev->node_props.device_id = gpu->adev->pdev->device; 2025 dev->node_props.capability |= 2026 ((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) & 2027 HSA_CAP_ASIC_REVISION_MASK); 2028 2029 dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); 2030 if (KFD_GC_VERSION(dev->gpu->kfd) == IP_VERSION(9, 4, 3)) 2031 dev->node_props.location_id |= dev->gpu->node_id; 2032 2033 dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); 2034 dev->node_props.max_engine_clk_fcompute = 2035 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev); 2036 dev->node_props.max_engine_clk_ccompute = 2037 cpufreq_quick_get_max(0) / 1000; 2038 2039 if (gpu->xcp) 2040 dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index; 2041 else 2042 dev->node_props.drm_render_minor = 2043 gpu->kfd->shared_resources.drm_render_minor; 2044 2045 dev->node_props.hive_id = gpu->kfd->hive_id; 2046 dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu); 2047 dev->node_props.num_sdma_xgmi_engines = 2048 kfd_get_num_xgmi_sdma_engines(gpu); 2049 dev->node_props.num_sdma_queues_per_engine = 2050 gpu->kfd->device_info.num_sdma_queues_per_engine - 2051 gpu->kfd->device_info.num_reserved_sdma_queues_per_engine; 2052 dev->node_props.num_gws = (dev->gpu->gws && 2053 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? 2054 dev->gpu->adev->gds.gws_size : 0; 2055 dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); 2056 2057 kfd_fill_mem_clk_max_info(dev); 2058 kfd_fill_iolink_non_crat_info(dev); 2059 2060 switch (dev->gpu->adev->asic_type) { 2061 case CHIP_KAVERI: 2062 case CHIP_HAWAII: 2063 case CHIP_TONGA: 2064 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 << 2065 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2066 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2067 break; 2068 case CHIP_CARRIZO: 2069 case CHIP_FIJI: 2070 case CHIP_POLARIS10: 2071 case CHIP_POLARIS11: 2072 case CHIP_POLARIS12: 2073 case CHIP_VEGAM: 2074 pr_debug("Adding doorbell packet type capability\n"); 2075 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 << 2076 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2077 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2078 break; 2079 default: 2080 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 0, 1)) 2081 WARN(1, "Unexpected ASIC family %u", 2082 dev->gpu->adev->asic_type); 2083 else 2084 kfd_topology_set_capabilities(dev); 2085 } 2086 2087 /* 2088 * Overwrite ATS capability according to needs_iommu_device to fix 2089 * potential missing corresponding bit in CRAT of BIOS. 2090 */ 2091 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT; 2092 2093 /* Fix errors in CZ CRAT. 2094 * simd_count: Carrizo CRAT reports wrong simd_count, probably 2095 * because it doesn't consider masked out CUs 2096 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd 2097 */ 2098 if (dev->gpu->adev->asic_type == CHIP_CARRIZO) { 2099 dev->node_props.simd_count = 2100 cu_info->simd_per_cu * cu_info->number; 2101 dev->node_props.max_waves_per_simd = 10; 2102 } 2103 2104 /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */ 2105 dev->node_props.capability |= 2106 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ? 2107 HSA_CAP_SRAM_EDCSUPPORTED : 0; 2108 dev->node_props.capability |= 2109 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? 2110 HSA_CAP_MEM_EDCSUPPORTED : 0; 2111 2112 if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1)) 2113 dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ? 2114 HSA_CAP_RASEVENTNOTIFY : 0; 2115 2116 if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev)) 2117 dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED; 2118 2119 if (dev->gpu->adev->gmc.is_app_apu || 2120 dev->gpu->adev->gmc.xgmi.connected_to_cpu) 2121 dev->node_props.capability |= HSA_CAP_FLAGS_COHERENTHOSTACCESS; 2122 2123 kfd_debug_print_topology(); 2124 2125 kfd_notify_gpu_change(gpu_id, 1); 2126 2127 return 0; 2128 } 2129 2130 /** 2131 * kfd_topology_update_io_links() - Update IO links after device removal. 2132 * @proximity_domain: Proximity domain value of the dev being removed. 2133 * 2134 * The topology list currently is arranged in increasing order of 2135 * proximity domain. 2136 * 2137 * Two things need to be done when a device is removed: 2138 * 1. All the IO links to this device need to be removed. 2139 * 2. All nodes after the current device node need to move 2140 * up once this device node is removed from the topology 2141 * list. As a result, the proximity domain values for 2142 * all nodes after the node being deleted reduce by 1. 2143 * This would also cause the proximity domain values for 2144 * io links to be updated based on new proximity domain 2145 * values. 2146 * 2147 * Context: The caller must hold write topology_lock. 2148 */ 2149 static void kfd_topology_update_io_links(int proximity_domain) 2150 { 2151 struct kfd_topology_device *dev; 2152 struct kfd_iolink_properties *iolink, *p2plink, *tmp; 2153 2154 list_for_each_entry(dev, &topology_device_list, list) { 2155 if (dev->proximity_domain > proximity_domain) 2156 dev->proximity_domain--; 2157 2158 list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) { 2159 /* 2160 * If there is an io link to the dev being deleted 2161 * then remove that IO link also. 2162 */ 2163 if (iolink->node_to == proximity_domain) { 2164 list_del(&iolink->list); 2165 dev->node_props.io_links_count--; 2166 } else { 2167 if (iolink->node_from > proximity_domain) 2168 iolink->node_from--; 2169 if (iolink->node_to > proximity_domain) 2170 iolink->node_to--; 2171 } 2172 } 2173 2174 list_for_each_entry_safe(p2plink, tmp, &dev->p2p_link_props, list) { 2175 /* 2176 * If there is a p2p link to the dev being deleted 2177 * then remove that p2p link also. 2178 */ 2179 if (p2plink->node_to == proximity_domain) { 2180 list_del(&p2plink->list); 2181 dev->node_props.p2p_links_count--; 2182 } else { 2183 if (p2plink->node_from > proximity_domain) 2184 p2plink->node_from--; 2185 if (p2plink->node_to > proximity_domain) 2186 p2plink->node_to--; 2187 } 2188 } 2189 } 2190 } 2191 2192 int kfd_topology_remove_device(struct kfd_node *gpu) 2193 { 2194 struct kfd_topology_device *dev, *tmp; 2195 uint32_t gpu_id; 2196 int res = -ENODEV; 2197 int i = 0; 2198 2199 down_write(&topology_lock); 2200 2201 list_for_each_entry_safe(dev, tmp, &topology_device_list, list) { 2202 if (dev->gpu == gpu) { 2203 gpu_id = dev->gpu_id; 2204 kfd_remove_sysfs_node_entry(dev); 2205 kfd_release_topology_device(dev); 2206 sys_props.num_devices--; 2207 kfd_topology_update_io_links(i); 2208 topology_crat_proximity_domain = sys_props.num_devices-1; 2209 sys_props.generation_count++; 2210 res = 0; 2211 if (kfd_topology_update_sysfs() < 0) 2212 kfd_topology_release_sysfs(); 2213 break; 2214 } 2215 i++; 2216 } 2217 2218 up_write(&topology_lock); 2219 2220 if (!res) 2221 kfd_notify_gpu_change(gpu_id, 0); 2222 2223 return res; 2224 } 2225 2226 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD 2227 * topology. If GPU device is found @idx, then valid kfd_dev pointer is 2228 * returned through @kdev 2229 * Return - 0: On success (@kdev will be NULL for non GPU nodes) 2230 * -1: If end of list 2231 */ 2232 int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev) 2233 { 2234 2235 struct kfd_topology_device *top_dev; 2236 uint8_t device_idx = 0; 2237 2238 *kdev = NULL; 2239 down_read(&topology_lock); 2240 2241 list_for_each_entry(top_dev, &topology_device_list, list) { 2242 if (device_idx == idx) { 2243 *kdev = top_dev->gpu; 2244 up_read(&topology_lock); 2245 return 0; 2246 } 2247 2248 device_idx++; 2249 } 2250 2251 up_read(&topology_lock); 2252 2253 return -1; 2254 2255 } 2256 2257 static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask) 2258 { 2259 int first_cpu_of_numa_node; 2260 2261 if (!cpumask || cpumask == cpu_none_mask) 2262 return -1; 2263 first_cpu_of_numa_node = cpumask_first(cpumask); 2264 if (first_cpu_of_numa_node >= nr_cpu_ids) 2265 return -1; 2266 #ifdef CONFIG_X86_64 2267 return cpu_data(first_cpu_of_numa_node).topo.apicid; 2268 #else 2269 return first_cpu_of_numa_node; 2270 #endif 2271 } 2272 2273 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor 2274 * of the given NUMA node (numa_node_id) 2275 * Return -1 on failure 2276 */ 2277 int kfd_numa_node_to_apic_id(int numa_node_id) 2278 { 2279 if (numa_node_id == -1) { 2280 pr_warn("Invalid NUMA Node. Use online CPU mask\n"); 2281 return kfd_cpumask_to_apic_id(cpu_online_mask); 2282 } 2283 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id)); 2284 } 2285 2286 #if defined(CONFIG_DEBUG_FS) 2287 2288 int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) 2289 { 2290 struct kfd_topology_device *dev; 2291 unsigned int i = 0; 2292 int r = 0; 2293 2294 down_read(&topology_lock); 2295 2296 list_for_each_entry(dev, &topology_device_list, list) { 2297 if (!dev->gpu) { 2298 i++; 2299 continue; 2300 } 2301 2302 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2303 r = dqm_debugfs_hqds(m, dev->gpu->dqm); 2304 if (r) 2305 break; 2306 } 2307 2308 up_read(&topology_lock); 2309 2310 return r; 2311 } 2312 2313 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) 2314 { 2315 struct kfd_topology_device *dev; 2316 unsigned int i = 0; 2317 int r = 0; 2318 2319 down_read(&topology_lock); 2320 2321 list_for_each_entry(dev, &topology_device_list, list) { 2322 if (!dev->gpu) { 2323 i++; 2324 continue; 2325 } 2326 2327 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2328 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr); 2329 if (r) 2330 break; 2331 } 2332 2333 up_read(&topology_lock); 2334 2335 return r; 2336 } 2337 2338 #endif 2339