1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019, Intel Corporation. 4 * 5 * Heterogeneous Memory Attributes Table (HMAT) representation 6 * 7 * This program parses and reports the platform's HMAT tables, and registers 8 * the applicable attributes with the node's interfaces. 9 */ 10 11 #define pr_fmt(fmt) "acpi/hmat: " fmt 12 13 #include <linux/acpi.h> 14 #include <linux/bitops.h> 15 #include <linux/device.h> 16 #include <linux/init.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/platform_device.h> 20 #include <linux/list_sort.h> 21 #include <linux/memregion.h> 22 #include <linux/memory.h> 23 #include <linux/mutex.h> 24 #include <linux/node.h> 25 #include <linux/sysfs.h> 26 #include <linux/dax.h> 27 #include <linux/memory-tiers.h> 28 29 static u8 hmat_revision; 30 static int hmat_disable __initdata; 31 32 void __init disable_hmat(void) 33 { 34 hmat_disable = 1; 35 } 36 37 static LIST_HEAD(targets); 38 static LIST_HEAD(initiators); 39 static LIST_HEAD(localities); 40 41 static DEFINE_MUTEX(target_lock); 42 43 /* 44 * The defined enum order is used to prioritize attributes to break ties when 45 * selecting the best performing node. 46 */ 47 enum locality_types { 48 WRITE_LATENCY, 49 READ_LATENCY, 50 WRITE_BANDWIDTH, 51 READ_BANDWIDTH, 52 }; 53 54 static struct memory_locality *localities_types[4]; 55 56 struct target_cache { 57 struct list_head node; 58 struct node_cache_attrs cache_attrs; 59 }; 60 61 enum { 62 NODE_ACCESS_CLASS_0 = 0, 63 NODE_ACCESS_CLASS_1, 64 NODE_ACCESS_CLASS_GENPORT_SINK, 65 NODE_ACCESS_CLASS_MAX, 66 }; 67 68 struct memory_target { 69 struct list_head node; 70 unsigned int memory_pxm; 71 unsigned int processor_pxm; 72 struct resource memregions; 73 struct access_coordinate coord[NODE_ACCESS_CLASS_MAX]; 74 struct list_head caches; 75 struct node_cache_attrs cache_attrs; 76 u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; 77 bool registered; 78 }; 79 80 struct memory_initiator { 81 struct list_head node; 82 unsigned int processor_pxm; 83 bool has_cpu; 84 }; 85 86 struct memory_locality { 87 struct list_head node; 88 struct acpi_hmat_locality *hmat_loc; 89 }; 90 91 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) 92 { 93 struct memory_initiator *initiator; 94 95 list_for_each_entry(initiator, &initiators, node) 96 if (initiator->processor_pxm == cpu_pxm) 97 return initiator; 98 return NULL; 99 } 100 101 static struct memory_target *find_mem_target(unsigned int mem_pxm) 102 { 103 struct memory_target *target; 104 105 list_for_each_entry(target, &targets, node) 106 if (target->memory_pxm == mem_pxm) 107 return target; 108 return NULL; 109 } 110 111 static struct memory_target *acpi_find_genport_target(u32 uid) 112 { 113 struct memory_target *target; 114 u32 target_uid; 115 u8 *uid_ptr; 116 117 list_for_each_entry(target, &targets, node) { 118 uid_ptr = target->gen_port_device_handle + 8; 119 target_uid = *(u32 *)uid_ptr; 120 if (uid == target_uid) 121 return target; 122 } 123 124 return NULL; 125 } 126 127 /** 128 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port 129 * @uid: ACPI unique id 130 * @coord: The access coordinates written back out for the generic port 131 * 132 * Return: 0 on success. Errno on failure. 133 * 134 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL. 135 */ 136 int acpi_get_genport_coordinates(u32 uid, 137 struct access_coordinate *coord) 138 { 139 struct memory_target *target; 140 141 guard(mutex)(&target_lock); 142 target = acpi_find_genport_target(uid); 143 if (!target) 144 return -ENOENT; 145 146 *coord = target->coord[NODE_ACCESS_CLASS_GENPORT_SINK]; 147 148 return 0; 149 } 150 EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL); 151 152 static __init void alloc_memory_initiator(unsigned int cpu_pxm) 153 { 154 struct memory_initiator *initiator; 155 156 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) 157 return; 158 159 initiator = find_mem_initiator(cpu_pxm); 160 if (initiator) 161 return; 162 163 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL); 164 if (!initiator) 165 return; 166 167 initiator->processor_pxm = cpu_pxm; 168 initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU); 169 list_add_tail(&initiator->node, &initiators); 170 } 171 172 static __init struct memory_target *alloc_target(unsigned int mem_pxm) 173 { 174 struct memory_target *target; 175 176 target = find_mem_target(mem_pxm); 177 if (!target) { 178 target = kzalloc(sizeof(*target), GFP_KERNEL); 179 if (!target) 180 return NULL; 181 target->memory_pxm = mem_pxm; 182 target->processor_pxm = PXM_INVAL; 183 target->memregions = (struct resource) { 184 .name = "ACPI mem", 185 .start = 0, 186 .end = -1, 187 .flags = IORESOURCE_MEM, 188 }; 189 list_add_tail(&target->node, &targets); 190 INIT_LIST_HEAD(&target->caches); 191 } 192 193 return target; 194 } 195 196 static __init void alloc_memory_target(unsigned int mem_pxm, 197 resource_size_t start, 198 resource_size_t len) 199 { 200 struct memory_target *target; 201 202 target = alloc_target(mem_pxm); 203 if (!target) 204 return; 205 206 /* 207 * There are potentially multiple ranges per PXM, so record each 208 * in the per-target memregions resource tree. 209 */ 210 if (!__request_region(&target->memregions, start, len, "memory target", 211 IORESOURCE_MEM)) 212 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n", 213 start, start + len, mem_pxm); 214 } 215 216 static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle) 217 { 218 struct memory_target *target; 219 220 target = alloc_target(mem_pxm); 221 if (!target) 222 return; 223 224 memcpy(target->gen_port_device_handle, handle, 225 ACPI_SRAT_DEVICE_HANDLE_SIZE); 226 } 227 228 static __init const char *hmat_data_type(u8 type) 229 { 230 switch (type) { 231 case ACPI_HMAT_ACCESS_LATENCY: 232 return "Access Latency"; 233 case ACPI_HMAT_READ_LATENCY: 234 return "Read Latency"; 235 case ACPI_HMAT_WRITE_LATENCY: 236 return "Write Latency"; 237 case ACPI_HMAT_ACCESS_BANDWIDTH: 238 return "Access Bandwidth"; 239 case ACPI_HMAT_READ_BANDWIDTH: 240 return "Read Bandwidth"; 241 case ACPI_HMAT_WRITE_BANDWIDTH: 242 return "Write Bandwidth"; 243 default: 244 return "Reserved"; 245 } 246 } 247 248 static __init const char *hmat_data_type_suffix(u8 type) 249 { 250 switch (type) { 251 case ACPI_HMAT_ACCESS_LATENCY: 252 case ACPI_HMAT_READ_LATENCY: 253 case ACPI_HMAT_WRITE_LATENCY: 254 return " nsec"; 255 case ACPI_HMAT_ACCESS_BANDWIDTH: 256 case ACPI_HMAT_READ_BANDWIDTH: 257 case ACPI_HMAT_WRITE_BANDWIDTH: 258 return " MB/s"; 259 default: 260 return ""; 261 } 262 } 263 264 static u32 hmat_normalize(u16 entry, u64 base, u8 type) 265 { 266 u32 value; 267 268 /* 269 * Check for invalid and overflow values 270 */ 271 if (entry == 0xffff || !entry) 272 return 0; 273 else if (base > (UINT_MAX / (entry))) 274 return 0; 275 276 /* 277 * Divide by the base unit for version 1, convert latency from 278 * picosenonds to nanoseconds if revision 2. 279 */ 280 value = entry * base; 281 if (hmat_revision == 1) { 282 if (value < 10) 283 return 0; 284 value = DIV_ROUND_UP(value, 10); 285 } else if (hmat_revision == 2) { 286 switch (type) { 287 case ACPI_HMAT_ACCESS_LATENCY: 288 case ACPI_HMAT_READ_LATENCY: 289 case ACPI_HMAT_WRITE_LATENCY: 290 value = DIV_ROUND_UP(value, 1000); 291 break; 292 default: 293 break; 294 } 295 } 296 return value; 297 } 298 299 static void hmat_update_target_access(struct memory_target *target, 300 u8 type, u32 value, int access) 301 { 302 switch (type) { 303 case ACPI_HMAT_ACCESS_LATENCY: 304 target->coord[access].read_latency = value; 305 target->coord[access].write_latency = value; 306 break; 307 case ACPI_HMAT_READ_LATENCY: 308 target->coord[access].read_latency = value; 309 break; 310 case ACPI_HMAT_WRITE_LATENCY: 311 target->coord[access].write_latency = value; 312 break; 313 case ACPI_HMAT_ACCESS_BANDWIDTH: 314 target->coord[access].read_bandwidth = value; 315 target->coord[access].write_bandwidth = value; 316 break; 317 case ACPI_HMAT_READ_BANDWIDTH: 318 target->coord[access].read_bandwidth = value; 319 break; 320 case ACPI_HMAT_WRITE_BANDWIDTH: 321 target->coord[access].write_bandwidth = value; 322 break; 323 default: 324 break; 325 } 326 } 327 328 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) 329 { 330 struct memory_locality *loc; 331 332 loc = kzalloc(sizeof(*loc), GFP_KERNEL); 333 if (!loc) { 334 pr_notice_once("Failed to allocate HMAT locality\n"); 335 return; 336 } 337 338 loc->hmat_loc = hmat_loc; 339 list_add_tail(&loc->node, &localities); 340 341 switch (hmat_loc->data_type) { 342 case ACPI_HMAT_ACCESS_LATENCY: 343 localities_types[READ_LATENCY] = loc; 344 localities_types[WRITE_LATENCY] = loc; 345 break; 346 case ACPI_HMAT_READ_LATENCY: 347 localities_types[READ_LATENCY] = loc; 348 break; 349 case ACPI_HMAT_WRITE_LATENCY: 350 localities_types[WRITE_LATENCY] = loc; 351 break; 352 case ACPI_HMAT_ACCESS_BANDWIDTH: 353 localities_types[READ_BANDWIDTH] = loc; 354 localities_types[WRITE_BANDWIDTH] = loc; 355 break; 356 case ACPI_HMAT_READ_BANDWIDTH: 357 localities_types[READ_BANDWIDTH] = loc; 358 break; 359 case ACPI_HMAT_WRITE_BANDWIDTH: 360 localities_types[WRITE_BANDWIDTH] = loc; 361 break; 362 default: 363 break; 364 } 365 } 366 367 static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm, 368 u8 mem_hier, u8 type, u32 value) 369 { 370 struct memory_target *target = find_mem_target(tgt_pxm); 371 372 if (mem_hier != ACPI_HMAT_MEMORY) 373 return; 374 375 if (target && target->processor_pxm == init_pxm) { 376 hmat_update_target_access(target, type, value, 377 NODE_ACCESS_CLASS_0); 378 /* If the node has a CPU, update access 1 */ 379 if (node_state(pxm_to_node(init_pxm), N_CPU)) 380 hmat_update_target_access(target, type, value, 381 NODE_ACCESS_CLASS_1); 382 } 383 } 384 385 static __init int hmat_parse_locality(union acpi_subtable_headers *header, 386 const unsigned long end) 387 { 388 struct acpi_hmat_locality *hmat_loc = (void *)header; 389 unsigned int init, targ, total_size, ipds, tpds; 390 u32 *inits, *targs, value; 391 u16 *entries; 392 u8 type, mem_hier; 393 394 if (hmat_loc->header.length < sizeof(*hmat_loc)) { 395 pr_notice("Unexpected locality header length: %u\n", 396 hmat_loc->header.length); 397 return -EINVAL; 398 } 399 400 type = hmat_loc->data_type; 401 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; 402 ipds = hmat_loc->number_of_initiator_Pds; 403 tpds = hmat_loc->number_of_target_Pds; 404 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + 405 sizeof(*inits) * ipds + sizeof(*targs) * tpds; 406 if (hmat_loc->header.length < total_size) { 407 pr_notice("Unexpected locality header length:%u, minimum required:%u\n", 408 hmat_loc->header.length, total_size); 409 return -EINVAL; 410 } 411 412 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", 413 hmat_loc->flags, hmat_data_type(type), ipds, tpds, 414 hmat_loc->entry_base_unit); 415 416 inits = (u32 *)(hmat_loc + 1); 417 targs = inits + ipds; 418 entries = (u16 *)(targs + tpds); 419 for (init = 0; init < ipds; init++) { 420 alloc_memory_initiator(inits[init]); 421 for (targ = 0; targ < tpds; targ++) { 422 value = hmat_normalize(entries[init * tpds + targ], 423 hmat_loc->entry_base_unit, 424 type); 425 pr_info(" Initiator-Target[%u-%u]:%u%s\n", 426 inits[init], targs[targ], value, 427 hmat_data_type_suffix(type)); 428 429 hmat_update_target(targs[targ], inits[init], 430 mem_hier, type, value); 431 } 432 } 433 434 if (mem_hier == ACPI_HMAT_MEMORY) 435 hmat_add_locality(hmat_loc); 436 437 return 0; 438 } 439 440 static __init int hmat_parse_cache(union acpi_subtable_headers *header, 441 const unsigned long end) 442 { 443 struct acpi_hmat_cache *cache = (void *)header; 444 struct memory_target *target; 445 struct target_cache *tcache; 446 u32 attrs; 447 448 if (cache->header.length < sizeof(*cache)) { 449 pr_notice("Unexpected cache header length: %u\n", 450 cache->header.length); 451 return -EINVAL; 452 } 453 454 attrs = cache->cache_attributes; 455 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", 456 cache->memory_PD, cache->cache_size, attrs, 457 cache->number_of_SMBIOShandles); 458 459 target = find_mem_target(cache->memory_PD); 460 if (!target) 461 return 0; 462 463 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL); 464 if (!tcache) { 465 pr_notice_once("Failed to allocate HMAT cache info\n"); 466 return 0; 467 } 468 469 tcache->cache_attrs.size = cache->cache_size; 470 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; 471 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; 472 473 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { 474 case ACPI_HMAT_CA_DIRECT_MAPPED: 475 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; 476 break; 477 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: 478 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; 479 break; 480 case ACPI_HMAT_CA_NONE: 481 default: 482 tcache->cache_attrs.indexing = NODE_CACHE_OTHER; 483 break; 484 } 485 486 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { 487 case ACPI_HMAT_CP_WB: 488 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; 489 break; 490 case ACPI_HMAT_CP_WT: 491 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; 492 break; 493 case ACPI_HMAT_CP_NONE: 494 default: 495 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; 496 break; 497 } 498 list_add_tail(&tcache->node, &target->caches); 499 500 return 0; 501 } 502 503 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header, 504 const unsigned long end) 505 { 506 struct acpi_hmat_proximity_domain *p = (void *)header; 507 struct memory_target *target = NULL; 508 509 if (p->header.length != sizeof(*p)) { 510 pr_notice("Unexpected address range header length: %u\n", 511 p->header.length); 512 return -EINVAL; 513 } 514 515 if (hmat_revision == 1) 516 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", 517 p->reserved3, p->reserved4, p->flags, p->processor_PD, 518 p->memory_PD); 519 else 520 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", 521 p->flags, p->processor_PD, p->memory_PD); 522 523 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || 524 hmat_revision > 1) { 525 target = find_mem_target(p->memory_PD); 526 if (!target) { 527 pr_debug("Memory Domain missing from SRAT\n"); 528 return -EINVAL; 529 } 530 } 531 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { 532 int p_node = pxm_to_node(p->processor_PD); 533 534 if (p_node == NUMA_NO_NODE) { 535 pr_debug("Invalid Processor Domain\n"); 536 return -EINVAL; 537 } 538 target->processor_pxm = p->processor_PD; 539 } 540 541 return 0; 542 } 543 544 static int __init hmat_parse_subtable(union acpi_subtable_headers *header, 545 const unsigned long end) 546 { 547 struct acpi_hmat_structure *hdr = (void *)header; 548 549 if (!hdr) 550 return -EINVAL; 551 552 switch (hdr->type) { 553 case ACPI_HMAT_TYPE_PROXIMITY: 554 return hmat_parse_proximity_domain(header, end); 555 case ACPI_HMAT_TYPE_LOCALITY: 556 return hmat_parse_locality(header, end); 557 case ACPI_HMAT_TYPE_CACHE: 558 return hmat_parse_cache(header, end); 559 default: 560 return -EINVAL; 561 } 562 } 563 564 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header, 565 const unsigned long end) 566 { 567 struct acpi_srat_mem_affinity *ma = (void *)header; 568 569 if (!ma) 570 return -EINVAL; 571 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) 572 return 0; 573 alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length); 574 return 0; 575 } 576 577 static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header, 578 const unsigned long end) 579 { 580 struct acpi_srat_generic_affinity *ga = (void *)header; 581 582 if (!ga) 583 return -EINVAL; 584 585 if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED)) 586 return 0; 587 588 /* Skip PCI device_handle for now */ 589 if (ga->device_handle_type != 0) 590 return 0; 591 592 alloc_genport_target(ga->proximity_domain, 593 (u8 *)ga->device_handle); 594 595 return 0; 596 } 597 598 static u32 hmat_initiator_perf(struct memory_target *target, 599 struct memory_initiator *initiator, 600 struct acpi_hmat_locality *hmat_loc) 601 { 602 unsigned int ipds, tpds, i, idx = 0, tdx = 0; 603 u32 *inits, *targs; 604 u16 *entries; 605 606 ipds = hmat_loc->number_of_initiator_Pds; 607 tpds = hmat_loc->number_of_target_Pds; 608 inits = (u32 *)(hmat_loc + 1); 609 targs = inits + ipds; 610 entries = (u16 *)(targs + tpds); 611 612 for (i = 0; i < ipds; i++) { 613 if (inits[i] == initiator->processor_pxm) { 614 idx = i; 615 break; 616 } 617 } 618 619 if (i == ipds) 620 return 0; 621 622 for (i = 0; i < tpds; i++) { 623 if (targs[i] == target->memory_pxm) { 624 tdx = i; 625 break; 626 } 627 } 628 if (i == tpds) 629 return 0; 630 631 return hmat_normalize(entries[idx * tpds + tdx], 632 hmat_loc->entry_base_unit, 633 hmat_loc->data_type); 634 } 635 636 static bool hmat_update_best(u8 type, u32 value, u32 *best) 637 { 638 bool updated = false; 639 640 if (!value) 641 return false; 642 643 switch (type) { 644 case ACPI_HMAT_ACCESS_LATENCY: 645 case ACPI_HMAT_READ_LATENCY: 646 case ACPI_HMAT_WRITE_LATENCY: 647 if (!*best || *best > value) { 648 *best = value; 649 updated = true; 650 } 651 break; 652 case ACPI_HMAT_ACCESS_BANDWIDTH: 653 case ACPI_HMAT_READ_BANDWIDTH: 654 case ACPI_HMAT_WRITE_BANDWIDTH: 655 if (!*best || *best < value) { 656 *best = value; 657 updated = true; 658 } 659 break; 660 } 661 662 return updated; 663 } 664 665 static int initiator_cmp(void *priv, const struct list_head *a, 666 const struct list_head *b) 667 { 668 struct memory_initiator *ia; 669 struct memory_initiator *ib; 670 671 ia = list_entry(a, struct memory_initiator, node); 672 ib = list_entry(b, struct memory_initiator, node); 673 674 return ia->processor_pxm - ib->processor_pxm; 675 } 676 677 static int initiators_to_nodemask(unsigned long *p_nodes) 678 { 679 struct memory_initiator *initiator; 680 681 if (list_empty(&initiators)) 682 return -ENXIO; 683 684 list_for_each_entry(initiator, &initiators, node) 685 set_bit(initiator->processor_pxm, p_nodes); 686 687 return 0; 688 } 689 690 static void hmat_update_target_attrs(struct memory_target *target, 691 unsigned long *p_nodes, int access) 692 { 693 struct memory_initiator *initiator; 694 unsigned int cpu_nid; 695 struct memory_locality *loc = NULL; 696 u32 best = 0; 697 int i; 698 699 /* Don't update for generic port if there's no device handle */ 700 if (access == NODE_ACCESS_CLASS_GENPORT_SINK && 701 !(*(u16 *)target->gen_port_device_handle)) 702 return; 703 704 bitmap_zero(p_nodes, MAX_NUMNODES); 705 /* 706 * If the Address Range Structure provides a local processor pxm, set 707 * only that one. Otherwise, find the best performance attributes and 708 * collect all initiators that match. 709 */ 710 if (target->processor_pxm != PXM_INVAL) { 711 cpu_nid = pxm_to_node(target->processor_pxm); 712 if (access == 0 || node_state(cpu_nid, N_CPU)) { 713 set_bit(target->processor_pxm, p_nodes); 714 return; 715 } 716 } 717 718 if (list_empty(&localities)) 719 return; 720 721 /* 722 * We need the initiator list sorted so we can use bitmap_clear for 723 * previously set initiators when we find a better memory accessor. 724 * We'll also use the sorting to prime the candidate nodes with known 725 * initiators. 726 */ 727 list_sort(NULL, &initiators, initiator_cmp); 728 if (initiators_to_nodemask(p_nodes) < 0) 729 return; 730 731 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { 732 loc = localities_types[i]; 733 if (!loc) 734 continue; 735 736 best = 0; 737 list_for_each_entry(initiator, &initiators, node) { 738 u32 value; 739 740 if (access == 1 && !initiator->has_cpu) { 741 clear_bit(initiator->processor_pxm, p_nodes); 742 continue; 743 } 744 if (!test_bit(initiator->processor_pxm, p_nodes)) 745 continue; 746 747 value = hmat_initiator_perf(target, initiator, loc->hmat_loc); 748 if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) 749 bitmap_clear(p_nodes, 0, initiator->processor_pxm); 750 if (value != best) 751 clear_bit(initiator->processor_pxm, p_nodes); 752 } 753 if (best) 754 hmat_update_target_access(target, loc->hmat_loc->data_type, best, access); 755 } 756 } 757 758 static void __hmat_register_target_initiators(struct memory_target *target, 759 unsigned long *p_nodes, 760 int access) 761 { 762 unsigned int mem_nid, cpu_nid; 763 int i; 764 765 mem_nid = pxm_to_node(target->memory_pxm); 766 hmat_update_target_attrs(target, p_nodes, access); 767 for_each_set_bit(i, p_nodes, MAX_NUMNODES) { 768 cpu_nid = pxm_to_node(i); 769 register_memory_node_under_compute_node(mem_nid, cpu_nid, access); 770 } 771 } 772 773 static void hmat_register_generic_target_initiators(struct memory_target *target) 774 { 775 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 776 777 __hmat_register_target_initiators(target, p_nodes, 778 NODE_ACCESS_CLASS_GENPORT_SINK); 779 } 780 781 static void hmat_register_target_initiators(struct memory_target *target) 782 { 783 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 784 785 __hmat_register_target_initiators(target, p_nodes, 0); 786 __hmat_register_target_initiators(target, p_nodes, 1); 787 } 788 789 static void hmat_register_target_cache(struct memory_target *target) 790 { 791 unsigned mem_nid = pxm_to_node(target->memory_pxm); 792 struct target_cache *tcache; 793 794 list_for_each_entry(tcache, &target->caches, node) 795 node_add_cache(mem_nid, &tcache->cache_attrs); 796 } 797 798 static void hmat_register_target_perf(struct memory_target *target, int access) 799 { 800 unsigned mem_nid = pxm_to_node(target->memory_pxm); 801 node_set_perf_attrs(mem_nid, &target->coord[access], access); 802 } 803 804 static void hmat_register_target_devices(struct memory_target *target) 805 { 806 struct resource *res; 807 808 /* 809 * Do not bother creating devices if no driver is available to 810 * consume them. 811 */ 812 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) 813 return; 814 815 for (res = target->memregions.child; res; res = res->sibling) { 816 int target_nid = pxm_to_node(target->memory_pxm); 817 818 hmem_register_resource(target_nid, res); 819 } 820 } 821 822 static void hmat_register_target(struct memory_target *target) 823 { 824 int nid = pxm_to_node(target->memory_pxm); 825 826 /* 827 * Devices may belong to either an offline or online 828 * node, so unconditionally add them. 829 */ 830 hmat_register_target_devices(target); 831 832 /* 833 * Register generic port perf numbers. The nid may not be 834 * initialized and is still NUMA_NO_NODE. 835 */ 836 mutex_lock(&target_lock); 837 if (*(u16 *)target->gen_port_device_handle) { 838 hmat_register_generic_target_initiators(target); 839 target->registered = true; 840 } 841 mutex_unlock(&target_lock); 842 843 /* 844 * Skip offline nodes. This can happen when memory 845 * marked EFI_MEMORY_SP, "specific purpose", is applied 846 * to all the memory in a proximity domain leading to 847 * the node being marked offline / unplugged, or if 848 * memory-only "hotplug" node is offline. 849 */ 850 if (nid == NUMA_NO_NODE || !node_online(nid)) 851 return; 852 853 mutex_lock(&target_lock); 854 if (!target->registered) { 855 hmat_register_target_initiators(target); 856 hmat_register_target_cache(target); 857 hmat_register_target_perf(target, NODE_ACCESS_CLASS_0); 858 hmat_register_target_perf(target, NODE_ACCESS_CLASS_1); 859 target->registered = true; 860 } 861 mutex_unlock(&target_lock); 862 } 863 864 static void hmat_register_targets(void) 865 { 866 struct memory_target *target; 867 868 list_for_each_entry(target, &targets, node) 869 hmat_register_target(target); 870 } 871 872 static int hmat_callback(struct notifier_block *self, 873 unsigned long action, void *arg) 874 { 875 struct memory_target *target; 876 struct memory_notify *mnb = arg; 877 int pxm, nid = mnb->status_change_nid; 878 879 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 880 return NOTIFY_OK; 881 882 pxm = node_to_pxm(nid); 883 target = find_mem_target(pxm); 884 if (!target) 885 return NOTIFY_OK; 886 887 hmat_register_target(target); 888 return NOTIFY_OK; 889 } 890 891 static int hmat_set_default_dram_perf(void) 892 { 893 int rc; 894 int nid, pxm; 895 struct memory_target *target; 896 struct access_coordinate *attrs; 897 898 if (!default_dram_type) 899 return -EIO; 900 901 for_each_node_mask(nid, default_dram_type->nodes) { 902 pxm = node_to_pxm(nid); 903 target = find_mem_target(pxm); 904 if (!target) 905 continue; 906 attrs = &target->coord[1]; 907 rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT"); 908 if (rc) 909 return rc; 910 } 911 912 return 0; 913 } 914 915 static int hmat_calculate_adistance(struct notifier_block *self, 916 unsigned long nid, void *data) 917 { 918 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 919 struct memory_target *target; 920 struct access_coordinate *perf; 921 int *adist = data; 922 int pxm; 923 924 pxm = node_to_pxm(nid); 925 target = find_mem_target(pxm); 926 if (!target) 927 return NOTIFY_OK; 928 929 mutex_lock(&target_lock); 930 hmat_update_target_attrs(target, p_nodes, 1); 931 mutex_unlock(&target_lock); 932 933 perf = &target->coord[1]; 934 935 if (mt_perf_to_adistance(perf, adist)) 936 return NOTIFY_OK; 937 938 return NOTIFY_STOP; 939 } 940 941 static struct notifier_block hmat_adist_nb __meminitdata = { 942 .notifier_call = hmat_calculate_adistance, 943 .priority = 100, 944 }; 945 946 static __init void hmat_free_structures(void) 947 { 948 struct memory_target *target, *tnext; 949 struct memory_locality *loc, *lnext; 950 struct memory_initiator *initiator, *inext; 951 struct target_cache *tcache, *cnext; 952 953 list_for_each_entry_safe(target, tnext, &targets, node) { 954 struct resource *res, *res_next; 955 956 list_for_each_entry_safe(tcache, cnext, &target->caches, node) { 957 list_del(&tcache->node); 958 kfree(tcache); 959 } 960 961 list_del(&target->node); 962 res = target->memregions.child; 963 while (res) { 964 res_next = res->sibling; 965 __release_region(&target->memregions, res->start, 966 resource_size(res)); 967 res = res_next; 968 } 969 kfree(target); 970 } 971 972 list_for_each_entry_safe(initiator, inext, &initiators, node) { 973 list_del(&initiator->node); 974 kfree(initiator); 975 } 976 977 list_for_each_entry_safe(loc, lnext, &localities, node) { 978 list_del(&loc->node); 979 kfree(loc); 980 } 981 } 982 983 static __init int hmat_init(void) 984 { 985 struct acpi_table_header *tbl; 986 enum acpi_hmat_type i; 987 acpi_status status; 988 989 if (srat_disabled() || hmat_disable) 990 return 0; 991 992 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl); 993 if (ACPI_FAILURE(status)) 994 return 0; 995 996 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 997 sizeof(struct acpi_table_srat), 998 ACPI_SRAT_TYPE_MEMORY_AFFINITY, 999 srat_parse_mem_affinity, 0) < 0) 1000 goto out_put; 1001 1002 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 1003 sizeof(struct acpi_table_srat), 1004 ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY, 1005 srat_parse_genport_affinity, 0) < 0) 1006 goto out_put; 1007 1008 acpi_put_table(tbl); 1009 1010 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl); 1011 if (ACPI_FAILURE(status)) 1012 goto out_put; 1013 1014 hmat_revision = tbl->revision; 1015 switch (hmat_revision) { 1016 case 1: 1017 case 2: 1018 break; 1019 default: 1020 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision); 1021 goto out_put; 1022 } 1023 1024 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { 1025 if (acpi_table_parse_entries(ACPI_SIG_HMAT, 1026 sizeof(struct acpi_table_hmat), i, 1027 hmat_parse_subtable, 0) < 0) { 1028 pr_notice("Ignoring: Invalid table"); 1029 goto out_put; 1030 } 1031 } 1032 hmat_register_targets(); 1033 1034 /* Keep the table and structures if the notifier may use them */ 1035 if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) 1036 goto out_put; 1037 1038 if (!hmat_set_default_dram_perf()) 1039 register_mt_adistance_algorithm(&hmat_adist_nb); 1040 1041 return 0; 1042 out_put: 1043 hmat_free_structures(); 1044 acpi_put_table(tbl); 1045 return 0; 1046 } 1047 subsys_initcall(hmat_init); 1048