1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019, Intel Corporation. 4 * 5 * Heterogeneous Memory Attributes Table (HMAT) representation 6 * 7 * This program parses and reports the platform's HMAT tables, and registers 8 * the applicable attributes with the node's interfaces. 9 */ 10 11 #define pr_fmt(fmt) "acpi/hmat: " fmt 12 13 #include <linux/acpi.h> 14 #include <linux/bitops.h> 15 #include <linux/device.h> 16 #include <linux/init.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/platform_device.h> 20 #include <linux/list_sort.h> 21 #include <linux/memregion.h> 22 #include <linux/memory.h> 23 #include <linux/mutex.h> 24 #include <linux/node.h> 25 #include <linux/sysfs.h> 26 #include <linux/dax.h> 27 #include <linux/memory-tiers.h> 28 29 static u8 hmat_revision; 30 static int hmat_disable __initdata; 31 32 void __init disable_hmat(void) 33 { 34 hmat_disable = 1; 35 } 36 37 static LIST_HEAD(targets); 38 static LIST_HEAD(initiators); 39 static LIST_HEAD(localities); 40 41 static DEFINE_MUTEX(target_lock); 42 43 /* 44 * The defined enum order is used to prioritize attributes to break ties when 45 * selecting the best performing node. 46 */ 47 enum locality_types { 48 WRITE_LATENCY, 49 READ_LATENCY, 50 WRITE_BANDWIDTH, 51 READ_BANDWIDTH, 52 }; 53 54 static struct memory_locality *localities_types[4]; 55 56 struct target_cache { 57 struct list_head node; 58 struct node_cache_attrs cache_attrs; 59 }; 60 61 struct memory_target { 62 struct list_head node; 63 unsigned int memory_pxm; 64 unsigned int processor_pxm; 65 struct resource memregions; 66 struct node_hmem_attrs hmem_attrs[2]; 67 struct list_head caches; 68 struct node_cache_attrs cache_attrs; 69 bool registered; 70 }; 71 72 struct memory_initiator { 73 struct list_head node; 74 unsigned int processor_pxm; 75 bool has_cpu; 76 }; 77 78 struct memory_locality { 79 struct list_head node; 80 struct acpi_hmat_locality *hmat_loc; 81 }; 82 83 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) 84 { 85 struct memory_initiator *initiator; 86 87 list_for_each_entry(initiator, &initiators, node) 88 if (initiator->processor_pxm == cpu_pxm) 89 return initiator; 90 return NULL; 91 } 92 93 static struct memory_target *find_mem_target(unsigned int mem_pxm) 94 { 95 struct memory_target *target; 96 97 list_for_each_entry(target, &targets, node) 98 if (target->memory_pxm == mem_pxm) 99 return target; 100 return NULL; 101 } 102 103 static __init void alloc_memory_initiator(unsigned int cpu_pxm) 104 { 105 struct memory_initiator *initiator; 106 107 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) 108 return; 109 110 initiator = find_mem_initiator(cpu_pxm); 111 if (initiator) 112 return; 113 114 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL); 115 if (!initiator) 116 return; 117 118 initiator->processor_pxm = cpu_pxm; 119 initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU); 120 list_add_tail(&initiator->node, &initiators); 121 } 122 123 static __init void alloc_memory_target(unsigned int mem_pxm, 124 resource_size_t start, resource_size_t len) 125 { 126 struct memory_target *target; 127 128 target = find_mem_target(mem_pxm); 129 if (!target) { 130 target = kzalloc(sizeof(*target), GFP_KERNEL); 131 if (!target) 132 return; 133 target->memory_pxm = mem_pxm; 134 target->processor_pxm = PXM_INVAL; 135 target->memregions = (struct resource) { 136 .name = "ACPI mem", 137 .start = 0, 138 .end = -1, 139 .flags = IORESOURCE_MEM, 140 }; 141 list_add_tail(&target->node, &targets); 142 INIT_LIST_HEAD(&target->caches); 143 } 144 145 /* 146 * There are potentially multiple ranges per PXM, so record each 147 * in the per-target memregions resource tree. 148 */ 149 if (!__request_region(&target->memregions, start, len, "memory target", 150 IORESOURCE_MEM)) 151 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n", 152 start, start + len, mem_pxm); 153 } 154 155 static __init const char *hmat_data_type(u8 type) 156 { 157 switch (type) { 158 case ACPI_HMAT_ACCESS_LATENCY: 159 return "Access Latency"; 160 case ACPI_HMAT_READ_LATENCY: 161 return "Read Latency"; 162 case ACPI_HMAT_WRITE_LATENCY: 163 return "Write Latency"; 164 case ACPI_HMAT_ACCESS_BANDWIDTH: 165 return "Access Bandwidth"; 166 case ACPI_HMAT_READ_BANDWIDTH: 167 return "Read Bandwidth"; 168 case ACPI_HMAT_WRITE_BANDWIDTH: 169 return "Write Bandwidth"; 170 default: 171 return "Reserved"; 172 } 173 } 174 175 static __init const char *hmat_data_type_suffix(u8 type) 176 { 177 switch (type) { 178 case ACPI_HMAT_ACCESS_LATENCY: 179 case ACPI_HMAT_READ_LATENCY: 180 case ACPI_HMAT_WRITE_LATENCY: 181 return " nsec"; 182 case ACPI_HMAT_ACCESS_BANDWIDTH: 183 case ACPI_HMAT_READ_BANDWIDTH: 184 case ACPI_HMAT_WRITE_BANDWIDTH: 185 return " MB/s"; 186 default: 187 return ""; 188 } 189 } 190 191 static u32 hmat_normalize(u16 entry, u64 base, u8 type) 192 { 193 u32 value; 194 195 /* 196 * Check for invalid and overflow values 197 */ 198 if (entry == 0xffff || !entry) 199 return 0; 200 else if (base > (UINT_MAX / (entry))) 201 return 0; 202 203 /* 204 * Divide by the base unit for version 1, convert latency from 205 * picosenonds to nanoseconds if revision 2. 206 */ 207 value = entry * base; 208 if (hmat_revision == 1) { 209 if (value < 10) 210 return 0; 211 value = DIV_ROUND_UP(value, 10); 212 } else if (hmat_revision == 2) { 213 switch (type) { 214 case ACPI_HMAT_ACCESS_LATENCY: 215 case ACPI_HMAT_READ_LATENCY: 216 case ACPI_HMAT_WRITE_LATENCY: 217 value = DIV_ROUND_UP(value, 1000); 218 break; 219 default: 220 break; 221 } 222 } 223 return value; 224 } 225 226 static void hmat_update_target_access(struct memory_target *target, 227 u8 type, u32 value, int access) 228 { 229 switch (type) { 230 case ACPI_HMAT_ACCESS_LATENCY: 231 target->hmem_attrs[access].read_latency = value; 232 target->hmem_attrs[access].write_latency = value; 233 break; 234 case ACPI_HMAT_READ_LATENCY: 235 target->hmem_attrs[access].read_latency = value; 236 break; 237 case ACPI_HMAT_WRITE_LATENCY: 238 target->hmem_attrs[access].write_latency = value; 239 break; 240 case ACPI_HMAT_ACCESS_BANDWIDTH: 241 target->hmem_attrs[access].read_bandwidth = value; 242 target->hmem_attrs[access].write_bandwidth = value; 243 break; 244 case ACPI_HMAT_READ_BANDWIDTH: 245 target->hmem_attrs[access].read_bandwidth = value; 246 break; 247 case ACPI_HMAT_WRITE_BANDWIDTH: 248 target->hmem_attrs[access].write_bandwidth = value; 249 break; 250 default: 251 break; 252 } 253 } 254 255 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) 256 { 257 struct memory_locality *loc; 258 259 loc = kzalloc(sizeof(*loc), GFP_KERNEL); 260 if (!loc) { 261 pr_notice_once("Failed to allocate HMAT locality\n"); 262 return; 263 } 264 265 loc->hmat_loc = hmat_loc; 266 list_add_tail(&loc->node, &localities); 267 268 switch (hmat_loc->data_type) { 269 case ACPI_HMAT_ACCESS_LATENCY: 270 localities_types[READ_LATENCY] = loc; 271 localities_types[WRITE_LATENCY] = loc; 272 break; 273 case ACPI_HMAT_READ_LATENCY: 274 localities_types[READ_LATENCY] = loc; 275 break; 276 case ACPI_HMAT_WRITE_LATENCY: 277 localities_types[WRITE_LATENCY] = loc; 278 break; 279 case ACPI_HMAT_ACCESS_BANDWIDTH: 280 localities_types[READ_BANDWIDTH] = loc; 281 localities_types[WRITE_BANDWIDTH] = loc; 282 break; 283 case ACPI_HMAT_READ_BANDWIDTH: 284 localities_types[READ_BANDWIDTH] = loc; 285 break; 286 case ACPI_HMAT_WRITE_BANDWIDTH: 287 localities_types[WRITE_BANDWIDTH] = loc; 288 break; 289 default: 290 break; 291 } 292 } 293 294 static __init int hmat_parse_locality(union acpi_subtable_headers *header, 295 const unsigned long end) 296 { 297 struct acpi_hmat_locality *hmat_loc = (void *)header; 298 struct memory_target *target; 299 unsigned int init, targ, total_size, ipds, tpds; 300 u32 *inits, *targs, value; 301 u16 *entries; 302 u8 type, mem_hier; 303 304 if (hmat_loc->header.length < sizeof(*hmat_loc)) { 305 pr_notice("Unexpected locality header length: %u\n", 306 hmat_loc->header.length); 307 return -EINVAL; 308 } 309 310 type = hmat_loc->data_type; 311 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; 312 ipds = hmat_loc->number_of_initiator_Pds; 313 tpds = hmat_loc->number_of_target_Pds; 314 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + 315 sizeof(*inits) * ipds + sizeof(*targs) * tpds; 316 if (hmat_loc->header.length < total_size) { 317 pr_notice("Unexpected locality header length:%u, minimum required:%u\n", 318 hmat_loc->header.length, total_size); 319 return -EINVAL; 320 } 321 322 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", 323 hmat_loc->flags, hmat_data_type(type), ipds, tpds, 324 hmat_loc->entry_base_unit); 325 326 inits = (u32 *)(hmat_loc + 1); 327 targs = inits + ipds; 328 entries = (u16 *)(targs + tpds); 329 for (init = 0; init < ipds; init++) { 330 alloc_memory_initiator(inits[init]); 331 for (targ = 0; targ < tpds; targ++) { 332 value = hmat_normalize(entries[init * tpds + targ], 333 hmat_loc->entry_base_unit, 334 type); 335 pr_info(" Initiator-Target[%u-%u]:%u%s\n", 336 inits[init], targs[targ], value, 337 hmat_data_type_suffix(type)); 338 339 if (mem_hier == ACPI_HMAT_MEMORY) { 340 target = find_mem_target(targs[targ]); 341 if (target && target->processor_pxm == inits[init]) { 342 hmat_update_target_access(target, type, value, 0); 343 /* If the node has a CPU, update access 1 */ 344 if (node_state(pxm_to_node(inits[init]), N_CPU)) 345 hmat_update_target_access(target, type, value, 1); 346 } 347 } 348 } 349 } 350 351 if (mem_hier == ACPI_HMAT_MEMORY) 352 hmat_add_locality(hmat_loc); 353 354 return 0; 355 } 356 357 static __init int hmat_parse_cache(union acpi_subtable_headers *header, 358 const unsigned long end) 359 { 360 struct acpi_hmat_cache *cache = (void *)header; 361 struct memory_target *target; 362 struct target_cache *tcache; 363 u32 attrs; 364 365 if (cache->header.length < sizeof(*cache)) { 366 pr_notice("Unexpected cache header length: %u\n", 367 cache->header.length); 368 return -EINVAL; 369 } 370 371 attrs = cache->cache_attributes; 372 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", 373 cache->memory_PD, cache->cache_size, attrs, 374 cache->number_of_SMBIOShandles); 375 376 target = find_mem_target(cache->memory_PD); 377 if (!target) 378 return 0; 379 380 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL); 381 if (!tcache) { 382 pr_notice_once("Failed to allocate HMAT cache info\n"); 383 return 0; 384 } 385 386 tcache->cache_attrs.size = cache->cache_size; 387 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; 388 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; 389 390 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { 391 case ACPI_HMAT_CA_DIRECT_MAPPED: 392 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; 393 break; 394 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: 395 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; 396 break; 397 case ACPI_HMAT_CA_NONE: 398 default: 399 tcache->cache_attrs.indexing = NODE_CACHE_OTHER; 400 break; 401 } 402 403 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { 404 case ACPI_HMAT_CP_WB: 405 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; 406 break; 407 case ACPI_HMAT_CP_WT: 408 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; 409 break; 410 case ACPI_HMAT_CP_NONE: 411 default: 412 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; 413 break; 414 } 415 list_add_tail(&tcache->node, &target->caches); 416 417 return 0; 418 } 419 420 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header, 421 const unsigned long end) 422 { 423 struct acpi_hmat_proximity_domain *p = (void *)header; 424 struct memory_target *target = NULL; 425 426 if (p->header.length != sizeof(*p)) { 427 pr_notice("Unexpected address range header length: %u\n", 428 p->header.length); 429 return -EINVAL; 430 } 431 432 if (hmat_revision == 1) 433 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", 434 p->reserved3, p->reserved4, p->flags, p->processor_PD, 435 p->memory_PD); 436 else 437 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", 438 p->flags, p->processor_PD, p->memory_PD); 439 440 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || 441 hmat_revision > 1) { 442 target = find_mem_target(p->memory_PD); 443 if (!target) { 444 pr_debug("Memory Domain missing from SRAT\n"); 445 return -EINVAL; 446 } 447 } 448 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { 449 int p_node = pxm_to_node(p->processor_PD); 450 451 if (p_node == NUMA_NO_NODE) { 452 pr_debug("Invalid Processor Domain\n"); 453 return -EINVAL; 454 } 455 target->processor_pxm = p->processor_PD; 456 } 457 458 return 0; 459 } 460 461 static int __init hmat_parse_subtable(union acpi_subtable_headers *header, 462 const unsigned long end) 463 { 464 struct acpi_hmat_structure *hdr = (void *)header; 465 466 if (!hdr) 467 return -EINVAL; 468 469 switch (hdr->type) { 470 case ACPI_HMAT_TYPE_PROXIMITY: 471 return hmat_parse_proximity_domain(header, end); 472 case ACPI_HMAT_TYPE_LOCALITY: 473 return hmat_parse_locality(header, end); 474 case ACPI_HMAT_TYPE_CACHE: 475 return hmat_parse_cache(header, end); 476 default: 477 return -EINVAL; 478 } 479 } 480 481 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header, 482 const unsigned long end) 483 { 484 struct acpi_srat_mem_affinity *ma = (void *)header; 485 486 if (!ma) 487 return -EINVAL; 488 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) 489 return 0; 490 alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length); 491 return 0; 492 } 493 494 static u32 hmat_initiator_perf(struct memory_target *target, 495 struct memory_initiator *initiator, 496 struct acpi_hmat_locality *hmat_loc) 497 { 498 unsigned int ipds, tpds, i, idx = 0, tdx = 0; 499 u32 *inits, *targs; 500 u16 *entries; 501 502 ipds = hmat_loc->number_of_initiator_Pds; 503 tpds = hmat_loc->number_of_target_Pds; 504 inits = (u32 *)(hmat_loc + 1); 505 targs = inits + ipds; 506 entries = (u16 *)(targs + tpds); 507 508 for (i = 0; i < ipds; i++) { 509 if (inits[i] == initiator->processor_pxm) { 510 idx = i; 511 break; 512 } 513 } 514 515 if (i == ipds) 516 return 0; 517 518 for (i = 0; i < tpds; i++) { 519 if (targs[i] == target->memory_pxm) { 520 tdx = i; 521 break; 522 } 523 } 524 if (i == tpds) 525 return 0; 526 527 return hmat_normalize(entries[idx * tpds + tdx], 528 hmat_loc->entry_base_unit, 529 hmat_loc->data_type); 530 } 531 532 static bool hmat_update_best(u8 type, u32 value, u32 *best) 533 { 534 bool updated = false; 535 536 if (!value) 537 return false; 538 539 switch (type) { 540 case ACPI_HMAT_ACCESS_LATENCY: 541 case ACPI_HMAT_READ_LATENCY: 542 case ACPI_HMAT_WRITE_LATENCY: 543 if (!*best || *best > value) { 544 *best = value; 545 updated = true; 546 } 547 break; 548 case ACPI_HMAT_ACCESS_BANDWIDTH: 549 case ACPI_HMAT_READ_BANDWIDTH: 550 case ACPI_HMAT_WRITE_BANDWIDTH: 551 if (!*best || *best < value) { 552 *best = value; 553 updated = true; 554 } 555 break; 556 } 557 558 return updated; 559 } 560 561 static int initiator_cmp(void *priv, const struct list_head *a, 562 const struct list_head *b) 563 { 564 struct memory_initiator *ia; 565 struct memory_initiator *ib; 566 567 ia = list_entry(a, struct memory_initiator, node); 568 ib = list_entry(b, struct memory_initiator, node); 569 570 return ia->processor_pxm - ib->processor_pxm; 571 } 572 573 static int initiators_to_nodemask(unsigned long *p_nodes) 574 { 575 struct memory_initiator *initiator; 576 577 if (list_empty(&initiators)) 578 return -ENXIO; 579 580 list_for_each_entry(initiator, &initiators, node) 581 set_bit(initiator->processor_pxm, p_nodes); 582 583 return 0; 584 } 585 586 static void hmat_update_target_attrs(struct memory_target *target, 587 unsigned long *p_nodes, int access) 588 { 589 struct memory_initiator *initiator; 590 unsigned int cpu_nid; 591 struct memory_locality *loc = NULL; 592 u32 best = 0; 593 int i; 594 595 bitmap_zero(p_nodes, MAX_NUMNODES); 596 /* 597 * If the Address Range Structure provides a local processor pxm, set 598 * only that one. Otherwise, find the best performance attributes and 599 * collect all initiators that match. 600 */ 601 if (target->processor_pxm != PXM_INVAL) { 602 cpu_nid = pxm_to_node(target->processor_pxm); 603 if (access == 0 || node_state(cpu_nid, N_CPU)) { 604 set_bit(target->processor_pxm, p_nodes); 605 return; 606 } 607 } 608 609 if (list_empty(&localities)) 610 return; 611 612 /* 613 * We need the initiator list sorted so we can use bitmap_clear for 614 * previously set initiators when we find a better memory accessor. 615 * We'll also use the sorting to prime the candidate nodes with known 616 * initiators. 617 */ 618 list_sort(NULL, &initiators, initiator_cmp); 619 if (initiators_to_nodemask(p_nodes) < 0) 620 return; 621 622 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { 623 loc = localities_types[i]; 624 if (!loc) 625 continue; 626 627 best = 0; 628 list_for_each_entry(initiator, &initiators, node) { 629 u32 value; 630 631 if (access == 1 && !initiator->has_cpu) { 632 clear_bit(initiator->processor_pxm, p_nodes); 633 continue; 634 } 635 if (!test_bit(initiator->processor_pxm, p_nodes)) 636 continue; 637 638 value = hmat_initiator_perf(target, initiator, loc->hmat_loc); 639 if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) 640 bitmap_clear(p_nodes, 0, initiator->processor_pxm); 641 if (value != best) 642 clear_bit(initiator->processor_pxm, p_nodes); 643 } 644 if (best) 645 hmat_update_target_access(target, loc->hmat_loc->data_type, best, access); 646 } 647 } 648 649 static void __hmat_register_target_initiators(struct memory_target *target, 650 unsigned long *p_nodes, 651 int access) 652 { 653 unsigned int mem_nid, cpu_nid; 654 int i; 655 656 mem_nid = pxm_to_node(target->memory_pxm); 657 hmat_update_target_attrs(target, p_nodes, access); 658 for_each_set_bit(i, p_nodes, MAX_NUMNODES) { 659 cpu_nid = pxm_to_node(i); 660 register_memory_node_under_compute_node(mem_nid, cpu_nid, access); 661 } 662 } 663 664 static void hmat_register_target_initiators(struct memory_target *target) 665 { 666 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 667 668 __hmat_register_target_initiators(target, p_nodes, 0); 669 __hmat_register_target_initiators(target, p_nodes, 1); 670 } 671 672 static void hmat_register_target_cache(struct memory_target *target) 673 { 674 unsigned mem_nid = pxm_to_node(target->memory_pxm); 675 struct target_cache *tcache; 676 677 list_for_each_entry(tcache, &target->caches, node) 678 node_add_cache(mem_nid, &tcache->cache_attrs); 679 } 680 681 static void hmat_register_target_perf(struct memory_target *target, int access) 682 { 683 unsigned mem_nid = pxm_to_node(target->memory_pxm); 684 node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access); 685 } 686 687 static void hmat_register_target_devices(struct memory_target *target) 688 { 689 struct resource *res; 690 691 /* 692 * Do not bother creating devices if no driver is available to 693 * consume them. 694 */ 695 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) 696 return; 697 698 for (res = target->memregions.child; res; res = res->sibling) { 699 int target_nid = pxm_to_node(target->memory_pxm); 700 701 hmem_register_resource(target_nid, res); 702 } 703 } 704 705 static void hmat_register_target(struct memory_target *target) 706 { 707 int nid = pxm_to_node(target->memory_pxm); 708 709 /* 710 * Devices may belong to either an offline or online 711 * node, so unconditionally add them. 712 */ 713 hmat_register_target_devices(target); 714 715 /* 716 * Skip offline nodes. This can happen when memory 717 * marked EFI_MEMORY_SP, "specific purpose", is applied 718 * to all the memory in a proximity domain leading to 719 * the node being marked offline / unplugged, or if 720 * memory-only "hotplug" node is offline. 721 */ 722 if (nid == NUMA_NO_NODE || !node_online(nid)) 723 return; 724 725 mutex_lock(&target_lock); 726 if (!target->registered) { 727 hmat_register_target_initiators(target); 728 hmat_register_target_cache(target); 729 hmat_register_target_perf(target, 0); 730 hmat_register_target_perf(target, 1); 731 target->registered = true; 732 } 733 mutex_unlock(&target_lock); 734 } 735 736 static void hmat_register_targets(void) 737 { 738 struct memory_target *target; 739 740 list_for_each_entry(target, &targets, node) 741 hmat_register_target(target); 742 } 743 744 static int hmat_callback(struct notifier_block *self, 745 unsigned long action, void *arg) 746 { 747 struct memory_target *target; 748 struct memory_notify *mnb = arg; 749 int pxm, nid = mnb->status_change_nid; 750 751 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 752 return NOTIFY_OK; 753 754 pxm = node_to_pxm(nid); 755 target = find_mem_target(pxm); 756 if (!target) 757 return NOTIFY_OK; 758 759 hmat_register_target(target); 760 return NOTIFY_OK; 761 } 762 763 static int hmat_set_default_dram_perf(void) 764 { 765 int rc; 766 int nid, pxm; 767 struct memory_target *target; 768 struct node_hmem_attrs *attrs; 769 770 if (!default_dram_type) 771 return -EIO; 772 773 for_each_node_mask(nid, default_dram_type->nodes) { 774 pxm = node_to_pxm(nid); 775 target = find_mem_target(pxm); 776 if (!target) 777 continue; 778 attrs = &target->hmem_attrs[1]; 779 rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT"); 780 if (rc) 781 return rc; 782 } 783 784 return 0; 785 } 786 787 static int hmat_calculate_adistance(struct notifier_block *self, 788 unsigned long nid, void *data) 789 { 790 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 791 struct memory_target *target; 792 struct node_hmem_attrs *perf; 793 int *adist = data; 794 int pxm; 795 796 pxm = node_to_pxm(nid); 797 target = find_mem_target(pxm); 798 if (!target) 799 return NOTIFY_OK; 800 801 mutex_lock(&target_lock); 802 hmat_update_target_attrs(target, p_nodes, 1); 803 mutex_unlock(&target_lock); 804 805 perf = &target->hmem_attrs[1]; 806 807 if (mt_perf_to_adistance(perf, adist)) 808 return NOTIFY_OK; 809 810 return NOTIFY_STOP; 811 } 812 813 static struct notifier_block hmat_adist_nb __meminitdata = { 814 .notifier_call = hmat_calculate_adistance, 815 .priority = 100, 816 }; 817 818 static __init void hmat_free_structures(void) 819 { 820 struct memory_target *target, *tnext; 821 struct memory_locality *loc, *lnext; 822 struct memory_initiator *initiator, *inext; 823 struct target_cache *tcache, *cnext; 824 825 list_for_each_entry_safe(target, tnext, &targets, node) { 826 struct resource *res, *res_next; 827 828 list_for_each_entry_safe(tcache, cnext, &target->caches, node) { 829 list_del(&tcache->node); 830 kfree(tcache); 831 } 832 833 list_del(&target->node); 834 res = target->memregions.child; 835 while (res) { 836 res_next = res->sibling; 837 __release_region(&target->memregions, res->start, 838 resource_size(res)); 839 res = res_next; 840 } 841 kfree(target); 842 } 843 844 list_for_each_entry_safe(initiator, inext, &initiators, node) { 845 list_del(&initiator->node); 846 kfree(initiator); 847 } 848 849 list_for_each_entry_safe(loc, lnext, &localities, node) { 850 list_del(&loc->node); 851 kfree(loc); 852 } 853 } 854 855 static __init int hmat_init(void) 856 { 857 struct acpi_table_header *tbl; 858 enum acpi_hmat_type i; 859 acpi_status status; 860 861 if (srat_disabled() || hmat_disable) 862 return 0; 863 864 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl); 865 if (ACPI_FAILURE(status)) 866 return 0; 867 868 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 869 sizeof(struct acpi_table_srat), 870 ACPI_SRAT_TYPE_MEMORY_AFFINITY, 871 srat_parse_mem_affinity, 0) < 0) 872 goto out_put; 873 acpi_put_table(tbl); 874 875 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl); 876 if (ACPI_FAILURE(status)) 877 goto out_put; 878 879 hmat_revision = tbl->revision; 880 switch (hmat_revision) { 881 case 1: 882 case 2: 883 break; 884 default: 885 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision); 886 goto out_put; 887 } 888 889 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { 890 if (acpi_table_parse_entries(ACPI_SIG_HMAT, 891 sizeof(struct acpi_table_hmat), i, 892 hmat_parse_subtable, 0) < 0) { 893 pr_notice("Ignoring: Invalid table"); 894 goto out_put; 895 } 896 } 897 hmat_register_targets(); 898 899 /* Keep the table and structures if the notifier may use them */ 900 if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) 901 goto out_put; 902 903 if (!hmat_set_default_dram_perf()) 904 register_mt_adistance_algorithm(&hmat_adist_nb); 905 906 return 0; 907 out_put: 908 hmat_free_structures(); 909 acpi_put_table(tbl); 910 return 0; 911 } 912 subsys_initcall(hmat_init); 913