1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS 29 * ================================================================ 30 * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access 31 * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of 32 * one or more CPUs and some local memory. The CPUs in each node can access 33 * the memory in the other nodes but at a higher latency than accessing their 34 * local memory. Typically, a system with only one node has Uniform Memory 35 * Access (UMA), but it may be possible to have a one node system that has 36 * some global memory outside of the node which is higher latency. 37 * 38 * Module Description 39 * ------------------ 40 * This module provides a platform interface for determining which CPUs and 41 * which memory (and how much) are in a NUMA node and how far each node is from 42 * each other. The interface is used by the Virtual Memory (VM) system and the 43 * common lgroup framework. The VM system uses the plat_*() routines to fill 44 * in its memory node (memnode) array with the physical address range spanned 45 * by each NUMA node to know which memory belongs to which node, so it can 46 * build and manage a physical page free list for each NUMA node and allocate 47 * local memory from each node as needed. The common lgroup framework uses the 48 * exported lgrp_plat_*() routines to figure out which CPUs and memory belong 49 * to each node (leaf lgroup) and how far each node is from each other, so it 50 * can build the latency (lgroup) topology for the machine in order to optimize 51 * for locality. Also, an lgroup platform handle instead of lgroups are used 52 * in the interface with this module, so this module shouldn't need to know 53 * anything about lgroups. Instead, it just needs to know which CPUs, memory, 54 * etc. are in each NUMA node, how far each node is from each other, and to use 55 * a unique lgroup platform handle to refer to each node through the interface. 56 * 57 * Determining NUMA Configuration 58 * ------------------------------ 59 * By default, this module will try to determine the NUMA configuration of the 60 * machine by reading the ACPI System Resource Affinity Table (SRAT) and System 61 * Locality Information Table (SLIT). The SRAT contains info to tell which 62 * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT 63 * is a matrix that gives the distance between each system locality (which is 64 * a NUMA node and should correspond to proximity domains in the SRAT). For 65 * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer 66 * specification. 67 * 68 * If the SRAT doesn't exist on a system with AMD Opteron processors, we 69 * examine registers in PCI configuration space to determine how many nodes are 70 * in the system and which CPUs and memory are in each node. 71 * do while booting the kernel. 72 * 73 * NOTE: Using these PCI configuration space registers to determine this 74 * locality info is not guaranteed to work or be compatible across all 75 * Opteron processor families. 76 * 77 * If the SLIT does not exist or look right, the kernel will probe to determine 78 * the distance between nodes as long as the NUMA CPU and memory configuration 79 * has been determined (see lgrp_plat_probe() for details). 80 * 81 * Data Structures 82 * --------------- 83 * The main data structures used by this code are the following: 84 * 85 * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by 86 * CPU ID (only used for SRAT) 87 * 88 * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and 89 * different nodes indexed by node ID 90 * 91 * - lgrp_plat_node_cnt Number of NUMA nodes in system 92 * 93 * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping 94 * table indexed by node ID (only used 95 * for SRAT) 96 * 97 * - lgrp_plat_node_memory[] Table with physical address range for 98 * each node indexed by node ID 99 * 100 * The code is implemented to make the following always be true: 101 * 102 * lgroup platform handle == node ID == memnode ID 103 * 104 * Moreover, it allows for the proximity domain ID to be equal to all of the 105 * above as long as the proximity domains IDs are numbered from 0 to <number of 106 * nodes - 1>. This is done by hashing each proximity domain ID into the range 107 * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID 108 * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] 109 * and be assigned node ID N. If the proximity domain IDs aren't numbered 110 * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into 111 * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs 112 * to node IDs. However, the proximity domain IDs may not map to the 113 * equivalent node ID since we want to keep the node IDs numbered from 0 to 114 * <number of nodes - 1> to minimize cost of searching and potentially space. 115 */ 116 117 118 #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */ 119 #include <sys/bootconf.h> 120 #include <sys/cmn_err.h> 121 #include <sys/controlregs.h> 122 #include <sys/cpupart.h> 123 #include <sys/cpuvar.h> 124 #include <sys/lgrp.h> 125 #include <sys/machsystm.h> 126 #include <sys/memlist.h> 127 #include <sys/memnode.h> 128 #include <sys/mman.h> 129 #include <sys/pci_cfgspace.h> 130 #include <sys/pci_impl.h> 131 #include <sys/param.h> 132 #include <sys/pghw.h> 133 #include <sys/promif.h> /* for prom_printf() */ 134 #include <sys/sysmacros.h> 135 #include <sys/systm.h> 136 #include <sys/thread.h> 137 #include <sys/types.h> 138 #include <sys/var.h> 139 #include <sys/x86_archext.h> /* for x86_feature and X86_AMD */ 140 #include <vm/hat_i86.h> 141 #include <vm/seg_kmem.h> 142 #include <vm/vm_dep.h> 143 144 #include "acpi_fw.h" /* for SRAT and SLIT */ 145 146 147 #define MAX_NODES 8 148 #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) 149 150 /* 151 * Constants for configuring probing 152 */ 153 #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ 154 #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ 155 #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ 156 157 /* 158 * Flags for probing 159 */ 160 #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ 161 #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ 162 #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ 163 164 /* 165 * Hash proximity domain ID into node to domain mapping table using to minimize 166 * span of entries used 167 */ 168 #define NODE_DOMAIN_HASH(domain, node_cnt) ((domain) % node_cnt) 169 170 171 /* 172 * CPU to node ID mapping structure (only used with SRAT) 173 */ 174 typedef struct cpu_node_map { 175 int exists; 176 uint_t node; 177 uint32_t apicid; 178 uint32_t prox_domain; 179 } cpu_node_map_t; 180 181 /* 182 * Latency statistics 183 */ 184 typedef struct lgrp_plat_latency_stats { 185 hrtime_t latencies[MAX_NODES][MAX_NODES]; 186 hrtime_t latency_max; 187 hrtime_t latency_min; 188 } lgrp_plat_latency_stats_t; 189 190 /* 191 * Memory configuration for probing 192 */ 193 typedef struct lgrp_plat_probe_mem_config { 194 size_t probe_memsize; /* how much memory to probe per node */ 195 caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ 196 pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ 197 } lgrp_plat_probe_mem_config_t; 198 199 /* 200 * Statistics kept for probing 201 */ 202 typedef struct lgrp_plat_probe_stats { 203 hrtime_t flush_cost; 204 hrtime_t probe_cost; 205 hrtime_t probe_cost_total; 206 hrtime_t probe_error_code; 207 hrtime_t probe_errors[MAX_NODES][MAX_NODES]; 208 int probe_suspect[MAX_NODES][MAX_NODES]; 209 hrtime_t probe_max[MAX_NODES][MAX_NODES]; 210 hrtime_t probe_min[MAX_NODES][MAX_NODES]; 211 } lgrp_plat_probe_stats_t; 212 213 /* 214 * Node to proximity domain ID mapping structure (only used with SRAT) 215 */ 216 typedef struct node_domain_map { 217 int exists; 218 uint32_t prox_domain; 219 } node_domain_map_t; 220 221 /* 222 * Node ID and starting and ending page for physical memory in node 223 */ 224 typedef struct node_phys_addr_map { 225 pfn_t start; 226 pfn_t end; 227 int exists; 228 uint32_t prox_domain; 229 } node_phys_addr_map_t; 230 231 /* 232 * Number of CPUs for which we got APIC IDs 233 */ 234 static int lgrp_plat_apic_ncpus = 0; 235 236 /* 237 * CPU to node ID mapping table (only used for SRAT) 238 */ 239 static cpu_node_map_t lgrp_plat_cpu_node[NCPU]; 240 241 /* 242 * Latency statistics 243 */ 244 lgrp_plat_latency_stats_t lgrp_plat_lat_stats; 245 246 /* 247 * Whether memory is interleaved across nodes causing MPO to be disabled 248 */ 249 static int lgrp_plat_mem_intrlv = 0; 250 251 /* 252 * Node ID to proximity domain ID mapping table (only used for SRAT) 253 */ 254 static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; 255 256 /* 257 * Physical address range for memory in each node 258 */ 259 static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES]; 260 261 /* 262 * Statistics gotten from probing 263 */ 264 static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; 265 266 /* 267 * Memory configuration for probing 268 */ 269 static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; 270 271 /* 272 * Error code from processing ACPI SRAT 273 */ 274 static int lgrp_plat_srat_error = 0; 275 276 /* 277 * Error code from processing ACPI SLIT 278 */ 279 static int lgrp_plat_slit_error = 0; 280 281 /* 282 * Allocate lgroup array statically 283 */ 284 static lgrp_t lgrp_space[NLGRP]; 285 static int nlgrps_alloc; 286 287 288 /* 289 * Number of nodes in system 290 */ 291 uint_t lgrp_plat_node_cnt = 1; 292 293 /* 294 * Configuration Parameters for Probing 295 * - lgrp_plat_probe_flags Flags to specify enabling probing, probe 296 * operation, etc. 297 * - lgrp_plat_probe_nrounds How many rounds of probing to do 298 * - lgrp_plat_probe_nsamples Number of samples to take when probing each 299 * node 300 * - lgrp_plat_probe_nreads Number of times to read vendor ID from 301 * Northbridge for each probe 302 */ 303 uint_t lgrp_plat_probe_flags = 0; 304 int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; 305 int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; 306 int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; 307 308 /* 309 * Enable use of ACPI System Resource Affinity Table (SRAT) and System 310 * Locality Information Table (SLIT) 311 */ 312 int lgrp_plat_srat_enable = 1; 313 int lgrp_plat_slit_enable = 1; 314 315 /* 316 * Static array to hold lgroup statistics 317 */ 318 struct lgrp_stats lgrp_stats[NLGRP]; 319 320 321 /* 322 * Forward declarations of platform interface routines 323 */ 324 void plat_build_mem_nodes(struct memlist *list); 325 326 int plat_lgrphand_to_mem_node(lgrp_handle_t hand); 327 328 lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); 329 330 int plat_mnode_xcheck(pfn_t pfncnt); 331 332 int plat_pfn_to_mem_node(pfn_t pfn); 333 334 /* 335 * Forward declarations of lgroup platform interface routines 336 */ 337 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 338 339 void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); 340 341 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); 342 343 void lgrp_plat_init(void); 344 345 int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); 346 347 void lgrp_plat_main_init(void); 348 349 int lgrp_plat_max_lgrps(void); 350 351 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, 352 lgrp_mem_query_t query); 353 354 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); 355 356 void lgrp_plat_probe(void); 357 358 lgrp_handle_t lgrp_plat_root_hand(void); 359 360 361 /* 362 * Forward declarations of local routines 363 */ 364 static int is_opteron(void); 365 366 static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, 367 int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, 368 uint32_t domain); 369 370 static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node); 371 372 static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, 373 int node_cnt, uint32_t domain); 374 375 static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 376 lgrp_plat_latency_stats_t *lat_stats, 377 lgrp_plat_probe_stats_t *probe_stats); 378 379 static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 380 lgrp_plat_latency_stats_t *lat_stats); 381 382 static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); 383 384 static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, 385 int node_cnt, uint32_t domain); 386 387 static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain, 388 int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start, 389 uint64_t end, uint32_t domain); 390 391 static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 392 lgrp_plat_probe_mem_config_t *probe_mem_config, 393 lgrp_plat_latency_stats_t *lat_stats, 394 lgrp_plat_probe_stats_t *probe_stats); 395 396 static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); 397 398 static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 399 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats); 400 401 static int lgrp_plat_process_srat(struct srat *tp, 402 node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, 403 node_phys_addr_map_t *node_memory); 404 405 static int lgrp_plat_srat_domains(struct srat *tp); 406 407 static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 408 lgrp_plat_latency_stats_t *lat_stats); 409 410 static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 411 node_phys_addr_map_t *node_memory); 412 413 static hrtime_t opt_probe_vendor(int dest_node, int nreads); 414 415 416 /* 417 * PLATFORM INTERFACE ROUTINES 418 */ 419 420 /* 421 * Configure memory nodes for machines with more than one node (ie NUMA) 422 */ 423 void 424 plat_build_mem_nodes(struct memlist *list) 425 { 426 pfn_t cur_start; /* start addr of subrange */ 427 pfn_t cur_end; /* end addr of subrange */ 428 pfn_t start; /* start addr of whole range */ 429 pfn_t end; /* end addr of whole range */ 430 431 /* 432 * Boot install lists are arranged <addr, len>, ... 433 */ 434 while (list) { 435 int node; 436 437 start = list->address >> PAGESHIFT; 438 end = (list->address + list->size - 1) >> PAGESHIFT; 439 440 if (start > physmax) { 441 list = list->next; 442 continue; 443 } 444 if (end > physmax) 445 end = physmax; 446 447 /* 448 * When there is only one memnode, just add memory to memnode 449 */ 450 if (max_mem_nodes == 1) { 451 mem_node_add_slice(start, end); 452 list = list->next; 453 continue; 454 } 455 456 /* 457 * mem_node_add_slice() expects to get a memory range that 458 * is within one memnode, so need to split any memory range 459 * that spans multiple memnodes into subranges that are each 460 * contained within one memnode when feeding them to 461 * mem_node_add_slice() 462 */ 463 cur_start = start; 464 do { 465 node = plat_pfn_to_mem_node(cur_start); 466 467 /* 468 * Panic if DRAM address map registers or SRAT say 469 * memory in node doesn't exist or address from 470 * boot installed memory list entry isn't in this node. 471 * This shouldn't happen and rest of code can't deal 472 * with this if it does. 473 */ 474 if (node < 0 || node >= lgrp_plat_node_cnt || 475 !lgrp_plat_node_memory[node].exists || 476 cur_start < lgrp_plat_node_memory[node].start || 477 cur_start > lgrp_plat_node_memory[node].end) { 478 cmn_err(CE_PANIC, "Don't know which memnode " 479 "to add installed memory address 0x%lx\n", 480 cur_start); 481 } 482 483 /* 484 * End of current subrange should not span memnodes 485 */ 486 cur_end = end; 487 if (lgrp_plat_node_memory[node].exists && 488 cur_end > lgrp_plat_node_memory[node].end) 489 cur_end = lgrp_plat_node_memory[node].end; 490 491 mem_node_add_slice(cur_start, cur_end); 492 493 /* 494 * Next subrange starts after end of current one 495 */ 496 cur_start = cur_end + 1; 497 } while (cur_end < end); 498 499 list = list->next; 500 } 501 mem_node_physalign = 0; 502 mem_node_pfn_shift = 0; 503 } 504 505 506 int 507 plat_lgrphand_to_mem_node(lgrp_handle_t hand) 508 { 509 if (max_mem_nodes == 1) 510 return (0); 511 512 return ((int)hand); 513 } 514 515 516 /* 517 * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt 518 * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if 519 * a crossing is found and returns 0 otherwise. 520 */ 521 int 522 plat_mnode_xcheck(pfn_t pfncnt) 523 { 524 int node, prevnode = -1, basenode; 525 pfn_t ea, sa; 526 527 for (node = 0; node < lgrp_plat_node_cnt; node++) { 528 529 if (lgrp_plat_node_memory[node].exists == 0) 530 continue; 531 532 if (prevnode == -1) { 533 prevnode = node; 534 basenode = node; 535 continue; 536 } 537 538 /* assume x86 node pfn ranges are in increasing order */ 539 ASSERT(lgrp_plat_node_memory[node].start > 540 lgrp_plat_node_memory[prevnode].end); 541 542 /* 543 * continue if the starting address of node is not contiguous 544 * with the previous node. 545 */ 546 547 if (lgrp_plat_node_memory[node].start != 548 (lgrp_plat_node_memory[prevnode].end + 1)) { 549 basenode = node; 550 prevnode = node; 551 continue; 552 } 553 554 /* check if the starting address of node is pfncnt aligned */ 555 if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) { 556 557 /* 558 * at this point, node starts at an unaligned boundary 559 * and is contiguous with the previous node(s) to 560 * basenode. Check if there is an aligned contiguous 561 * range of length pfncnt that crosses this boundary. 562 */ 563 564 sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end, 565 pfncnt); 566 ea = P2ROUNDUP((lgrp_plat_node_memory[node].start), 567 pfncnt); 568 569 ASSERT((ea - sa) == pfncnt); 570 if (sa >= lgrp_plat_node_memory[basenode].start && 571 ea <= (lgrp_plat_node_memory[node].end + 1)) 572 return (1); 573 } 574 prevnode = node; 575 } 576 return (0); 577 } 578 579 580 lgrp_handle_t 581 plat_mem_node_to_lgrphand(int mnode) 582 { 583 if (max_mem_nodes == 1) 584 return (LGRP_DEFAULT_HANDLE); 585 586 return ((lgrp_handle_t)mnode); 587 } 588 589 590 int 591 plat_pfn_to_mem_node(pfn_t pfn) 592 { 593 int node; 594 595 if (max_mem_nodes == 1) 596 return (0); 597 598 for (node = 0; node < lgrp_plat_node_cnt; node++) { 599 /* 600 * Skip nodes with no memory 601 */ 602 if (!lgrp_plat_node_memory[node].exists) 603 continue; 604 605 if (pfn >= lgrp_plat_node_memory[node].start && 606 pfn <= lgrp_plat_node_memory[node].end) 607 return (node); 608 } 609 610 /* 611 * Didn't find memnode where this PFN lives which should never happen 612 */ 613 ASSERT(node < lgrp_plat_node_cnt); 614 return (-1); 615 } 616 617 618 /* 619 * LGROUP PLATFORM INTERFACE ROUTINES 620 */ 621 622 /* 623 * Allocate additional space for an lgroup. 624 */ 625 /* ARGSUSED */ 626 lgrp_t * 627 lgrp_plat_alloc(lgrp_id_t lgrpid) 628 { 629 lgrp_t *lgrp; 630 631 lgrp = &lgrp_space[nlgrps_alloc++]; 632 if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) 633 return (NULL); 634 return (lgrp); 635 } 636 637 638 /* 639 * Platform handling for (re)configuration changes 640 */ 641 /* ARGSUSED */ 642 void 643 lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) 644 { 645 } 646 647 648 /* 649 * Return the platform handle for the lgroup containing the given CPU 650 */ 651 /* ARGSUSED */ 652 lgrp_handle_t 653 lgrp_plat_cpu_to_hand(processorid_t id) 654 { 655 lgrp_handle_t hand; 656 657 if (lgrp_plat_node_cnt == 1) 658 return (LGRP_DEFAULT_HANDLE); 659 660 hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], 661 lgrp_plat_cpu_node); 662 663 ASSERT(hand != (lgrp_handle_t)-1); 664 if (hand == (lgrp_handle_t)-1) 665 return (LGRP_NULL_HANDLE); 666 667 return (hand); 668 } 669 670 671 /* 672 * Platform-specific initialization of lgroups 673 */ 674 void 675 lgrp_plat_init(void) 676 { 677 #if defined(__xpv) 678 /* 679 * XXPV For now, the hypervisor treats all memory equally. 680 */ 681 lgrp_plat_node_cnt = max_mem_nodes = 1; 682 #else /* __xpv */ 683 uint_t probe_op; 684 u_longlong_t value; 685 686 /* 687 * Get boot property for lgroup topology height limit 688 */ 689 if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0) 690 (void) lgrp_topo_ht_limit_set((int)value); 691 692 /* 693 * Get boot property for enabling/disabling SRAT 694 */ 695 if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0) 696 lgrp_plat_srat_enable = (int)value; 697 698 /* 699 * Get boot property for enabling/disabling SLIT 700 */ 701 if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0) 702 lgrp_plat_slit_enable = (int)value; 703 704 /* 705 * Initialize as a UMA machine 706 */ 707 if (lgrp_topo_ht_limit() == 1) { 708 lgrp_plat_node_cnt = max_mem_nodes = 1; 709 return; 710 } 711 712 /* 713 * Read boot property with CPU to APIC ID mapping table/array and fill 714 * in CPU to node ID mapping table with APIC ID for each CPU 715 */ 716 lgrp_plat_apic_ncpus = 717 lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); 718 719 /* 720 * Determine which CPUs and memory are local to each other and number 721 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) 722 */ 723 if (lgrp_plat_apic_ncpus > 0) { 724 int retval; 725 726 retval = lgrp_plat_process_srat(srat_ptr, 727 lgrp_plat_node_domain, lgrp_plat_cpu_node, 728 lgrp_plat_apic_ncpus, lgrp_plat_node_memory); 729 if (retval <= 0) { 730 lgrp_plat_srat_error = retval; 731 lgrp_plat_node_cnt = 1; 732 } else { 733 lgrp_plat_srat_error = 0; 734 lgrp_plat_node_cnt = retval; 735 } 736 } 737 738 /* 739 * Try to use PCI config space registers on Opteron if there's an error 740 * processing CPU to APIC ID mapping or SRAT 741 */ 742 if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && 743 is_opteron()) 744 opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, 745 lgrp_plat_node_memory); 746 747 /* 748 * Don't bother to setup system for multiple lgroups and only use one 749 * memory node when memory is interleaved between any nodes or there is 750 * only one NUMA node 751 * 752 * NOTE: May need to change this for Dynamic Reconfiguration (DR) 753 * when and if it happens for x86/x64 754 */ 755 if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { 756 lgrp_plat_node_cnt = max_mem_nodes = 1; 757 (void) lgrp_topo_ht_limit_set(1); 758 return; 759 } 760 761 /* 762 * Leaf lgroups on x86/x64 architectures contain one physical 763 * processor chip. Tune lgrp_expand_proc_thresh and 764 * lgrp_expand_proc_diff so that lgrp_choose() will spread 765 * things out aggressively. 766 */ 767 lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; 768 lgrp_expand_proc_diff = 0; 769 770 /* 771 * There should be one memnode (physical page free list(s)) for 772 * each node 773 */ 774 max_mem_nodes = lgrp_plat_node_cnt; 775 776 /* 777 * Initialize min and max latency before reading SLIT or probing 778 */ 779 lgrp_plat_lat_stats.latency_min = -1; 780 lgrp_plat_lat_stats.latency_max = 0; 781 782 /* 783 * Determine how far each NUMA node is from each other by 784 * reading ACPI System Locality Information Table (SLIT) if it 785 * exists 786 */ 787 lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, 788 lgrp_plat_node_cnt, lgrp_plat_node_memory, 789 &lgrp_plat_lat_stats); 790 if (lgrp_plat_slit_error == 0) 791 return; 792 793 /* 794 * Probe to determine latency between NUMA nodes when SLIT 795 * doesn't exist or make sense 796 */ 797 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; 798 799 /* 800 * Specify whether to probe using vendor ID register or page copy 801 * if hasn't been specified already or is overspecified 802 */ 803 probe_op = lgrp_plat_probe_flags & 804 (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 805 806 if (probe_op == 0 || 807 probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { 808 lgrp_plat_probe_flags &= 809 ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 810 if (is_opteron()) 811 lgrp_plat_probe_flags |= 812 LGRP_PLAT_PROBE_VENDOR; 813 else 814 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; 815 } 816 817 /* 818 * Probing errors can mess up the lgroup topology and 819 * force us fall back to a 2 level lgroup topology. 820 * Here we bound how tall the lgroup topology can grow 821 * in hopes of avoiding any anamolies in probing from 822 * messing up the lgroup topology by limiting the 823 * accuracy of the latency topology. 824 * 825 * Assume that nodes will at least be configured in a 826 * ring, so limit height of lgroup topology to be less 827 * than number of nodes on a system with 4 or more 828 * nodes 829 */ 830 if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == 831 lgrp_topo_ht_limit_default()) 832 (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); 833 #endif /* __xpv */ 834 } 835 836 837 /* 838 * Return latency between "from" and "to" lgroups 839 * 840 * This latency number can only be used for relative comparison 841 * between lgroups on the running system, cannot be used across platforms, 842 * and may not reflect the actual latency. It is platform and implementation 843 * specific, so platform gets to decide its value. It would be nice if the 844 * number was at least proportional to make comparisons more meaningful though. 845 */ 846 /* ARGSUSED */ 847 int 848 lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) 849 { 850 lgrp_handle_t src, dest; 851 int node; 852 853 if (max_mem_nodes == 1) 854 return (0); 855 856 /* 857 * Return max latency for root lgroup 858 */ 859 if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) 860 return (lgrp_plat_lat_stats.latency_max); 861 862 src = from; 863 dest = to; 864 865 /* 866 * Return 0 for nodes (lgroup platform handles) out of range 867 */ 868 if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES) 869 return (0); 870 871 /* 872 * Probe from current CPU if its lgroup latencies haven't been set yet 873 * and we are trying to get latency from current CPU to some node 874 */ 875 node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 876 ASSERT(node >= 0 && node < lgrp_plat_node_cnt); 877 if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src) 878 lgrp_plat_probe(); 879 880 return (lgrp_plat_lat_stats.latencies[src][dest]); 881 } 882 883 884 /* 885 * Platform-specific initialization 886 */ 887 void 888 lgrp_plat_main_init(void) 889 { 890 int curnode; 891 int ht_limit; 892 int i; 893 894 /* 895 * Print a notice that MPO is disabled when memory is interleaved 896 * across nodes....Would do this when it is discovered, but can't 897 * because it happens way too early during boot.... 898 */ 899 if (lgrp_plat_mem_intrlv) 900 cmn_err(CE_NOTE, 901 "MPO disabled because memory is interleaved\n"); 902 903 /* 904 * Don't bother to do any probing if it is disabled, there is only one 905 * node, or the height of the lgroup topology less than or equal to 2 906 */ 907 ht_limit = lgrp_topo_ht_limit(); 908 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 909 max_mem_nodes == 1 || ht_limit <= 2) { 910 /* 911 * Setup lgroup latencies for 2 level lgroup topology 912 * (ie. local and remote only) if they haven't been set yet 913 */ 914 if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && 915 lgrp_plat_lat_stats.latency_max == 0) 916 lgrp_plat_2level_setup(lgrp_plat_node_memory, 917 &lgrp_plat_lat_stats); 918 return; 919 } 920 921 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 922 /* 923 * Should have been able to probe from CPU 0 when it was added 924 * to lgroup hierarchy, but may not have been able to then 925 * because it happens so early in boot that gethrtime() hasn't 926 * been initialized. (:-( 927 */ 928 curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 929 ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt); 930 if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) 931 lgrp_plat_probe(); 932 933 return; 934 } 935 936 /* 937 * When probing memory, use one page for every sample to determine 938 * lgroup topology and taking multiple samples 939 */ 940 if (lgrp_plat_probe_mem_config.probe_memsize == 0) 941 lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * 942 lgrp_plat_probe_nsamples; 943 944 /* 945 * Map memory in each node needed for probing to determine latency 946 * topology 947 */ 948 for (i = 0; i < lgrp_plat_node_cnt; i++) { 949 int mnode; 950 951 /* 952 * Skip this node and leave its probe page NULL 953 * if it doesn't have any memory 954 */ 955 mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i); 956 if (!mem_node_config[mnode].exists) { 957 lgrp_plat_probe_mem_config.probe_va[i] = NULL; 958 continue; 959 } 960 961 /* 962 * Allocate one kernel virtual page 963 */ 964 lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, 965 lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); 966 if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { 967 cmn_err(CE_WARN, 968 "lgrp_plat_main_init: couldn't allocate memory"); 969 return; 970 } 971 972 /* 973 * Get PFN for first page in each node 974 */ 975 lgrp_plat_probe_mem_config.probe_pfn[i] = 976 mem_node_config[mnode].physbase; 977 978 /* 979 * Map virtual page to first page in node 980 */ 981 hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], 982 lgrp_plat_probe_mem_config.probe_memsize, 983 lgrp_plat_probe_mem_config.probe_pfn[i], 984 PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, 985 HAT_LOAD_NOCONSIST); 986 } 987 988 /* 989 * Probe from current CPU 990 */ 991 lgrp_plat_probe(); 992 } 993 994 995 /* 996 * Return the maximum number of lgrps supported by the platform. 997 * Before lgrp topology is known it returns an estimate based on the number of 998 * nodes. Once topology is known it returns the actual maximim number of lgrps 999 * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and 1000 * dynamic addition of new nodes, this number may not grow during system 1001 * lifetime (yet). 1002 */ 1003 int 1004 lgrp_plat_max_lgrps(void) 1005 { 1006 return (lgrp_topo_initialized ? 1007 lgrp_alloc_max + 1 : 1008 lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); 1009 } 1010 1011 1012 /* 1013 * Return the number of free pages in an lgroup. 1014 * 1015 * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize 1016 * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the 1017 * number of allocatable base pagesize pages corresponding to the 1018 * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) 1019 * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical 1020 * memory installed, regardless of whether or not it's usable. 1021 */ 1022 pgcnt_t 1023 lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) 1024 { 1025 int mnode; 1026 pgcnt_t npgs = (pgcnt_t)0; 1027 extern struct memlist *phys_avail; 1028 extern struct memlist *phys_install; 1029 1030 1031 if (plathand == LGRP_DEFAULT_HANDLE) 1032 return (lgrp_plat_mem_size_default(plathand, query)); 1033 1034 if (plathand != LGRP_NULL_HANDLE) { 1035 mnode = plat_lgrphand_to_mem_node(plathand); 1036 if (mnode >= 0 && mem_node_config[mnode].exists) { 1037 switch (query) { 1038 case LGRP_MEM_SIZE_FREE: 1039 npgs = MNODE_PGCNT(mnode); 1040 break; 1041 case LGRP_MEM_SIZE_AVAIL: 1042 npgs = mem_node_memlist_pages(mnode, 1043 phys_avail); 1044 break; 1045 case LGRP_MEM_SIZE_INSTALL: 1046 npgs = mem_node_memlist_pages(mnode, 1047 phys_install); 1048 break; 1049 default: 1050 break; 1051 } 1052 } 1053 } 1054 return (npgs); 1055 } 1056 1057 1058 /* 1059 * Return the platform handle of the lgroup that contains the physical memory 1060 * corresponding to the given page frame number 1061 */ 1062 /* ARGSUSED */ 1063 lgrp_handle_t 1064 lgrp_plat_pfn_to_hand(pfn_t pfn) 1065 { 1066 int mnode; 1067 1068 if (max_mem_nodes == 1) 1069 return (LGRP_DEFAULT_HANDLE); 1070 1071 if (pfn > physmax) 1072 return (LGRP_NULL_HANDLE); 1073 1074 mnode = plat_pfn_to_mem_node(pfn); 1075 if (mnode < 0) 1076 return (LGRP_NULL_HANDLE); 1077 1078 return (MEM_NODE_2_LGRPHAND(mnode)); 1079 } 1080 1081 1082 /* 1083 * Probe memory in each node from current CPU to determine latency topology 1084 * 1085 * The probing code will probe the vendor ID register on the Northbridge of 1086 * Opteron processors and probe memory for other processors by default. 1087 * 1088 * Since probing is inherently error prone, the code takes laps across all the 1089 * nodes probing from each node to each of the other nodes some number of 1090 * times. Furthermore, each node is probed some number of times before moving 1091 * onto the next one during each lap. The minimum latency gotten between nodes 1092 * is kept as the latency between the nodes. 1093 * 1094 * After all that, the probe times are adjusted by normalizing values that are 1095 * close to each other and local latencies are made the same. Lastly, the 1096 * latencies are verified to make sure that certain conditions are met (eg. 1097 * local < remote, latency(a, b) == latency(b, a), etc.). 1098 * 1099 * If any of the conditions aren't met, the code will export a NUMA 1100 * configuration with the local CPUs and memory given by the SRAT or PCI config 1101 * space registers and one remote memory latency since it can't tell exactly 1102 * how far each node is from each other. 1103 */ 1104 void 1105 lgrp_plat_probe(void) 1106 { 1107 int from; 1108 int i; 1109 lgrp_plat_latency_stats_t *lat_stats; 1110 hrtime_t probe_time; 1111 int to; 1112 1113 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 1114 max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) 1115 return; 1116 1117 /* 1118 * Determine ID of node containing current CPU 1119 */ 1120 from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 1121 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 1122 if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) 1123 ASSERT(lgrp_plat_node_domain[from].exists); 1124 1125 /* 1126 * Don't need to probe if got times already 1127 */ 1128 lat_stats = &lgrp_plat_lat_stats; 1129 if (lat_stats->latencies[from][from] != 0) 1130 return; 1131 1132 /* 1133 * Read vendor ID in Northbridge or read and write page(s) 1134 * in each node from current CPU and remember how long it takes, 1135 * so we can build latency topology of machine later. 1136 * This should approximate the memory latency between each node. 1137 */ 1138 for (i = 0; i < lgrp_plat_probe_nrounds; i++) { 1139 for (to = 0; to < lgrp_plat_node_cnt; to++) { 1140 /* 1141 * Get probe time and bail out if can't get it yet 1142 */ 1143 probe_time = lgrp_plat_probe_time(to, 1144 lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config, 1145 &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); 1146 if (probe_time == 0) 1147 return; 1148 1149 /* 1150 * Keep lowest probe time as latency between nodes 1151 */ 1152 if (lat_stats->latencies[from][to] == 0 || 1153 probe_time < lat_stats->latencies[from][to]) 1154 lat_stats->latencies[from][to] = probe_time; 1155 1156 /* 1157 * Update overall minimum and maximum probe times 1158 * across all nodes 1159 */ 1160 if (probe_time < lat_stats->latency_min || 1161 lat_stats->latency_min == -1) 1162 lat_stats->latency_min = probe_time; 1163 if (probe_time > lat_stats->latency_max) 1164 lat_stats->latency_max = probe_time; 1165 } 1166 } 1167 1168 /* 1169 * - Fix up latencies such that local latencies are same, 1170 * latency(i, j) == latency(j, i), etc. (if possible) 1171 * 1172 * - Verify that latencies look ok 1173 * 1174 * - Fallback to just optimizing for local and remote if 1175 * latencies didn't look right 1176 */ 1177 lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats, 1178 &lgrp_plat_probe_stats); 1179 lgrp_plat_probe_stats.probe_error_code = 1180 lgrp_plat_latency_verify(lgrp_plat_node_memory, 1181 &lgrp_plat_lat_stats); 1182 if (lgrp_plat_probe_stats.probe_error_code) 1183 lgrp_plat_2level_setup(lgrp_plat_node_memory, 1184 &lgrp_plat_lat_stats); 1185 } 1186 1187 1188 /* 1189 * Return platform handle for root lgroup 1190 */ 1191 lgrp_handle_t 1192 lgrp_plat_root_hand(void) 1193 { 1194 return (LGRP_DEFAULT_HANDLE); 1195 } 1196 1197 1198 /* 1199 * INTERNAL ROUTINES 1200 */ 1201 1202 1203 /* 1204 * Update CPU to node mapping for given CPU and proximity domain (and returns 1205 * negative numbers for errors and positive ones for success) 1206 */ 1207 static int 1208 lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, 1209 cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) 1210 { 1211 uint_t i; 1212 int node; 1213 1214 /* 1215 * Get node number for proximity domain 1216 */ 1217 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1218 if (node == -1) { 1219 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1220 domain); 1221 if (node == -1) 1222 return (-1); 1223 } 1224 1225 /* 1226 * Search for entry with given APIC ID and fill in its node and 1227 * proximity domain IDs (if they haven't been set already) 1228 */ 1229 for (i = 0; i < nentries; i++) { 1230 /* 1231 * Skip nonexistent entries and ones without matching APIC ID 1232 */ 1233 if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) 1234 continue; 1235 1236 /* 1237 * Just return if entry completely and correctly filled in 1238 * already 1239 */ 1240 if (cpu_node[i].prox_domain == domain && 1241 cpu_node[i].node == node) 1242 return (1); 1243 1244 /* 1245 * Fill in node and proximity domain IDs 1246 */ 1247 cpu_node[i].prox_domain = domain; 1248 cpu_node[i].node = node; 1249 1250 return (0); 1251 } 1252 1253 /* 1254 * Return error when entry for APIC ID wasn't found in table 1255 */ 1256 return (-2); 1257 } 1258 1259 1260 /* 1261 * Get node ID for given CPU 1262 */ 1263 static int 1264 lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node) 1265 { 1266 processorid_t cpuid; 1267 1268 if (cp == NULL) 1269 return (-1); 1270 1271 cpuid = cp->cpu_id; 1272 if (cpuid < 0 || cpuid >= max_ncpus) 1273 return (-1); 1274 1275 /* 1276 * SRAT doesn't exist, isn't enabled, or there was an error processing 1277 * it, so return chip ID for Opteron and -1 otherwise. 1278 */ 1279 if (srat_ptr == NULL || !lgrp_plat_srat_enable || 1280 lgrp_plat_srat_error) { 1281 if (is_opteron()) 1282 return (pg_plat_hw_instance_id(cp, PGHW_CHIP)); 1283 return (-1); 1284 } 1285 1286 /* 1287 * Return -1 when CPU to node ID mapping entry doesn't exist for given 1288 * CPU 1289 */ 1290 if (!cpu_node[cpuid].exists) 1291 return (-1); 1292 1293 return (cpu_node[cpuid].node); 1294 } 1295 1296 1297 /* 1298 * Return node number for given proximity domain/system locality 1299 */ 1300 static int 1301 lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, 1302 uint32_t domain) 1303 { 1304 uint_t node; 1305 uint_t start; 1306 1307 /* 1308 * Hash proximity domain ID into node to domain mapping table (array), 1309 * search for entry with matching proximity domain ID, and return index 1310 * of matching entry as node ID. 1311 */ 1312 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1313 do { 1314 if (node_domain[node].prox_domain == domain && 1315 node_domain[node].exists) 1316 return (node); 1317 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1318 } while (node != start); 1319 return (-1); 1320 } 1321 1322 1323 /* 1324 * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to 1325 * be considered same 1326 */ 1327 #define LGRP_LAT_TOLERANCE_SHIFT 4 1328 1329 int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; 1330 1331 1332 /* 1333 * Adjust latencies between nodes to be symmetric, normalize latencies between 1334 * any nodes that are within some tolerance to be same, and make local 1335 * latencies be same 1336 */ 1337 static void 1338 lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 1339 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 1340 { 1341 int i; 1342 int j; 1343 int k; 1344 int l; 1345 u_longlong_t max; 1346 u_longlong_t min; 1347 u_longlong_t t; 1348 u_longlong_t t1; 1349 u_longlong_t t2; 1350 const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; 1351 int lat_corrected[MAX_NODES][MAX_NODES]; 1352 1353 /* 1354 * Nothing to do when this is an UMA machine or don't have args needed 1355 */ 1356 if (max_mem_nodes == 1) 1357 return; 1358 1359 ASSERT(node_memory != NULL && lat_stats != NULL && 1360 probe_stats != NULL); 1361 1362 /* 1363 * Make sure that latencies are symmetric between any two nodes 1364 * (ie. latency(node0, node1) == latency(node1, node0)) 1365 */ 1366 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1367 if (!node_memory[i].exists) 1368 continue; 1369 1370 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1371 if (!node_memory[j].exists) 1372 continue; 1373 1374 t1 = lat_stats->latencies[i][j]; 1375 t2 = lat_stats->latencies[j][i]; 1376 1377 if (t1 == 0 || t2 == 0 || t1 == t2) 1378 continue; 1379 1380 /* 1381 * Latencies should be same 1382 * - Use minimum of two latencies which should be same 1383 * - Track suspect probe times not within tolerance of 1384 * min value 1385 * - Remember how much values are corrected by 1386 */ 1387 if (t1 > t2) { 1388 t = t2; 1389 probe_stats->probe_errors[i][j] += t1 - t2; 1390 if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { 1391 probe_stats->probe_suspect[i][j]++; 1392 probe_stats->probe_suspect[j][i]++; 1393 } 1394 } else if (t2 > t1) { 1395 t = t1; 1396 probe_stats->probe_errors[j][i] += t2 - t1; 1397 if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { 1398 probe_stats->probe_suspect[i][j]++; 1399 probe_stats->probe_suspect[j][i]++; 1400 } 1401 } 1402 1403 lat_stats->latencies[i][j] = 1404 lat_stats->latencies[j][i] = t; 1405 lgrp_config(cflag, t1, t); 1406 lgrp_config(cflag, t2, t); 1407 } 1408 } 1409 1410 /* 1411 * Keep track of which latencies get corrected 1412 */ 1413 for (i = 0; i < MAX_NODES; i++) 1414 for (j = 0; j < MAX_NODES; j++) 1415 lat_corrected[i][j] = 0; 1416 1417 /* 1418 * For every two nodes, see whether there is another pair of nodes which 1419 * are about the same distance apart and make the latencies be the same 1420 * if they are close enough together 1421 */ 1422 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1423 if (!node_memory[i].exists) 1424 continue; 1425 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1426 if (!node_memory[j].exists) 1427 continue; 1428 /* 1429 * Pick one pair of nodes (i, j) 1430 * and get latency between them 1431 */ 1432 t1 = lat_stats->latencies[i][j]; 1433 1434 /* 1435 * Skip this pair of nodes if there isn't a latency 1436 * for it yet 1437 */ 1438 if (t1 == 0) 1439 continue; 1440 1441 for (k = 0; k < lgrp_plat_node_cnt; k++) { 1442 if (!node_memory[k].exists) 1443 continue; 1444 for (l = 0; l < lgrp_plat_node_cnt; l++) { 1445 if (!node_memory[l].exists) 1446 continue; 1447 /* 1448 * Pick another pair of nodes (k, l) 1449 * not same as (i, j) and get latency 1450 * between them 1451 */ 1452 if (k == i && l == j) 1453 continue; 1454 1455 t2 = lat_stats->latencies[k][l]; 1456 1457 /* 1458 * Skip this pair of nodes if there 1459 * isn't a latency for it yet 1460 */ 1461 1462 if (t2 == 0) 1463 continue; 1464 1465 /* 1466 * Skip nodes (k, l) if they already 1467 * have same latency as (i, j) or 1468 * their latency isn't close enough to 1469 * be considered/made the same 1470 */ 1471 if (t1 == t2 || (t1 > t2 && t1 - t2 > 1472 t1 >> lgrp_plat_probe_lt_shift) || 1473 (t2 > t1 && t2 - t1 > 1474 t2 >> lgrp_plat_probe_lt_shift)) 1475 continue; 1476 1477 /* 1478 * Make latency(i, j) same as 1479 * latency(k, l), try to use latency 1480 * that has been adjusted already to get 1481 * more consistency (if possible), and 1482 * remember which latencies were 1483 * adjusted for next time 1484 */ 1485 if (lat_corrected[i][j]) { 1486 t = t1; 1487 lgrp_config(cflag, t2, t); 1488 t2 = t; 1489 } else if (lat_corrected[k][l]) { 1490 t = t2; 1491 lgrp_config(cflag, t1, t); 1492 t1 = t; 1493 } else { 1494 if (t1 > t2) 1495 t = t2; 1496 else 1497 t = t1; 1498 lgrp_config(cflag, t1, t); 1499 lgrp_config(cflag, t2, t); 1500 t1 = t2 = t; 1501 } 1502 1503 lat_stats->latencies[i][j] = 1504 lat_stats->latencies[k][l] = t; 1505 1506 lat_corrected[i][j] = 1507 lat_corrected[k][l] = 1; 1508 } 1509 } 1510 } 1511 } 1512 1513 /* 1514 * Local latencies should be same 1515 * - Find min and max local latencies 1516 * - Make all local latencies be minimum 1517 */ 1518 min = -1; 1519 max = 0; 1520 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1521 if (!node_memory[i].exists) 1522 continue; 1523 t = lat_stats->latencies[i][i]; 1524 if (t == 0) 1525 continue; 1526 if (min == -1 || t < min) 1527 min = t; 1528 if (t > max) 1529 max = t; 1530 } 1531 if (min != max) { 1532 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1533 int local; 1534 1535 if (!node_memory[i].exists) 1536 continue; 1537 1538 local = lat_stats->latencies[i][i]; 1539 if (local == 0) 1540 continue; 1541 1542 /* 1543 * Track suspect probe times that aren't within 1544 * tolerance of minimum local latency and how much 1545 * probe times are corrected by 1546 */ 1547 if (local - min > min >> lgrp_plat_probe_lt_shift) 1548 probe_stats->probe_suspect[i][i]++; 1549 1550 probe_stats->probe_errors[i][i] += local - min; 1551 1552 /* 1553 * Make local latencies be minimum 1554 */ 1555 lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); 1556 lat_stats->latencies[i][i] = min; 1557 } 1558 } 1559 1560 /* 1561 * Determine max probe time again since just adjusted latencies 1562 */ 1563 lat_stats->latency_max = 0; 1564 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1565 if (!node_memory[i].exists) 1566 continue; 1567 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1568 if (!node_memory[j].exists) 1569 continue; 1570 t = lat_stats->latencies[i][j]; 1571 if (t > lat_stats->latency_max) 1572 lat_stats->latency_max = t; 1573 } 1574 } 1575 } 1576 1577 1578 /* 1579 * Verify following about latencies between nodes: 1580 * 1581 * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) 1582 * - Local latencies same 1583 * - Local < remote 1584 * - Number of latencies seen is reasonable 1585 * - Number of occurrences of a given latency should be more than 1 1586 * 1587 * Returns: 1588 * 0 Success 1589 * -1 Not symmetric 1590 * -2 Local latencies not same 1591 * -3 Local >= remote 1592 */ 1593 static int 1594 lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 1595 lgrp_plat_latency_stats_t *lat_stats) 1596 { 1597 int i; 1598 int j; 1599 u_longlong_t t1; 1600 u_longlong_t t2; 1601 1602 ASSERT(node_memory != NULL && lat_stats != NULL); 1603 1604 /* 1605 * Nothing to do when this is an UMA machine, lgroup topology is 1606 * limited to 2 levels, or there aren't any probe times yet 1607 */ 1608 if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || 1609 lat_stats->latencies[0][0] == 0) 1610 return (0); 1611 1612 /* 1613 * Make sure that latencies are symmetric between any two nodes 1614 * (ie. latency(node0, node1) == latency(node1, node0)) 1615 */ 1616 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1617 if (!node_memory[i].exists) 1618 continue; 1619 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1620 if (!node_memory[j].exists) 1621 continue; 1622 t1 = lat_stats->latencies[i][j]; 1623 t2 = lat_stats->latencies[j][i]; 1624 1625 if (t1 == 0 || t2 == 0 || t1 == t2) 1626 continue; 1627 1628 return (-1); 1629 } 1630 } 1631 1632 /* 1633 * Local latencies should be same 1634 */ 1635 t1 = lat_stats->latencies[0][0]; 1636 for (i = 1; i < lgrp_plat_node_cnt; i++) { 1637 if (!node_memory[i].exists) 1638 continue; 1639 1640 t2 = lat_stats->latencies[i][i]; 1641 if (t2 == 0) 1642 continue; 1643 1644 if (t1 == 0) { 1645 t1 = t2; 1646 continue; 1647 } 1648 1649 if (t1 != t2) 1650 return (-2); 1651 } 1652 1653 /* 1654 * Local latencies should be less than remote 1655 */ 1656 if (t1) { 1657 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1658 if (!node_memory[i].exists) 1659 continue; 1660 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1661 if (!node_memory[j].exists) 1662 continue; 1663 t2 = lat_stats->latencies[i][j]; 1664 if (i == j || t2 == 0) 1665 continue; 1666 1667 if (t1 >= t2) 1668 return (-3); 1669 } 1670 } 1671 } 1672 1673 return (0); 1674 } 1675 1676 1677 /* 1678 * Return the number of free, allocatable, or installed 1679 * pages in an lgroup 1680 * This is a copy of the MAX_MEM_NODES == 1 version of the routine 1681 * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup 1682 */ 1683 /* ARGSUSED */ 1684 static pgcnt_t 1685 lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) 1686 { 1687 struct memlist *mlist; 1688 pgcnt_t npgs = 0; 1689 extern struct memlist *phys_avail; 1690 extern struct memlist *phys_install; 1691 1692 switch (query) { 1693 case LGRP_MEM_SIZE_FREE: 1694 return ((pgcnt_t)freemem); 1695 case LGRP_MEM_SIZE_AVAIL: 1696 memlist_read_lock(); 1697 for (mlist = phys_avail; mlist; mlist = mlist->next) 1698 npgs += btop(mlist->size); 1699 memlist_read_unlock(); 1700 return (npgs); 1701 case LGRP_MEM_SIZE_INSTALL: 1702 memlist_read_lock(); 1703 for (mlist = phys_install; mlist; mlist = mlist->next) 1704 npgs += btop(mlist->size); 1705 memlist_read_unlock(); 1706 return (npgs); 1707 default: 1708 return ((pgcnt_t)0); 1709 } 1710 } 1711 1712 1713 /* 1714 * Update node to proximity domain mappings for given domain and return node ID 1715 */ 1716 static int 1717 lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, 1718 uint32_t domain) 1719 { 1720 uint_t node; 1721 uint_t start; 1722 1723 /* 1724 * Hash proximity domain ID into node to domain mapping table (array) 1725 * and add entry for it into first non-existent or matching entry found 1726 */ 1727 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1728 do { 1729 /* 1730 * Entry doesn't exist yet, so create one for this proximity 1731 * domain and return node ID which is index into mapping table. 1732 */ 1733 if (!node_domain[node].exists) { 1734 node_domain[node].exists = 1; 1735 node_domain[node].prox_domain = domain; 1736 return (node); 1737 } 1738 1739 /* 1740 * Entry exists for this proximity domain already, so just 1741 * return node ID (index into table). 1742 */ 1743 if (node_domain[node].prox_domain == domain) 1744 return (node); 1745 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1746 } while (node != start); 1747 1748 /* 1749 * Ran out of supported number of entries which shouldn't happen.... 1750 */ 1751 ASSERT(node != start); 1752 return (-1); 1753 } 1754 1755 1756 /* 1757 * Update node memory information for given proximity domain with specified 1758 * starting and ending physical address range (and return positive numbers for 1759 * success and negative ones for errors) 1760 */ 1761 static int 1762 lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt, 1763 node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end, 1764 uint32_t domain) 1765 { 1766 int node; 1767 1768 /* 1769 * Get node number for proximity domain 1770 */ 1771 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1772 if (node == -1) { 1773 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1774 domain); 1775 if (node == -1) 1776 return (-1); 1777 } 1778 1779 /* 1780 * Create entry in table for node if it doesn't exist 1781 */ 1782 if (!node_memory[node].exists) { 1783 node_memory[node].exists = 1; 1784 node_memory[node].start = btop(start); 1785 node_memory[node].end = btop(end); 1786 node_memory[node].prox_domain = domain; 1787 return (0); 1788 } 1789 1790 /* 1791 * Entry already exists for this proximity domain 1792 * 1793 * There may be more than one SRAT memory entry for a domain, so we may 1794 * need to update existing start or end address for the node. 1795 */ 1796 if (node_memory[node].prox_domain == domain) { 1797 if (btop(start) < node_memory[node].start) 1798 node_memory[node].start = btop(start); 1799 if (btop(end) > node_memory[node].end) 1800 node_memory[node].end = btop(end); 1801 return (1); 1802 } 1803 return (-2); 1804 } 1805 1806 1807 /* 1808 * Return time needed to probe from current CPU to memory in given node 1809 */ 1810 static hrtime_t 1811 lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 1812 lgrp_plat_probe_mem_config_t *probe_mem_config, 1813 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 1814 { 1815 caddr_t buf; 1816 hrtime_t elapsed; 1817 hrtime_t end; 1818 int from; 1819 int i; 1820 int ipl; 1821 hrtime_t max; 1822 hrtime_t min; 1823 hrtime_t start; 1824 extern int use_sse_pagecopy; 1825 1826 /* 1827 * Determine ID of node containing current CPU 1828 */ 1829 from = lgrp_plat_cpu_to_node(CPU, cpu_node); 1830 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 1831 1832 /* 1833 * Do common work for probing main memory 1834 */ 1835 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { 1836 /* 1837 * Skip probing any nodes without memory and 1838 * set probe time to 0 1839 */ 1840 if (probe_mem_config->probe_va[to] == NULL) { 1841 lat_stats->latencies[from][to] = 0; 1842 return (0); 1843 } 1844 1845 /* 1846 * Invalidate caches once instead of once every sample 1847 * which should cut cost of probing by a lot 1848 */ 1849 probe_stats->flush_cost = gethrtime(); 1850 invalidate_cache(); 1851 probe_stats->flush_cost = gethrtime() - 1852 probe_stats->flush_cost; 1853 probe_stats->probe_cost_total += probe_stats->flush_cost; 1854 } 1855 1856 /* 1857 * Probe from current CPU to given memory using specified operation 1858 * and take specified number of samples 1859 */ 1860 max = 0; 1861 min = -1; 1862 for (i = 0; i < lgrp_plat_probe_nsamples; i++) { 1863 probe_stats->probe_cost = gethrtime(); 1864 1865 /* 1866 * Can't measure probe time if gethrtime() isn't working yet 1867 */ 1868 if (probe_stats->probe_cost == 0 && gethrtime() == 0) 1869 return (0); 1870 1871 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 1872 /* 1873 * Measure how long it takes to read vendor ID from 1874 * Northbridge 1875 */ 1876 elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); 1877 } else { 1878 /* 1879 * Measure how long it takes to copy page 1880 * on top of itself 1881 */ 1882 buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); 1883 1884 kpreempt_disable(); 1885 ipl = splhigh(); 1886 start = gethrtime(); 1887 if (use_sse_pagecopy) 1888 hwblkpagecopy(buf, buf); 1889 else 1890 bcopy(buf, buf, PAGESIZE); 1891 end = gethrtime(); 1892 elapsed = end - start; 1893 splx(ipl); 1894 kpreempt_enable(); 1895 } 1896 1897 probe_stats->probe_cost = gethrtime() - 1898 probe_stats->probe_cost; 1899 probe_stats->probe_cost_total += probe_stats->probe_cost; 1900 1901 if (min == -1 || elapsed < min) 1902 min = elapsed; 1903 if (elapsed > max) 1904 max = elapsed; 1905 } 1906 1907 /* 1908 * Update minimum and maximum probe times between 1909 * these two nodes 1910 */ 1911 if (min < probe_stats->probe_min[from][to] || 1912 probe_stats->probe_min[from][to] == 0) 1913 probe_stats->probe_min[from][to] = min; 1914 1915 if (max > probe_stats->probe_max[from][to]) 1916 probe_stats->probe_max[from][to] = max; 1917 1918 return (min); 1919 } 1920 1921 1922 /* 1923 * Read boot property with CPU to APIC ID array, fill in CPU to node ID 1924 * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs. 1925 * 1926 * NOTE: This code assumes that CPU IDs are assigned in order that they appear 1927 * in in cpu_apicid_array boot property which is based on and follows 1928 * same ordering as processor list in ACPI MADT. If the code in 1929 * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns 1930 * CPU IDs ever changes, then this code will need to change too.... 1931 */ 1932 static int 1933 lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) 1934 { 1935 int boot_prop_len; 1936 char *boot_prop_name = BP_CPU_APICID_ARRAY; 1937 uint8_t cpu_apicid_array[UINT8_MAX + 1]; 1938 int i; 1939 int n; 1940 1941 /* 1942 * Nothing to do when no array to fill in or not enough CPUs 1943 */ 1944 if (cpu_node == NULL) 1945 return (-1); 1946 1947 /* 1948 * Check length of property value 1949 */ 1950 boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); 1951 if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array)) 1952 return (-2); 1953 1954 /* 1955 * Calculate number of entries in array and return when there's just 1956 * one CPU since that's not very interesting for NUMA 1957 */ 1958 n = boot_prop_len / sizeof (uint8_t); 1959 if (n == 1) 1960 return (-3); 1961 1962 /* 1963 * Get CPU to APIC ID property value 1964 */ 1965 if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) 1966 return (-4); 1967 1968 /* 1969 * Fill in CPU to node ID mapping table with APIC ID for each CPU 1970 */ 1971 for (i = 0; i < n; i++) { 1972 cpu_node[i].exists = 1; 1973 cpu_node[i].apicid = cpu_apicid_array[i]; 1974 } 1975 1976 /* 1977 * Return number of CPUs based on number of APIC IDs 1978 */ 1979 return (n); 1980 } 1981 1982 1983 /* 1984 * Read ACPI System Locality Information Table (SLIT) to determine how far each 1985 * NUMA node is from each other 1986 */ 1987 static int 1988 lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 1989 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats) 1990 { 1991 int i; 1992 int j; 1993 int localities; 1994 hrtime_t max; 1995 hrtime_t min; 1996 int retval; 1997 uint8_t *slit_entries; 1998 1999 if (tp == NULL || !lgrp_plat_slit_enable) 2000 return (1); 2001 2002 if (lat_stats == NULL) 2003 return (2); 2004 2005 localities = tp->number; 2006 if (localities != node_cnt) 2007 return (3); 2008 2009 min = lat_stats->latency_min; 2010 max = lat_stats->latency_max; 2011 2012 /* 2013 * Fill in latency matrix based on SLIT entries 2014 */ 2015 slit_entries = tp->entry; 2016 for (i = 0; i < localities; i++) { 2017 for (j = 0; j < localities; j++) { 2018 uint8_t latency; 2019 2020 latency = slit_entries[(i * localities) + j]; 2021 lat_stats->latencies[i][j] = latency; 2022 if (latency < min || min == -1) 2023 min = latency; 2024 if (latency > max) 2025 max = latency; 2026 } 2027 } 2028 2029 /* 2030 * Verify that latencies/distances given in SLIT look reasonable 2031 */ 2032 retval = lgrp_plat_latency_verify(node_memory, lat_stats); 2033 2034 if (retval) { 2035 /* 2036 * Reinitialize (zero) latency table since SLIT doesn't look 2037 * right 2038 */ 2039 for (i = 0; i < localities; i++) { 2040 for (j = 0; j < localities; j++) 2041 lat_stats->latencies[i][j] = 0; 2042 } 2043 } else { 2044 /* 2045 * Update min and max latencies seen since SLIT looks valid 2046 */ 2047 lat_stats->latency_min = min; 2048 lat_stats->latency_max = max; 2049 } 2050 2051 return (retval); 2052 } 2053 2054 2055 /* 2056 * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs 2057 * and memory are local to each other in the same NUMA node and return number 2058 * of nodes 2059 */ 2060 static int 2061 lgrp_plat_process_srat(struct srat *tp, node_domain_map_t *node_domain, 2062 cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory) 2063 { 2064 struct srat_item *srat_end; 2065 int i; 2066 struct srat_item *item; 2067 int node_cnt; 2068 int proc_entry_count; 2069 2070 /* 2071 * Nothing to do when no SRAT or disabled 2072 */ 2073 if (tp == NULL || !lgrp_plat_srat_enable) 2074 return (-1); 2075 2076 /* 2077 * Determine number of nodes by counting number of proximity domains in 2078 * SRAT and return if number of nodes is 1 or less since don't need to 2079 * read SRAT then 2080 */ 2081 node_cnt = lgrp_plat_srat_domains(tp); 2082 if (node_cnt == 1) 2083 return (1); 2084 else if (node_cnt <= 0) 2085 return (-2); 2086 2087 /* 2088 * Walk through SRAT, examining each CPU and memory entry to determine 2089 * which CPUs and memory belong to which node. 2090 */ 2091 item = tp->list; 2092 srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2093 proc_entry_count = 0; 2094 while (item < srat_end) { 2095 uint32_t apic_id; 2096 uint32_t domain; 2097 uint64_t end; 2098 uint64_t length; 2099 uint64_t start; 2100 2101 switch (item->type) { 2102 case SRAT_PROCESSOR: /* CPU entry */ 2103 if (!(item->i.p.flags & SRAT_ENABLED) || 2104 cpu_node == NULL) 2105 break; 2106 2107 /* 2108 * Calculate domain (node) ID and fill in APIC ID to 2109 * domain/node mapping table 2110 */ 2111 domain = item->i.p.domain1; 2112 for (i = 0; i < 3; i++) { 2113 domain += item->i.p.domain2[i] << 2114 ((i + 1) * 8); 2115 } 2116 apic_id = item->i.p.apic_id; 2117 2118 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2119 cpu_node, cpu_count, apic_id, domain) < 0) 2120 return (-3); 2121 2122 proc_entry_count++; 2123 break; 2124 2125 case SRAT_MEMORY: /* memory entry */ 2126 if (!(item->i.m.flags & SRAT_ENABLED) || 2127 node_memory == NULL) 2128 break; 2129 2130 /* 2131 * Get domain (node) ID and fill in domain/node 2132 * to memory mapping table 2133 */ 2134 domain = item->i.m.domain; 2135 start = item->i.m.base_addr; 2136 length = item->i.m.len; 2137 end = start + length - 1; 2138 2139 if (lgrp_plat_node_memory_update(node_domain, node_cnt, 2140 node_memory, start, end, domain) < 0) 2141 return (-4); 2142 break; 2143 case SRAT_X2APIC: /* x2apic CPU entry */ 2144 if (!(item->i.xp.flags & SRAT_ENABLED) || 2145 cpu_node == NULL) 2146 break; 2147 2148 /* 2149 * Calculate domain (node) ID and fill in APIC ID to 2150 * domain/node mapping table 2151 */ 2152 domain = item->i.xp.domain; 2153 apic_id = item->i.xp.x2apic_id; 2154 2155 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2156 cpu_node, cpu_count, apic_id, domain) < 0) 2157 return (-3); 2158 2159 proc_entry_count++; 2160 break; 2161 2162 default: 2163 break; 2164 } 2165 2166 item = (struct srat_item *)((uintptr_t)item + item->len); 2167 } 2168 2169 /* 2170 * Should have seen at least as many SRAT processor entries as CPUs 2171 */ 2172 if (proc_entry_count < cpu_count) 2173 return (-5); 2174 2175 return (node_cnt); 2176 } 2177 2178 2179 /* 2180 * Return number of proximity domains given in ACPI SRAT 2181 */ 2182 static int 2183 lgrp_plat_srat_domains(struct srat *tp) 2184 { 2185 int domain_cnt; 2186 struct srat_item *end; 2187 int i; 2188 struct srat_item *item; 2189 node_domain_map_t node_domain[MAX_NODES]; 2190 2191 2192 if (tp == NULL || !lgrp_plat_srat_enable) 2193 return (1); 2194 2195 /* 2196 * Walk through SRAT, examining each CPU and memory entry to determine 2197 * proximity domain ID for each. 2198 */ 2199 domain_cnt = 0; 2200 item = tp->list; 2201 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2202 bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); 2203 while (item < end) { 2204 uint32_t domain; 2205 boolean_t overflow; 2206 uint_t start; 2207 2208 switch (item->type) { 2209 case SRAT_PROCESSOR: /* CPU entry */ 2210 if (!(item->i.p.flags & SRAT_ENABLED)) 2211 break; 2212 domain = item->i.p.domain1; 2213 for (i = 0; i < 3; i++) { 2214 domain += item->i.p.domain2[i] << 2215 ((i + 1) * 8); 2216 } 2217 break; 2218 2219 case SRAT_MEMORY: /* memory entry */ 2220 if (!(item->i.m.flags & SRAT_ENABLED)) 2221 break; 2222 domain = item->i.m.domain; 2223 break; 2224 2225 case SRAT_X2APIC: /* x2apic CPU entry */ 2226 if (!(item->i.xp.flags & SRAT_ENABLED)) 2227 break; 2228 domain = item->i.xp.domain; 2229 break; 2230 2231 default: 2232 break; 2233 } 2234 2235 /* 2236 * Count and keep track of which proximity domain IDs seen 2237 */ 2238 start = i = domain % MAX_NODES; 2239 overflow = B_TRUE; 2240 do { 2241 /* 2242 * Create entry for proximity domain and increment 2243 * count when no entry exists where proximity domain 2244 * hashed 2245 */ 2246 if (!node_domain[i].exists) { 2247 node_domain[i].exists = 1; 2248 node_domain[i].prox_domain = domain; 2249 domain_cnt++; 2250 overflow = B_FALSE; 2251 break; 2252 } 2253 2254 /* 2255 * Nothing to do when proximity domain seen already 2256 * and its entry exists 2257 */ 2258 if (node_domain[i].prox_domain == domain) { 2259 overflow = B_FALSE; 2260 break; 2261 } 2262 2263 /* 2264 * Entry exists where proximity domain hashed, but for 2265 * different proximity domain so keep search for empty 2266 * slot to put it or matching entry whichever comes 2267 * first. 2268 */ 2269 i = (i + 1) % MAX_NODES; 2270 } while (i != start); 2271 2272 /* 2273 * Didn't find empty or matching entry which means have more 2274 * proximity domains than supported nodes (:-( 2275 */ 2276 ASSERT(overflow != B_TRUE); 2277 if (overflow == B_TRUE) 2278 return (-1); 2279 2280 item = (struct srat_item *)((uintptr_t)item + item->len); 2281 } 2282 return (domain_cnt); 2283 } 2284 2285 2286 /* 2287 * Set lgroup latencies for 2 level lgroup topology 2288 */ 2289 static void 2290 lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 2291 lgrp_plat_latency_stats_t *lat_stats) 2292 { 2293 int i; 2294 2295 ASSERT(node_memory != NULL && lat_stats != NULL); 2296 2297 if (lgrp_plat_node_cnt >= 4) 2298 cmn_err(CE_NOTE, 2299 "MPO only optimizing for local and remote\n"); 2300 for (i = 0; i < lgrp_plat_node_cnt; i++) { 2301 int j; 2302 2303 if (!node_memory[i].exists) 2304 continue; 2305 for (j = 0; j < lgrp_plat_node_cnt; j++) { 2306 if (!node_memory[j].exists) 2307 continue; 2308 if (i == j) 2309 lat_stats->latencies[i][j] = 2; 2310 else 2311 lat_stats->latencies[i][j] = 3; 2312 } 2313 } 2314 lat_stats->latency_min = 2; 2315 lat_stats->latency_max = 3; 2316 lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); 2317 } 2318 2319 2320 /* 2321 * The following Opteron specific constants, macros, types, and routines define 2322 * PCI configuration space registers and how to read them to determine the NUMA 2323 * configuration of *supported* Opteron processors. They provide the same 2324 * information that may be gotten from the ACPI System Resource Affinity Table 2325 * (SRAT) if it exists on the machine of interest. 2326 * 2327 * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family 2328 * of interest describes all of these registers and their contents. The main 2329 * registers used by this code to determine the NUMA configuration of the 2330 * machine are the node ID register for the number of NUMA nodes and the DRAM 2331 * address map registers for the physical address range of each node. 2332 * 2333 * NOTE: The format and how to determine the NUMA configuration using PCI 2334 * config space registers may change or may not be supported in future 2335 * Opteron processor families. 2336 */ 2337 2338 /* 2339 * How many bits to shift Opteron DRAM Address Map base and limit registers 2340 * to get actual value 2341 */ 2342 #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ 2343 #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ 2344 2345 #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ 2346 #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ 2347 2348 #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ 2349 2350 /* 2351 * Macros to derive addresses from Opteron DRAM Address Map registers 2352 */ 2353 #define OPT_DRAMADDR_HI(reg) \ 2354 (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ 2355 OPT_DRAMADDR_HI_LSHIFT_ADDR) 2356 2357 #define OPT_DRAMADDR_LO(reg) \ 2358 (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ 2359 OPT_DRAMADDR_LO_LSHIFT_ADDR) 2360 2361 #define OPT_DRAMADDR(high, low) \ 2362 (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) 2363 2364 /* 2365 * Bit masks defining what's in Opteron DRAM Address Map base register 2366 */ 2367 #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ 2368 #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ 2369 #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ 2370 2371 /* 2372 * Bit masks defining what's in Opteron DRAM Address Map limit register 2373 */ 2374 #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ 2375 #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ 2376 2377 2378 /* 2379 * Opteron Node ID register in PCI configuration space contains 2380 * number of nodes in system, etc. for Opteron K8. The following 2381 * constants and macros define its contents, structure, and access. 2382 */ 2383 2384 /* 2385 * Bit masks defining what's in Opteron Node ID register 2386 */ 2387 #define OPT_NODE_MASK_ID 0x7 /* node ID */ 2388 #define OPT_NODE_MASK_CNT 0x70 /* node count */ 2389 #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ 2390 #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ 2391 #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ 2392 2393 /* 2394 * How many bits in Opteron Node ID register to shift right to get actual value 2395 */ 2396 #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ 2397 2398 /* 2399 * Macros to get values from Opteron Node ID register 2400 */ 2401 #define OPT_NODE_CNT(reg) \ 2402 ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) 2403 2404 /* 2405 * Macro to setup PCI Extended Configuration Space (ECS) address to give to 2406 * "in/out" instructions 2407 * 2408 * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any 2409 * other uses should just do MMIO to access PCI ECS. 2410 * Must enable special bit in Northbridge Configuration Register on 2411 * Greyhound for extended CF8 space access to be able to access PCI ECS 2412 * using "in/out" instructions and restore special bit after done 2413 * accessing PCI ECS. 2414 */ 2415 #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ 2416 (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ 2417 (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ 2418 ((((reg) >> 8) & 0xf) << 24)) 2419 2420 /* 2421 * PCI configuration space registers accessed by specifying 2422 * a bus, device, function, and offset. The following constants 2423 * define the values needed to access Opteron K8 configuration 2424 * info to determine its node topology 2425 */ 2426 2427 #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ 2428 2429 /* 2430 * Opteron PCI configuration space register function values 2431 */ 2432 #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ 2433 #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ 2434 #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ 2435 #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ 2436 2437 /* 2438 * PCI Configuration Space register offsets 2439 */ 2440 #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ 2441 #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ 2442 #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ 2443 #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ 2444 2445 /* 2446 * Opteron PCI Configuration Space device IDs for nodes 2447 */ 2448 #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ 2449 2450 2451 /* 2452 * Opteron DRAM address map gives base and limit for physical memory in a node 2453 */ 2454 typedef struct opt_dram_addr_map { 2455 uint32_t base_hi; 2456 uint32_t base_lo; 2457 uint32_t limit_hi; 2458 uint32_t limit_lo; 2459 } opt_dram_addr_map_t; 2460 2461 2462 /* 2463 * Supported AMD processor families 2464 */ 2465 #define AMD_FAMILY_HAMMER 15 2466 #define AMD_FAMILY_GREYHOUND 16 2467 2468 /* 2469 * Whether to have is_opteron() return 1 even when processor isn't supported 2470 */ 2471 uint_t is_opteron_override = 0; 2472 2473 /* 2474 * AMD processor family for current CPU 2475 */ 2476 uint_t opt_family = 0; 2477 2478 2479 /* 2480 * Determine whether we're running on a supported AMD Opteron since reading 2481 * node count and DRAM address map registers may have different format or 2482 * may not be supported across processor families 2483 */ 2484 static int 2485 is_opteron(void) 2486 { 2487 2488 if (x86_vendor != X86_VENDOR_AMD) 2489 return (0); 2490 2491 opt_family = cpuid_getfamily(CPU); 2492 if (opt_family == AMD_FAMILY_HAMMER || 2493 opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) 2494 return (1); 2495 else 2496 return (0); 2497 } 2498 2499 2500 /* 2501 * Determine NUMA configuration for Opteron from registers that live in PCI 2502 * configuration space 2503 */ 2504 static void 2505 opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 2506 node_phys_addr_map_t *node_memory) 2507 { 2508 uint_t bus; 2509 uint_t dev; 2510 struct opt_dram_addr_map dram_map[MAX_NODES]; 2511 uint_t node; 2512 uint_t node_info[MAX_NODES]; 2513 uint_t off_hi; 2514 uint_t off_lo; 2515 uint64_t nb_cfg_reg; 2516 2517 /* 2518 * Read configuration registers from PCI configuration space to 2519 * determine node information, which memory is in each node, etc. 2520 * 2521 * Write to PCI configuration space address register to specify 2522 * which configuration register to read and read/write PCI 2523 * configuration space data register to get/set contents 2524 */ 2525 bus = OPT_PCS_BUS_CONFIG; 2526 dev = OPT_PCS_DEV_NODE0; 2527 off_hi = OPT_PCS_OFF_DRAMBASE_HI; 2528 off_lo = OPT_PCS_OFF_DRAMBASE_LO; 2529 2530 /* 2531 * Read node ID register for node 0 to get node count 2532 */ 2533 node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, 2534 OPT_PCS_OFF_NODEID); 2535 *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; 2536 2537 /* 2538 * If number of nodes is more than maximum supported, then set node 2539 * count to 1 and treat system as UMA instead of NUMA. 2540 */ 2541 if (*node_cnt > MAX_NODES) { 2542 *node_cnt = 1; 2543 return; 2544 } 2545 2546 /* 2547 * For Greyhound, PCI Extended Configuration Space must be enabled to 2548 * read high DRAM address map base and limit registers 2549 */ 2550 if (opt_family == AMD_FAMILY_GREYHOUND) { 2551 nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); 2552 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2553 wrmsr(MSR_AMD_NB_CFG, 2554 nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); 2555 } 2556 2557 for (node = 0; node < *node_cnt; node++) { 2558 uint32_t base_hi; 2559 uint32_t base_lo; 2560 uint32_t limit_hi; 2561 uint32_t limit_lo; 2562 2563 /* 2564 * Read node ID register (except for node 0 which we just read) 2565 */ 2566 if (node > 0) { 2567 node_info[node] = pci_getl_func(bus, dev, 2568 OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); 2569 } 2570 2571 /* 2572 * Read DRAM base and limit registers which specify 2573 * physical memory range of each node 2574 */ 2575 if (opt_family != AMD_FAMILY_GREYHOUND) 2576 base_hi = 0; 2577 else { 2578 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2579 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2580 base_hi = dram_map[node].base_hi = 2581 inl(PCI_CONFDATA); 2582 } 2583 base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, 2584 OPT_PCS_FUNC_ADDRMAP, off_lo); 2585 2586 if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && 2587 mem_intrlv) 2588 *mem_intrlv = *mem_intrlv + 1; 2589 2590 off_hi += 4; /* high limit register offset */ 2591 if (opt_family != AMD_FAMILY_GREYHOUND) 2592 limit_hi = 0; 2593 else { 2594 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2595 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2596 limit_hi = dram_map[node].limit_hi = 2597 inl(PCI_CONFDATA); 2598 } 2599 2600 off_lo += 4; /* low limit register offset */ 2601 limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, 2602 dev, OPT_PCS_FUNC_ADDRMAP, off_lo); 2603 2604 /* 2605 * Increment device number to next node and register offsets 2606 * for DRAM base register of next node 2607 */ 2608 off_hi += 4; 2609 off_lo += 4; 2610 dev++; 2611 2612 /* 2613 * Both read and write enable bits must be enabled in DRAM 2614 * address map base register for physical memory to exist in 2615 * node 2616 */ 2617 if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || 2618 (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { 2619 /* 2620 * Mark node memory as non-existent and set start and 2621 * end addresses to be same in node_memory[] 2622 */ 2623 node_memory[node].exists = 0; 2624 node_memory[node].start = node_memory[node].end = 2625 (pfn_t)-1; 2626 continue; 2627 } 2628 2629 /* 2630 * Mark node memory as existing and remember physical address 2631 * range of each node for use later 2632 */ 2633 node_memory[node].exists = 1; 2634 2635 node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); 2636 2637 node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | 2638 OPT_DRAMADDR_LO_MASK_OFF); 2639 } 2640 2641 /* 2642 * Restore PCI Extended Configuration Space enable bit 2643 */ 2644 if (opt_family == AMD_FAMILY_GREYHOUND) { 2645 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2646 wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); 2647 } 2648 } 2649 2650 2651 /* 2652 * Return average amount of time to read vendor ID register on Northbridge 2653 * N times on specified destination node from current CPU 2654 */ 2655 static hrtime_t 2656 opt_probe_vendor(int dest_node, int nreads) 2657 { 2658 int cnt; 2659 uint_t dev; 2660 /* LINTED: set but not used in function */ 2661 volatile uint_t dev_vendor; 2662 hrtime_t elapsed; 2663 hrtime_t end; 2664 int ipl; 2665 hrtime_t start; 2666 2667 dev = OPT_PCS_DEV_NODE0 + dest_node; 2668 kpreempt_disable(); 2669 ipl = spl8(); 2670 outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, 2671 OPT_PCS_OFF_VENDOR)); 2672 start = gethrtime(); 2673 for (cnt = 0; cnt < nreads; cnt++) 2674 dev_vendor = inl(PCI_CONFDATA); 2675 end = gethrtime(); 2676 elapsed = (end - start) / nreads; 2677 splx(ipl); 2678 kpreempt_enable(); 2679 return (elapsed); 2680 } 2681