1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS 29 * ================================================================ 30 * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access 31 * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of 32 * one or more CPUs and some local memory. The CPUs in each node can access 33 * the memory in the other nodes but at a higher latency than accessing their 34 * local memory. Typically, a system with only one node has Uniform Memory 35 * Access (UMA), but it may be possible to have a one node system that has 36 * some global memory outside of the node which is higher latency. 37 * 38 * Module Description 39 * ------------------ 40 * This module provides a platform interface for determining which CPUs and 41 * which memory (and how much) are in a NUMA node and how far each node is from 42 * each other. The interface is used by the Virtual Memory (VM) system and the 43 * common lgroup framework. The VM system uses the plat_*() routines to fill 44 * in its memory node (memnode) array with the physical address range spanned 45 * by each NUMA node to know which memory belongs to which node, so it can 46 * build and manage a physical page free list for each NUMA node and allocate 47 * local memory from each node as needed. The common lgroup framework uses the 48 * exported lgrp_plat_*() routines to figure out which CPUs and memory belong 49 * to each node (leaf lgroup) and how far each node is from each other, so it 50 * can build the latency (lgroup) topology for the machine in order to optimize 51 * for locality. Also, an lgroup platform handle instead of lgroups are used 52 * in the interface with this module, so this module shouldn't need to know 53 * anything about lgroups. Instead, it just needs to know which CPUs, memory, 54 * etc. are in each NUMA node, how far each node is from each other, and to use 55 * a unique lgroup platform handle to refer to each node through the interface. 56 * 57 * Determining NUMA Configuration 58 * ------------------------------ 59 * By default, this module will try to determine the NUMA configuration of the 60 * machine by reading the ACPI System Resource Affinity Table (SRAT) and System 61 * Locality Information Table (SLIT). The SRAT contains info to tell which 62 * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT 63 * is a matrix that gives the distance between each system locality (which is 64 * a NUMA node and should correspond to proximity domains in the SRAT). For 65 * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer 66 * specification. 67 * 68 * If the SRAT doesn't exist on a system with AMD Opteron processors, we 69 * examine registers in PCI configuration space to determine how many nodes are 70 * in the system and which CPUs and memory are in each node. 71 * do while booting the kernel. 72 * 73 * NOTE: Using these PCI configuration space registers to determine this 74 * locality info is not guaranteed to work or be compatible across all 75 * Opteron processor families. 76 * 77 * If the SLIT does not exist or look right, the kernel will probe to determine 78 * the distance between nodes as long as the NUMA CPU and memory configuration 79 * has been determined (see lgrp_plat_probe() for details). 80 * 81 * Data Structures 82 * --------------- 83 * The main data structures used by this code are the following: 84 * 85 * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by 86 * CPU ID (only used for SRAT) 87 * 88 * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and 89 * different nodes indexed by node ID 90 * 91 * - lgrp_plat_node_cnt Number of NUMA nodes in system 92 * 93 * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping 94 * table indexed by node ID (only used 95 * for SRAT) 96 * 97 * - lgrp_plat_node_memory[] Table with physical address range for 98 * each node indexed by node ID 99 * 100 * The code is implemented to make the following always be true: 101 * 102 * lgroup platform handle == node ID == memnode ID 103 * 104 * Moreover, it allows for the proximity domain ID to be equal to all of the 105 * above as long as the proximity domains IDs are numbered from 0 to <number of 106 * nodes - 1>. This is done by hashing each proximity domain ID into the range 107 * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID 108 * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] 109 * and be assigned node ID N. If the proximity domain IDs aren't numbered 110 * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into 111 * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs 112 * to node IDs. However, the proximity domain IDs may not map to the 113 * equivalent node ID since we want to keep the node IDs numbered from 0 to 114 * <number of nodes - 1> to minimize cost of searching and potentially space. 115 * 116 * The code below really tries to do the above. However, the virtual memory 117 * system expects the memnodes which describe the physical address range for 118 * each NUMA node to be arranged in ascending order by physical address. (:-( 119 * Otherwise, the kernel will panic in different semi-random places in the VM 120 * system. 121 * 122 * Consequently, this module has to try to sort the nodes in ascending order by 123 * each node's starting physical address to try to meet this "constraint" in 124 * the VM system (see lgrp_plat_node_sort()). Also, the lowest numbered 125 * proximity domain ID in the system is deteremined and used to make the lowest 126 * numbered proximity domain map to node 0 in hopes that the proximity domains 127 * are sorted in ascending order by physical address already even if their IDs 128 * don't start at 0 (see NODE_DOMAIN_HASH() and lgrp_plat_srat_domains()). 129 * Finally, it is important to note that these workarounds may not be 130 * sufficient if/when memory hotplugging is supported and the VM system may 131 * ultimately need to be fixed to handle this.... 132 */ 133 134 135 #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */ 136 #include <sys/bootconf.h> 137 #include <sys/cmn_err.h> 138 #include <sys/controlregs.h> 139 #include <sys/cpupart.h> 140 #include <sys/cpuvar.h> 141 #include <sys/lgrp.h> 142 #include <sys/machsystm.h> 143 #include <sys/memlist.h> 144 #include <sys/memnode.h> 145 #include <sys/mman.h> 146 #include <sys/pci_cfgspace.h> 147 #include <sys/pci_impl.h> 148 #include <sys/param.h> 149 #include <sys/pghw.h> 150 #include <sys/promif.h> /* for prom_printf() */ 151 #include <sys/sysmacros.h> 152 #include <sys/systm.h> 153 #include <sys/thread.h> 154 #include <sys/types.h> 155 #include <sys/var.h> 156 #include <sys/x86_archext.h> /* for x86_feature and X86_AMD */ 157 #include <vm/hat_i86.h> 158 #include <vm/seg_kmem.h> 159 #include <vm/vm_dep.h> 160 161 #include "acpi_fw.h" /* for SRAT and SLIT */ 162 163 164 #define MAX_NODES 8 165 #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) 166 167 /* 168 * Constants for configuring probing 169 */ 170 #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ 171 #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ 172 #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ 173 174 /* 175 * Flags for probing 176 */ 177 #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ 178 #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ 179 #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ 180 181 /* 182 * Hash proximity domain ID into node to domain mapping table "mod" number of 183 * nodes to minimize span of entries used and try to have lowest numbered 184 * proximity domain be node 0 185 */ 186 #define NODE_DOMAIN_HASH(domain, node_cnt) \ 187 ((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \ 188 ((domain) - lgrp_plat_prox_domain_min) % node_cnt) 189 190 191 /* 192 * CPU to node ID mapping structure (only used with SRAT) 193 */ 194 typedef struct cpu_node_map { 195 int exists; 196 uint_t node; 197 uint32_t apicid; 198 uint32_t prox_domain; 199 } cpu_node_map_t; 200 201 /* 202 * Latency statistics 203 */ 204 typedef struct lgrp_plat_latency_stats { 205 hrtime_t latencies[MAX_NODES][MAX_NODES]; 206 hrtime_t latency_max; 207 hrtime_t latency_min; 208 } lgrp_plat_latency_stats_t; 209 210 /* 211 * Memory configuration for probing 212 */ 213 typedef struct lgrp_plat_probe_mem_config { 214 size_t probe_memsize; /* how much memory to probe per node */ 215 caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ 216 pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ 217 } lgrp_plat_probe_mem_config_t; 218 219 /* 220 * Statistics kept for probing 221 */ 222 typedef struct lgrp_plat_probe_stats { 223 hrtime_t flush_cost; 224 hrtime_t probe_cost; 225 hrtime_t probe_cost_total; 226 hrtime_t probe_error_code; 227 hrtime_t probe_errors[MAX_NODES][MAX_NODES]; 228 int probe_suspect[MAX_NODES][MAX_NODES]; 229 hrtime_t probe_max[MAX_NODES][MAX_NODES]; 230 hrtime_t probe_min[MAX_NODES][MAX_NODES]; 231 } lgrp_plat_probe_stats_t; 232 233 /* 234 * Node to proximity domain ID mapping structure (only used with SRAT) 235 */ 236 typedef struct node_domain_map { 237 int exists; 238 uint32_t prox_domain; 239 } node_domain_map_t; 240 241 /* 242 * Node ID and starting and ending page for physical memory in node 243 */ 244 typedef struct node_phys_addr_map { 245 pfn_t start; 246 pfn_t end; 247 int exists; 248 uint32_t prox_domain; 249 } node_phys_addr_map_t; 250 251 /* 252 * Number of CPUs for which we got APIC IDs 253 */ 254 static int lgrp_plat_apic_ncpus = 0; 255 256 /* 257 * CPU to node ID mapping table (only used for SRAT) 258 */ 259 static cpu_node_map_t lgrp_plat_cpu_node[NCPU]; 260 261 /* 262 * Latency statistics 263 */ 264 lgrp_plat_latency_stats_t lgrp_plat_lat_stats; 265 266 /* 267 * Whether memory is interleaved across nodes causing MPO to be disabled 268 */ 269 static int lgrp_plat_mem_intrlv = 0; 270 271 /* 272 * Node ID to proximity domain ID mapping table (only used for SRAT) 273 */ 274 static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; 275 276 /* 277 * Physical address range for memory in each node 278 */ 279 static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES]; 280 281 /* 282 * Statistics gotten from probing 283 */ 284 static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; 285 286 /* 287 * Memory configuration for probing 288 */ 289 static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; 290 291 /* 292 * Lowest proximity domain ID seen in ACPI SRAT 293 */ 294 static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX; 295 296 /* 297 * Error code from processing ACPI SRAT 298 */ 299 static int lgrp_plat_srat_error = 0; 300 301 /* 302 * Error code from processing ACPI SLIT 303 */ 304 static int lgrp_plat_slit_error = 0; 305 306 /* 307 * Allocate lgroup array statically 308 */ 309 static lgrp_t lgrp_space[NLGRP]; 310 static int nlgrps_alloc; 311 312 313 /* 314 * Enable finding and using minimum proximity domain ID when hashing 315 */ 316 int lgrp_plat_domain_min_enable = 1; 317 318 /* 319 * Number of nodes in system 320 */ 321 uint_t lgrp_plat_node_cnt = 1; 322 323 /* 324 * Enable sorting nodes in ascending order by starting physical address 325 */ 326 int lgrp_plat_node_sort_enable = 1; 327 328 /* 329 * Configuration Parameters for Probing 330 * - lgrp_plat_probe_flags Flags to specify enabling probing, probe 331 * operation, etc. 332 * - lgrp_plat_probe_nrounds How many rounds of probing to do 333 * - lgrp_plat_probe_nsamples Number of samples to take when probing each 334 * node 335 * - lgrp_plat_probe_nreads Number of times to read vendor ID from 336 * Northbridge for each probe 337 */ 338 uint_t lgrp_plat_probe_flags = 0; 339 int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; 340 int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; 341 int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; 342 343 /* 344 * Enable use of ACPI System Resource Affinity Table (SRAT) and System 345 * Locality Information Table (SLIT) 346 */ 347 int lgrp_plat_srat_enable = 1; 348 int lgrp_plat_slit_enable = 1; 349 350 /* 351 * mnode_xwa: set to non-zero value to initiate workaround if large pages are 352 * found to be crossing memory node boundaries. The workaround will eliminate 353 * a base size page at the end of each memory node boundary to ensure that 354 * a large page with constituent pages that span more than 1 memory node 355 * can never be formed. 356 * 357 */ 358 int mnode_xwa = 1; 359 360 /* 361 * Static array to hold lgroup statistics 362 */ 363 struct lgrp_stats lgrp_stats[NLGRP]; 364 365 366 /* 367 * Forward declarations of platform interface routines 368 */ 369 void plat_build_mem_nodes(struct memlist *list); 370 371 int plat_lgrphand_to_mem_node(lgrp_handle_t hand); 372 373 lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); 374 375 int plat_mnode_xcheck(pfn_t pfncnt); 376 377 int plat_pfn_to_mem_node(pfn_t pfn); 378 379 /* 380 * Forward declarations of lgroup platform interface routines 381 */ 382 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 383 384 void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); 385 386 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); 387 388 void lgrp_plat_init(void); 389 390 int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); 391 392 void lgrp_plat_main_init(void); 393 394 int lgrp_plat_max_lgrps(void); 395 396 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, 397 lgrp_mem_query_t query); 398 399 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); 400 401 void lgrp_plat_probe(void); 402 403 lgrp_handle_t lgrp_plat_root_hand(void); 404 405 406 /* 407 * Forward declarations of local routines 408 */ 409 static int is_opteron(void); 410 411 static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, 412 int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, 413 uint32_t domain); 414 415 static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node); 416 417 static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, 418 int node_cnt, uint32_t domain); 419 420 static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 421 lgrp_plat_latency_stats_t *lat_stats, 422 lgrp_plat_probe_stats_t *probe_stats); 423 424 static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 425 lgrp_plat_latency_stats_t *lat_stats); 426 427 static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); 428 429 static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, 430 int node_cnt, uint32_t domain); 431 432 static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain, 433 int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start, 434 uint64_t end, uint32_t domain); 435 436 static void lgrp_plat_node_sort(node_domain_map_t *node_domain, 437 int node_cnt, cpu_node_map_t *cpu_node, int cpu_count, 438 node_phys_addr_map_t *node_memory); 439 440 static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 441 lgrp_plat_probe_mem_config_t *probe_mem_config, 442 lgrp_plat_latency_stats_t *lat_stats, 443 lgrp_plat_probe_stats_t *probe_stats); 444 445 static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); 446 447 static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 448 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats); 449 450 static int lgrp_plat_process_srat(struct srat *tp, 451 uint32_t *prox_domain_min, node_domain_map_t *node_domain, 452 cpu_node_map_t *cpu_node, int cpu_count, 453 node_phys_addr_map_t *node_memory); 454 455 static int lgrp_plat_srat_domains(struct srat *tp, 456 uint32_t *prox_domain_min); 457 458 static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 459 lgrp_plat_latency_stats_t *lat_stats); 460 461 static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 462 node_phys_addr_map_t *node_memory); 463 464 static hrtime_t opt_probe_vendor(int dest_node, int nreads); 465 466 467 /* 468 * PLATFORM INTERFACE ROUTINES 469 */ 470 471 /* 472 * Configure memory nodes for machines with more than one node (ie NUMA) 473 */ 474 void 475 plat_build_mem_nodes(struct memlist *list) 476 { 477 pfn_t cur_start; /* start addr of subrange */ 478 pfn_t cur_end; /* end addr of subrange */ 479 pfn_t start; /* start addr of whole range */ 480 pfn_t end; /* end addr of whole range */ 481 pgcnt_t endcnt; /* pages to sacrifice */ 482 483 /* 484 * Boot install lists are arranged <addr, len>, ... 485 */ 486 while (list) { 487 int node; 488 489 start = list->address >> PAGESHIFT; 490 end = (list->address + list->size - 1) >> PAGESHIFT; 491 492 if (start > physmax) { 493 list = list->next; 494 continue; 495 } 496 if (end > physmax) 497 end = physmax; 498 499 /* 500 * When there is only one memnode, just add memory to memnode 501 */ 502 if (max_mem_nodes == 1) { 503 mem_node_add_slice(start, end); 504 list = list->next; 505 continue; 506 } 507 508 /* 509 * mem_node_add_slice() expects to get a memory range that 510 * is within one memnode, so need to split any memory range 511 * that spans multiple memnodes into subranges that are each 512 * contained within one memnode when feeding them to 513 * mem_node_add_slice() 514 */ 515 cur_start = start; 516 do { 517 node = plat_pfn_to_mem_node(cur_start); 518 519 /* 520 * Panic if DRAM address map registers or SRAT say 521 * memory in node doesn't exist or address from 522 * boot installed memory list entry isn't in this node. 523 * This shouldn't happen and rest of code can't deal 524 * with this if it does. 525 */ 526 if (node < 0 || node >= lgrp_plat_node_cnt || 527 !lgrp_plat_node_memory[node].exists || 528 cur_start < lgrp_plat_node_memory[node].start || 529 cur_start > lgrp_plat_node_memory[node].end) { 530 cmn_err(CE_PANIC, "Don't know which memnode " 531 "to add installed memory address 0x%lx\n", 532 cur_start); 533 } 534 535 /* 536 * End of current subrange should not span memnodes 537 */ 538 cur_end = end; 539 endcnt = 0; 540 if (lgrp_plat_node_memory[node].exists && 541 cur_end > lgrp_plat_node_memory[node].end) { 542 cur_end = lgrp_plat_node_memory[node].end; 543 if (mnode_xwa > 1) { 544 /* 545 * sacrifice the last page in each 546 * node to eliminate large pages 547 * that span more than 1 memory node. 548 */ 549 endcnt = 1; 550 physinstalled--; 551 } 552 } 553 554 mem_node_add_slice(cur_start, cur_end - endcnt); 555 556 /* 557 * Next subrange starts after end of current one 558 */ 559 cur_start = cur_end + 1; 560 } while (cur_end < end); 561 562 list = list->next; 563 } 564 mem_node_physalign = 0; 565 mem_node_pfn_shift = 0; 566 } 567 568 569 int 570 plat_lgrphand_to_mem_node(lgrp_handle_t hand) 571 { 572 if (max_mem_nodes == 1) 573 return (0); 574 575 return ((int)hand); 576 } 577 578 579 /* 580 * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt 581 * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if 582 * a crossing is found and returns 0 otherwise. 583 */ 584 int 585 plat_mnode_xcheck(pfn_t pfncnt) 586 { 587 int node, prevnode = -1, basenode; 588 pfn_t ea, sa; 589 590 for (node = 0; node < lgrp_plat_node_cnt; node++) { 591 592 if (lgrp_plat_node_memory[node].exists == 0) 593 continue; 594 595 if (prevnode == -1) { 596 prevnode = node; 597 basenode = node; 598 continue; 599 } 600 601 /* assume x86 node pfn ranges are in increasing order */ 602 ASSERT(lgrp_plat_node_memory[node].start > 603 lgrp_plat_node_memory[prevnode].end); 604 605 /* 606 * continue if the starting address of node is not contiguous 607 * with the previous node. 608 */ 609 610 if (lgrp_plat_node_memory[node].start != 611 (lgrp_plat_node_memory[prevnode].end + 1)) { 612 basenode = node; 613 prevnode = node; 614 continue; 615 } 616 617 /* check if the starting address of node is pfncnt aligned */ 618 if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) { 619 620 /* 621 * at this point, node starts at an unaligned boundary 622 * and is contiguous with the previous node(s) to 623 * basenode. Check if there is an aligned contiguous 624 * range of length pfncnt that crosses this boundary. 625 */ 626 627 sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end, 628 pfncnt); 629 ea = P2ROUNDUP((lgrp_plat_node_memory[node].start), 630 pfncnt); 631 632 ASSERT((ea - sa) == pfncnt); 633 if (sa >= lgrp_plat_node_memory[basenode].start && 634 ea <= (lgrp_plat_node_memory[node].end + 1)) { 635 /* 636 * large page found to cross mnode boundary. 637 * Return Failure if workaround not enabled. 638 */ 639 if (mnode_xwa == 0) 640 return (1); 641 mnode_xwa++; 642 } 643 } 644 prevnode = node; 645 } 646 return (0); 647 } 648 649 650 lgrp_handle_t 651 plat_mem_node_to_lgrphand(int mnode) 652 { 653 if (max_mem_nodes == 1) 654 return (LGRP_DEFAULT_HANDLE); 655 656 return ((lgrp_handle_t)mnode); 657 } 658 659 660 int 661 plat_pfn_to_mem_node(pfn_t pfn) 662 { 663 int node; 664 665 if (max_mem_nodes == 1) 666 return (0); 667 668 for (node = 0; node < lgrp_plat_node_cnt; node++) { 669 /* 670 * Skip nodes with no memory 671 */ 672 if (!lgrp_plat_node_memory[node].exists) 673 continue; 674 675 if (pfn >= lgrp_plat_node_memory[node].start && 676 pfn <= lgrp_plat_node_memory[node].end) 677 return (node); 678 } 679 680 /* 681 * Didn't find memnode where this PFN lives which should never happen 682 */ 683 ASSERT(node < lgrp_plat_node_cnt); 684 return (-1); 685 } 686 687 688 /* 689 * LGROUP PLATFORM INTERFACE ROUTINES 690 */ 691 692 /* 693 * Allocate additional space for an lgroup. 694 */ 695 /* ARGSUSED */ 696 lgrp_t * 697 lgrp_plat_alloc(lgrp_id_t lgrpid) 698 { 699 lgrp_t *lgrp; 700 701 lgrp = &lgrp_space[nlgrps_alloc++]; 702 if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) 703 return (NULL); 704 return (lgrp); 705 } 706 707 708 /* 709 * Platform handling for (re)configuration changes 710 */ 711 /* ARGSUSED */ 712 void 713 lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) 714 { 715 } 716 717 718 /* 719 * Return the platform handle for the lgroup containing the given CPU 720 */ 721 /* ARGSUSED */ 722 lgrp_handle_t 723 lgrp_plat_cpu_to_hand(processorid_t id) 724 { 725 lgrp_handle_t hand; 726 727 if (lgrp_plat_node_cnt == 1) 728 return (LGRP_DEFAULT_HANDLE); 729 730 hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], 731 lgrp_plat_cpu_node); 732 733 ASSERT(hand != (lgrp_handle_t)-1); 734 if (hand == (lgrp_handle_t)-1) 735 return (LGRP_NULL_HANDLE); 736 737 return (hand); 738 } 739 740 741 /* 742 * Platform-specific initialization of lgroups 743 */ 744 void 745 lgrp_plat_init(void) 746 { 747 #if defined(__xpv) 748 /* 749 * XXPV For now, the hypervisor treats all memory equally. 750 */ 751 lgrp_plat_node_cnt = max_mem_nodes = 1; 752 #else /* __xpv */ 753 uint_t probe_op; 754 u_longlong_t value; 755 756 /* 757 * Get boot property for lgroup topology height limit 758 */ 759 if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0) 760 (void) lgrp_topo_ht_limit_set((int)value); 761 762 /* 763 * Get boot property for enabling/disabling SRAT 764 */ 765 if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0) 766 lgrp_plat_srat_enable = (int)value; 767 768 /* 769 * Get boot property for enabling/disabling SLIT 770 */ 771 if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0) 772 lgrp_plat_slit_enable = (int)value; 773 774 /* 775 * Initialize as a UMA machine 776 */ 777 if (lgrp_topo_ht_limit() == 1) { 778 lgrp_plat_node_cnt = max_mem_nodes = 1; 779 return; 780 } 781 782 /* 783 * Read boot property with CPU to APIC ID mapping table/array and fill 784 * in CPU to node ID mapping table with APIC ID for each CPU 785 */ 786 lgrp_plat_apic_ncpus = 787 lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); 788 789 /* 790 * Determine which CPUs and memory are local to each other and number 791 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) 792 */ 793 if (lgrp_plat_apic_ncpus > 0) { 794 int retval; 795 796 retval = lgrp_plat_process_srat(srat_ptr, 797 &lgrp_plat_prox_domain_min, 798 lgrp_plat_node_domain, lgrp_plat_cpu_node, 799 lgrp_plat_apic_ncpus, lgrp_plat_node_memory); 800 if (retval <= 0) { 801 lgrp_plat_srat_error = retval; 802 lgrp_plat_node_cnt = 1; 803 } else { 804 lgrp_plat_srat_error = 0; 805 lgrp_plat_node_cnt = retval; 806 } 807 } 808 809 /* 810 * Try to use PCI config space registers on Opteron if there's an error 811 * processing CPU to APIC ID mapping or SRAT 812 */ 813 if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && 814 is_opteron()) 815 opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, 816 lgrp_plat_node_memory); 817 818 /* 819 * Don't bother to setup system for multiple lgroups and only use one 820 * memory node when memory is interleaved between any nodes or there is 821 * only one NUMA node 822 * 823 * NOTE: May need to change this for Dynamic Reconfiguration (DR) 824 * when and if it happens for x86/x64 825 */ 826 if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { 827 lgrp_plat_node_cnt = max_mem_nodes = 1; 828 (void) lgrp_topo_ht_limit_set(1); 829 return; 830 } 831 832 /* 833 * Leaf lgroups on x86/x64 architectures contain one physical 834 * processor chip. Tune lgrp_expand_proc_thresh and 835 * lgrp_expand_proc_diff so that lgrp_choose() will spread 836 * things out aggressively. 837 */ 838 lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; 839 lgrp_expand_proc_diff = 0; 840 841 /* 842 * There should be one memnode (physical page free list(s)) for 843 * each node 844 */ 845 max_mem_nodes = lgrp_plat_node_cnt; 846 847 /* 848 * Initialize min and max latency before reading SLIT or probing 849 */ 850 lgrp_plat_lat_stats.latency_min = -1; 851 lgrp_plat_lat_stats.latency_max = 0; 852 853 /* 854 * Determine how far each NUMA node is from each other by 855 * reading ACPI System Locality Information Table (SLIT) if it 856 * exists 857 */ 858 lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, 859 lgrp_plat_node_cnt, lgrp_plat_node_memory, 860 &lgrp_plat_lat_stats); 861 if (lgrp_plat_slit_error == 0) 862 return; 863 864 /* 865 * Probe to determine latency between NUMA nodes when SLIT 866 * doesn't exist or make sense 867 */ 868 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; 869 870 /* 871 * Specify whether to probe using vendor ID register or page copy 872 * if hasn't been specified already or is overspecified 873 */ 874 probe_op = lgrp_plat_probe_flags & 875 (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 876 877 if (probe_op == 0 || 878 probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { 879 lgrp_plat_probe_flags &= 880 ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 881 if (is_opteron()) 882 lgrp_plat_probe_flags |= 883 LGRP_PLAT_PROBE_VENDOR; 884 else 885 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; 886 } 887 888 /* 889 * Probing errors can mess up the lgroup topology and 890 * force us fall back to a 2 level lgroup topology. 891 * Here we bound how tall the lgroup topology can grow 892 * in hopes of avoiding any anamolies in probing from 893 * messing up the lgroup topology by limiting the 894 * accuracy of the latency topology. 895 * 896 * Assume that nodes will at least be configured in a 897 * ring, so limit height of lgroup topology to be less 898 * than number of nodes on a system with 4 or more 899 * nodes 900 */ 901 if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == 902 lgrp_topo_ht_limit_default()) 903 (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); 904 #endif /* __xpv */ 905 } 906 907 908 /* 909 * Return latency between "from" and "to" lgroups 910 * 911 * This latency number can only be used for relative comparison 912 * between lgroups on the running system, cannot be used across platforms, 913 * and may not reflect the actual latency. It is platform and implementation 914 * specific, so platform gets to decide its value. It would be nice if the 915 * number was at least proportional to make comparisons more meaningful though. 916 */ 917 /* ARGSUSED */ 918 int 919 lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) 920 { 921 lgrp_handle_t src, dest; 922 int node; 923 924 if (max_mem_nodes == 1) 925 return (0); 926 927 /* 928 * Return max latency for root lgroup 929 */ 930 if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) 931 return (lgrp_plat_lat_stats.latency_max); 932 933 src = from; 934 dest = to; 935 936 /* 937 * Return 0 for nodes (lgroup platform handles) out of range 938 */ 939 if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES) 940 return (0); 941 942 /* 943 * Probe from current CPU if its lgroup latencies haven't been set yet 944 * and we are trying to get latency from current CPU to some node 945 */ 946 node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 947 ASSERT(node >= 0 && node < lgrp_plat_node_cnt); 948 if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src) 949 lgrp_plat_probe(); 950 951 return (lgrp_plat_lat_stats.latencies[src][dest]); 952 } 953 954 955 /* 956 * Platform-specific initialization 957 */ 958 void 959 lgrp_plat_main_init(void) 960 { 961 int curnode; 962 int ht_limit; 963 int i; 964 965 /* 966 * Print a notice that MPO is disabled when memory is interleaved 967 * across nodes....Would do this when it is discovered, but can't 968 * because it happens way too early during boot.... 969 */ 970 if (lgrp_plat_mem_intrlv) 971 cmn_err(CE_NOTE, 972 "MPO disabled because memory is interleaved\n"); 973 974 /* 975 * Don't bother to do any probing if it is disabled, there is only one 976 * node, or the height of the lgroup topology less than or equal to 2 977 */ 978 ht_limit = lgrp_topo_ht_limit(); 979 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 980 max_mem_nodes == 1 || ht_limit <= 2) { 981 /* 982 * Setup lgroup latencies for 2 level lgroup topology 983 * (ie. local and remote only) if they haven't been set yet 984 */ 985 if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && 986 lgrp_plat_lat_stats.latency_max == 0) 987 lgrp_plat_2level_setup(lgrp_plat_node_memory, 988 &lgrp_plat_lat_stats); 989 return; 990 } 991 992 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 993 /* 994 * Should have been able to probe from CPU 0 when it was added 995 * to lgroup hierarchy, but may not have been able to then 996 * because it happens so early in boot that gethrtime() hasn't 997 * been initialized. (:-( 998 */ 999 curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 1000 ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt); 1001 if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) 1002 lgrp_plat_probe(); 1003 1004 return; 1005 } 1006 1007 /* 1008 * When probing memory, use one page for every sample to determine 1009 * lgroup topology and taking multiple samples 1010 */ 1011 if (lgrp_plat_probe_mem_config.probe_memsize == 0) 1012 lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * 1013 lgrp_plat_probe_nsamples; 1014 1015 /* 1016 * Map memory in each node needed for probing to determine latency 1017 * topology 1018 */ 1019 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1020 int mnode; 1021 1022 /* 1023 * Skip this node and leave its probe page NULL 1024 * if it doesn't have any memory 1025 */ 1026 mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i); 1027 if (!mem_node_config[mnode].exists) { 1028 lgrp_plat_probe_mem_config.probe_va[i] = NULL; 1029 continue; 1030 } 1031 1032 /* 1033 * Allocate one kernel virtual page 1034 */ 1035 lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, 1036 lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); 1037 if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { 1038 cmn_err(CE_WARN, 1039 "lgrp_plat_main_init: couldn't allocate memory"); 1040 return; 1041 } 1042 1043 /* 1044 * Get PFN for first page in each node 1045 */ 1046 lgrp_plat_probe_mem_config.probe_pfn[i] = 1047 mem_node_config[mnode].physbase; 1048 1049 /* 1050 * Map virtual page to first page in node 1051 */ 1052 hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], 1053 lgrp_plat_probe_mem_config.probe_memsize, 1054 lgrp_plat_probe_mem_config.probe_pfn[i], 1055 PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, 1056 HAT_LOAD_NOCONSIST); 1057 } 1058 1059 /* 1060 * Probe from current CPU 1061 */ 1062 lgrp_plat_probe(); 1063 } 1064 1065 1066 /* 1067 * Return the maximum number of lgrps supported by the platform. 1068 * Before lgrp topology is known it returns an estimate based on the number of 1069 * nodes. Once topology is known it returns the actual maximim number of lgrps 1070 * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and 1071 * dynamic addition of new nodes, this number may not grow during system 1072 * lifetime (yet). 1073 */ 1074 int 1075 lgrp_plat_max_lgrps(void) 1076 { 1077 return (lgrp_topo_initialized ? 1078 lgrp_alloc_max + 1 : 1079 lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); 1080 } 1081 1082 1083 /* 1084 * Return the number of free pages in an lgroup. 1085 * 1086 * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize 1087 * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the 1088 * number of allocatable base pagesize pages corresponding to the 1089 * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) 1090 * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical 1091 * memory installed, regardless of whether or not it's usable. 1092 */ 1093 pgcnt_t 1094 lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) 1095 { 1096 int mnode; 1097 pgcnt_t npgs = (pgcnt_t)0; 1098 extern struct memlist *phys_avail; 1099 extern struct memlist *phys_install; 1100 1101 1102 if (plathand == LGRP_DEFAULT_HANDLE) 1103 return (lgrp_plat_mem_size_default(plathand, query)); 1104 1105 if (plathand != LGRP_NULL_HANDLE) { 1106 mnode = plat_lgrphand_to_mem_node(plathand); 1107 if (mnode >= 0 && mem_node_config[mnode].exists) { 1108 switch (query) { 1109 case LGRP_MEM_SIZE_FREE: 1110 npgs = MNODE_PGCNT(mnode); 1111 break; 1112 case LGRP_MEM_SIZE_AVAIL: 1113 npgs = mem_node_memlist_pages(mnode, 1114 phys_avail); 1115 break; 1116 case LGRP_MEM_SIZE_INSTALL: 1117 npgs = mem_node_memlist_pages(mnode, 1118 phys_install); 1119 break; 1120 default: 1121 break; 1122 } 1123 } 1124 } 1125 return (npgs); 1126 } 1127 1128 1129 /* 1130 * Return the platform handle of the lgroup that contains the physical memory 1131 * corresponding to the given page frame number 1132 */ 1133 /* ARGSUSED */ 1134 lgrp_handle_t 1135 lgrp_plat_pfn_to_hand(pfn_t pfn) 1136 { 1137 int mnode; 1138 1139 if (max_mem_nodes == 1) 1140 return (LGRP_DEFAULT_HANDLE); 1141 1142 if (pfn > physmax) 1143 return (LGRP_NULL_HANDLE); 1144 1145 mnode = plat_pfn_to_mem_node(pfn); 1146 if (mnode < 0) 1147 return (LGRP_NULL_HANDLE); 1148 1149 return (MEM_NODE_2_LGRPHAND(mnode)); 1150 } 1151 1152 1153 /* 1154 * Probe memory in each node from current CPU to determine latency topology 1155 * 1156 * The probing code will probe the vendor ID register on the Northbridge of 1157 * Opteron processors and probe memory for other processors by default. 1158 * 1159 * Since probing is inherently error prone, the code takes laps across all the 1160 * nodes probing from each node to each of the other nodes some number of 1161 * times. Furthermore, each node is probed some number of times before moving 1162 * onto the next one during each lap. The minimum latency gotten between nodes 1163 * is kept as the latency between the nodes. 1164 * 1165 * After all that, the probe times are adjusted by normalizing values that are 1166 * close to each other and local latencies are made the same. Lastly, the 1167 * latencies are verified to make sure that certain conditions are met (eg. 1168 * local < remote, latency(a, b) == latency(b, a), etc.). 1169 * 1170 * If any of the conditions aren't met, the code will export a NUMA 1171 * configuration with the local CPUs and memory given by the SRAT or PCI config 1172 * space registers and one remote memory latency since it can't tell exactly 1173 * how far each node is from each other. 1174 */ 1175 void 1176 lgrp_plat_probe(void) 1177 { 1178 int from; 1179 int i; 1180 lgrp_plat_latency_stats_t *lat_stats; 1181 boolean_t probed; 1182 hrtime_t probe_time; 1183 int to; 1184 1185 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 1186 max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) 1187 return; 1188 1189 /* 1190 * Determine ID of node containing current CPU 1191 */ 1192 from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 1193 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 1194 if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) 1195 ASSERT(lgrp_plat_node_domain[from].exists); 1196 1197 /* 1198 * Don't need to probe if got times already 1199 */ 1200 lat_stats = &lgrp_plat_lat_stats; 1201 if (lat_stats->latencies[from][from] != 0) 1202 return; 1203 1204 /* 1205 * Read vendor ID in Northbridge or read and write page(s) 1206 * in each node from current CPU and remember how long it takes, 1207 * so we can build latency topology of machine later. 1208 * This should approximate the memory latency between each node. 1209 */ 1210 probed = B_FALSE; 1211 for (i = 0; i < lgrp_plat_probe_nrounds; i++) { 1212 for (to = 0; to < lgrp_plat_node_cnt; to++) { 1213 /* 1214 * Get probe time and skip over any nodes that can't be 1215 * probed yet or don't have memory 1216 */ 1217 probe_time = lgrp_plat_probe_time(to, 1218 lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config, 1219 &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); 1220 if (probe_time == 0) 1221 continue; 1222 1223 probed = B_TRUE; 1224 1225 /* 1226 * Keep lowest probe time as latency between nodes 1227 */ 1228 if (lat_stats->latencies[from][to] == 0 || 1229 probe_time < lat_stats->latencies[from][to]) 1230 lat_stats->latencies[from][to] = probe_time; 1231 1232 /* 1233 * Update overall minimum and maximum probe times 1234 * across all nodes 1235 */ 1236 if (probe_time < lat_stats->latency_min || 1237 lat_stats->latency_min == -1) 1238 lat_stats->latency_min = probe_time; 1239 if (probe_time > lat_stats->latency_max) 1240 lat_stats->latency_max = probe_time; 1241 } 1242 } 1243 1244 /* 1245 * Bail out if weren't able to probe any nodes from current CPU 1246 */ 1247 if (probed == B_FALSE) 1248 return; 1249 1250 /* 1251 * - Fix up latencies such that local latencies are same, 1252 * latency(i, j) == latency(j, i), etc. (if possible) 1253 * 1254 * - Verify that latencies look ok 1255 * 1256 * - Fallback to just optimizing for local and remote if 1257 * latencies didn't look right 1258 */ 1259 lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats, 1260 &lgrp_plat_probe_stats); 1261 lgrp_plat_probe_stats.probe_error_code = 1262 lgrp_plat_latency_verify(lgrp_plat_node_memory, 1263 &lgrp_plat_lat_stats); 1264 if (lgrp_plat_probe_stats.probe_error_code) 1265 lgrp_plat_2level_setup(lgrp_plat_node_memory, 1266 &lgrp_plat_lat_stats); 1267 } 1268 1269 1270 /* 1271 * Return platform handle for root lgroup 1272 */ 1273 lgrp_handle_t 1274 lgrp_plat_root_hand(void) 1275 { 1276 return (LGRP_DEFAULT_HANDLE); 1277 } 1278 1279 1280 /* 1281 * INTERNAL ROUTINES 1282 */ 1283 1284 1285 /* 1286 * Update CPU to node mapping for given CPU and proximity domain (and returns 1287 * negative numbers for errors and positive ones for success) 1288 */ 1289 static int 1290 lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, 1291 cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) 1292 { 1293 uint_t i; 1294 int node; 1295 1296 /* 1297 * Get node number for proximity domain 1298 */ 1299 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1300 if (node == -1) { 1301 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1302 domain); 1303 if (node == -1) 1304 return (-1); 1305 } 1306 1307 /* 1308 * Search for entry with given APIC ID and fill in its node and 1309 * proximity domain IDs (if they haven't been set already) 1310 */ 1311 for (i = 0; i < nentries; i++) { 1312 /* 1313 * Skip nonexistent entries and ones without matching APIC ID 1314 */ 1315 if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) 1316 continue; 1317 1318 /* 1319 * Just return if entry completely and correctly filled in 1320 * already 1321 */ 1322 if (cpu_node[i].prox_domain == domain && 1323 cpu_node[i].node == node) 1324 return (1); 1325 1326 /* 1327 * Fill in node and proximity domain IDs 1328 */ 1329 cpu_node[i].prox_domain = domain; 1330 cpu_node[i].node = node; 1331 1332 return (0); 1333 } 1334 1335 /* 1336 * Return error when entry for APIC ID wasn't found in table 1337 */ 1338 return (-2); 1339 } 1340 1341 1342 /* 1343 * Get node ID for given CPU 1344 */ 1345 static int 1346 lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node) 1347 { 1348 processorid_t cpuid; 1349 1350 if (cp == NULL) 1351 return (-1); 1352 1353 cpuid = cp->cpu_id; 1354 if (cpuid < 0 || cpuid >= max_ncpus) 1355 return (-1); 1356 1357 /* 1358 * SRAT doesn't exist, isn't enabled, or there was an error processing 1359 * it, so return chip ID for Opteron and -1 otherwise. 1360 */ 1361 if (srat_ptr == NULL || !lgrp_plat_srat_enable || 1362 lgrp_plat_srat_error) { 1363 if (is_opteron()) 1364 return (pg_plat_hw_instance_id(cp, PGHW_CHIP)); 1365 return (-1); 1366 } 1367 1368 /* 1369 * Return -1 when CPU to node ID mapping entry doesn't exist for given 1370 * CPU 1371 */ 1372 if (!cpu_node[cpuid].exists) 1373 return (-1); 1374 1375 return (cpu_node[cpuid].node); 1376 } 1377 1378 1379 /* 1380 * Return node number for given proximity domain/system locality 1381 */ 1382 static int 1383 lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, 1384 uint32_t domain) 1385 { 1386 uint_t node; 1387 uint_t start; 1388 1389 /* 1390 * Hash proximity domain ID into node to domain mapping table (array), 1391 * search for entry with matching proximity domain ID, and return index 1392 * of matching entry as node ID. 1393 */ 1394 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1395 do { 1396 if (node_domain[node].prox_domain == domain && 1397 node_domain[node].exists) 1398 return (node); 1399 node = (node + 1) % node_cnt; 1400 } while (node != start); 1401 return (-1); 1402 } 1403 1404 1405 /* 1406 * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to 1407 * be considered same 1408 */ 1409 #define LGRP_LAT_TOLERANCE_SHIFT 4 1410 1411 int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; 1412 1413 1414 /* 1415 * Adjust latencies between nodes to be symmetric, normalize latencies between 1416 * any nodes that are within some tolerance to be same, and make local 1417 * latencies be same 1418 */ 1419 static void 1420 lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 1421 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 1422 { 1423 int i; 1424 int j; 1425 int k; 1426 int l; 1427 u_longlong_t max; 1428 u_longlong_t min; 1429 u_longlong_t t; 1430 u_longlong_t t1; 1431 u_longlong_t t2; 1432 const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; 1433 int lat_corrected[MAX_NODES][MAX_NODES]; 1434 1435 /* 1436 * Nothing to do when this is an UMA machine or don't have args needed 1437 */ 1438 if (max_mem_nodes == 1) 1439 return; 1440 1441 ASSERT(node_memory != NULL && lat_stats != NULL && 1442 probe_stats != NULL); 1443 1444 /* 1445 * Make sure that latencies are symmetric between any two nodes 1446 * (ie. latency(node0, node1) == latency(node1, node0)) 1447 */ 1448 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1449 if (!node_memory[i].exists) 1450 continue; 1451 1452 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1453 if (!node_memory[j].exists) 1454 continue; 1455 1456 t1 = lat_stats->latencies[i][j]; 1457 t2 = lat_stats->latencies[j][i]; 1458 1459 if (t1 == 0 || t2 == 0 || t1 == t2) 1460 continue; 1461 1462 /* 1463 * Latencies should be same 1464 * - Use minimum of two latencies which should be same 1465 * - Track suspect probe times not within tolerance of 1466 * min value 1467 * - Remember how much values are corrected by 1468 */ 1469 if (t1 > t2) { 1470 t = t2; 1471 probe_stats->probe_errors[i][j] += t1 - t2; 1472 if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { 1473 probe_stats->probe_suspect[i][j]++; 1474 probe_stats->probe_suspect[j][i]++; 1475 } 1476 } else if (t2 > t1) { 1477 t = t1; 1478 probe_stats->probe_errors[j][i] += t2 - t1; 1479 if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { 1480 probe_stats->probe_suspect[i][j]++; 1481 probe_stats->probe_suspect[j][i]++; 1482 } 1483 } 1484 1485 lat_stats->latencies[i][j] = 1486 lat_stats->latencies[j][i] = t; 1487 lgrp_config(cflag, t1, t); 1488 lgrp_config(cflag, t2, t); 1489 } 1490 } 1491 1492 /* 1493 * Keep track of which latencies get corrected 1494 */ 1495 for (i = 0; i < MAX_NODES; i++) 1496 for (j = 0; j < MAX_NODES; j++) 1497 lat_corrected[i][j] = 0; 1498 1499 /* 1500 * For every two nodes, see whether there is another pair of nodes which 1501 * are about the same distance apart and make the latencies be the same 1502 * if they are close enough together 1503 */ 1504 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1505 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1506 if (!node_memory[j].exists) 1507 continue; 1508 /* 1509 * Pick one pair of nodes (i, j) 1510 * and get latency between them 1511 */ 1512 t1 = lat_stats->latencies[i][j]; 1513 1514 /* 1515 * Skip this pair of nodes if there isn't a latency 1516 * for it yet 1517 */ 1518 if (t1 == 0) 1519 continue; 1520 1521 for (k = 0; k < lgrp_plat_node_cnt; k++) { 1522 for (l = 0; l < lgrp_plat_node_cnt; l++) { 1523 if (!node_memory[l].exists) 1524 continue; 1525 /* 1526 * Pick another pair of nodes (k, l) 1527 * not same as (i, j) and get latency 1528 * between them 1529 */ 1530 if (k == i && l == j) 1531 continue; 1532 1533 t2 = lat_stats->latencies[k][l]; 1534 1535 /* 1536 * Skip this pair of nodes if there 1537 * isn't a latency for it yet 1538 */ 1539 1540 if (t2 == 0) 1541 continue; 1542 1543 /* 1544 * Skip nodes (k, l) if they already 1545 * have same latency as (i, j) or 1546 * their latency isn't close enough to 1547 * be considered/made the same 1548 */ 1549 if (t1 == t2 || (t1 > t2 && t1 - t2 > 1550 t1 >> lgrp_plat_probe_lt_shift) || 1551 (t2 > t1 && t2 - t1 > 1552 t2 >> lgrp_plat_probe_lt_shift)) 1553 continue; 1554 1555 /* 1556 * Make latency(i, j) same as 1557 * latency(k, l), try to use latency 1558 * that has been adjusted already to get 1559 * more consistency (if possible), and 1560 * remember which latencies were 1561 * adjusted for next time 1562 */ 1563 if (lat_corrected[i][j]) { 1564 t = t1; 1565 lgrp_config(cflag, t2, t); 1566 t2 = t; 1567 } else if (lat_corrected[k][l]) { 1568 t = t2; 1569 lgrp_config(cflag, t1, t); 1570 t1 = t; 1571 } else { 1572 if (t1 > t2) 1573 t = t2; 1574 else 1575 t = t1; 1576 lgrp_config(cflag, t1, t); 1577 lgrp_config(cflag, t2, t); 1578 t1 = t2 = t; 1579 } 1580 1581 lat_stats->latencies[i][j] = 1582 lat_stats->latencies[k][l] = t; 1583 1584 lat_corrected[i][j] = 1585 lat_corrected[k][l] = 1; 1586 } 1587 } 1588 } 1589 } 1590 1591 /* 1592 * Local latencies should be same 1593 * - Find min and max local latencies 1594 * - Make all local latencies be minimum 1595 */ 1596 min = -1; 1597 max = 0; 1598 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1599 if (!node_memory[i].exists) 1600 continue; 1601 t = lat_stats->latencies[i][i]; 1602 if (t == 0) 1603 continue; 1604 if (min == -1 || t < min) 1605 min = t; 1606 if (t > max) 1607 max = t; 1608 } 1609 if (min != max) { 1610 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1611 int local; 1612 1613 if (!node_memory[i].exists) 1614 continue; 1615 1616 local = lat_stats->latencies[i][i]; 1617 if (local == 0) 1618 continue; 1619 1620 /* 1621 * Track suspect probe times that aren't within 1622 * tolerance of minimum local latency and how much 1623 * probe times are corrected by 1624 */ 1625 if (local - min > min >> lgrp_plat_probe_lt_shift) 1626 probe_stats->probe_suspect[i][i]++; 1627 1628 probe_stats->probe_errors[i][i] += local - min; 1629 1630 /* 1631 * Make local latencies be minimum 1632 */ 1633 lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); 1634 lat_stats->latencies[i][i] = min; 1635 } 1636 } 1637 1638 /* 1639 * Determine max probe time again since just adjusted latencies 1640 */ 1641 lat_stats->latency_max = 0; 1642 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1643 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1644 if (!node_memory[j].exists) 1645 continue; 1646 t = lat_stats->latencies[i][j]; 1647 if (t > lat_stats->latency_max) 1648 lat_stats->latency_max = t; 1649 } 1650 } 1651 } 1652 1653 1654 /* 1655 * Verify following about latencies between nodes: 1656 * 1657 * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) 1658 * - Local latencies same 1659 * - Local < remote 1660 * - Number of latencies seen is reasonable 1661 * - Number of occurrences of a given latency should be more than 1 1662 * 1663 * Returns: 1664 * 0 Success 1665 * -1 Not symmetric 1666 * -2 Local latencies not same 1667 * -3 Local >= remote 1668 */ 1669 static int 1670 lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 1671 lgrp_plat_latency_stats_t *lat_stats) 1672 { 1673 int i; 1674 int j; 1675 u_longlong_t t1; 1676 u_longlong_t t2; 1677 1678 ASSERT(node_memory != NULL && lat_stats != NULL); 1679 1680 /* 1681 * Nothing to do when this is an UMA machine, lgroup topology is 1682 * limited to 2 levels, or there aren't any probe times yet 1683 */ 1684 if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || 1685 lat_stats->latencies[0][0] == 0) 1686 return (0); 1687 1688 /* 1689 * Make sure that latencies are symmetric between any two nodes 1690 * (ie. latency(node0, node1) == latency(node1, node0)) 1691 */ 1692 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1693 if (!node_memory[i].exists) 1694 continue; 1695 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1696 if (!node_memory[j].exists) 1697 continue; 1698 t1 = lat_stats->latencies[i][j]; 1699 t2 = lat_stats->latencies[j][i]; 1700 1701 if (t1 == 0 || t2 == 0 || t1 == t2) 1702 continue; 1703 1704 return (-1); 1705 } 1706 } 1707 1708 /* 1709 * Local latencies should be same 1710 */ 1711 t1 = lat_stats->latencies[0][0]; 1712 for (i = 1; i < lgrp_plat_node_cnt; i++) { 1713 if (!node_memory[i].exists) 1714 continue; 1715 1716 t2 = lat_stats->latencies[i][i]; 1717 if (t2 == 0) 1718 continue; 1719 1720 if (t1 == 0) { 1721 t1 = t2; 1722 continue; 1723 } 1724 1725 if (t1 != t2) 1726 return (-2); 1727 } 1728 1729 /* 1730 * Local latencies should be less than remote 1731 */ 1732 if (t1) { 1733 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1734 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1735 if (!node_memory[j].exists) 1736 continue; 1737 t2 = lat_stats->latencies[i][j]; 1738 if (i == j || t2 == 0) 1739 continue; 1740 1741 if (t1 >= t2) 1742 return (-3); 1743 } 1744 } 1745 } 1746 1747 return (0); 1748 } 1749 1750 1751 /* 1752 * Return the number of free, allocatable, or installed 1753 * pages in an lgroup 1754 * This is a copy of the MAX_MEM_NODES == 1 version of the routine 1755 * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup 1756 */ 1757 /* ARGSUSED */ 1758 static pgcnt_t 1759 lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) 1760 { 1761 struct memlist *mlist; 1762 pgcnt_t npgs = 0; 1763 extern struct memlist *phys_avail; 1764 extern struct memlist *phys_install; 1765 1766 switch (query) { 1767 case LGRP_MEM_SIZE_FREE: 1768 return ((pgcnt_t)freemem); 1769 case LGRP_MEM_SIZE_AVAIL: 1770 memlist_read_lock(); 1771 for (mlist = phys_avail; mlist; mlist = mlist->next) 1772 npgs += btop(mlist->size); 1773 memlist_read_unlock(); 1774 return (npgs); 1775 case LGRP_MEM_SIZE_INSTALL: 1776 memlist_read_lock(); 1777 for (mlist = phys_install; mlist; mlist = mlist->next) 1778 npgs += btop(mlist->size); 1779 memlist_read_unlock(); 1780 return (npgs); 1781 default: 1782 return ((pgcnt_t)0); 1783 } 1784 } 1785 1786 1787 /* 1788 * Update node to proximity domain mappings for given domain and return node ID 1789 */ 1790 static int 1791 lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, 1792 uint32_t domain) 1793 { 1794 uint_t node; 1795 uint_t start; 1796 1797 /* 1798 * Hash proximity domain ID into node to domain mapping table (array) 1799 * and add entry for it into first non-existent or matching entry found 1800 */ 1801 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1802 do { 1803 /* 1804 * Entry doesn't exist yet, so create one for this proximity 1805 * domain and return node ID which is index into mapping table. 1806 */ 1807 if (!node_domain[node].exists) { 1808 node_domain[node].exists = 1; 1809 node_domain[node].prox_domain = domain; 1810 return (node); 1811 } 1812 1813 /* 1814 * Entry exists for this proximity domain already, so just 1815 * return node ID (index into table). 1816 */ 1817 if (node_domain[node].prox_domain == domain) 1818 return (node); 1819 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1820 } while (node != start); 1821 1822 /* 1823 * Ran out of supported number of entries which shouldn't happen.... 1824 */ 1825 ASSERT(node != start); 1826 return (-1); 1827 } 1828 1829 1830 /* 1831 * Update node memory information for given proximity domain with specified 1832 * starting and ending physical address range (and return positive numbers for 1833 * success and negative ones for errors) 1834 */ 1835 static int 1836 lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt, 1837 node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end, 1838 uint32_t domain) 1839 { 1840 int node; 1841 1842 /* 1843 * Get node number for proximity domain 1844 */ 1845 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1846 if (node == -1) { 1847 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1848 domain); 1849 if (node == -1) 1850 return (-1); 1851 } 1852 1853 /* 1854 * Create entry in table for node if it doesn't exist 1855 */ 1856 if (!node_memory[node].exists) { 1857 node_memory[node].exists = 1; 1858 node_memory[node].start = btop(start); 1859 node_memory[node].end = btop(end); 1860 node_memory[node].prox_domain = domain; 1861 return (0); 1862 } 1863 1864 /* 1865 * Entry already exists for this proximity domain 1866 * 1867 * There may be more than one SRAT memory entry for a domain, so we may 1868 * need to update existing start or end address for the node. 1869 */ 1870 if (node_memory[node].prox_domain == domain) { 1871 if (btop(start) < node_memory[node].start) 1872 node_memory[node].start = btop(start); 1873 if (btop(end) > node_memory[node].end) 1874 node_memory[node].end = btop(end); 1875 return (1); 1876 } 1877 return (-2); 1878 } 1879 1880 1881 /* 1882 * Have to sort node by starting physical address because VM system (physical 1883 * page free list management) assumes and expects memnodes to be sorted in 1884 * ascending order by physical address. If not, the kernel will panic in 1885 * potentially a number of different places. (:-( 1886 * NOTE: This workaround will not be sufficient if/when hotplugging memory is 1887 * supported on x86/x64. 1888 */ 1889 static void 1890 lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt, 1891 cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory) 1892 { 1893 boolean_t found; 1894 int i; 1895 int j; 1896 int n; 1897 boolean_t sorted; 1898 boolean_t swapped; 1899 1900 if (!lgrp_plat_node_sort_enable || node_cnt <= 1 || 1901 node_domain == NULL || node_memory == NULL) 1902 return; 1903 1904 /* 1905 * Sorted already? 1906 */ 1907 sorted = B_TRUE; 1908 for (i = 0; i < node_cnt - 1; i++) { 1909 /* 1910 * Skip entries that don't exist 1911 */ 1912 if (!node_memory[i].exists) 1913 continue; 1914 1915 /* 1916 * Try to find next existing entry to compare against 1917 */ 1918 found = B_FALSE; 1919 for (j = i + 1; j < node_cnt; j++) { 1920 if (node_memory[j].exists) { 1921 found = B_TRUE; 1922 break; 1923 } 1924 } 1925 1926 /* 1927 * Done if no more existing entries to compare against 1928 */ 1929 if (found == B_FALSE) 1930 break; 1931 1932 /* 1933 * Not sorted if starting address of current entry is bigger 1934 * than starting address of next existing entry 1935 */ 1936 if (node_memory[i].start > node_memory[j].start) { 1937 sorted = B_FALSE; 1938 break; 1939 } 1940 } 1941 1942 /* 1943 * Don't need to sort if sorted already 1944 */ 1945 if (sorted == B_TRUE) 1946 return; 1947 1948 /* 1949 * Just use bubble sort since number of nodes is small 1950 */ 1951 n = node_cnt; 1952 do { 1953 swapped = B_FALSE; 1954 n--; 1955 for (i = 0; i < n; i++) { 1956 /* 1957 * Skip entries that don't exist 1958 */ 1959 if (!node_memory[i].exists) 1960 continue; 1961 1962 /* 1963 * Try to find next existing entry to compare against 1964 */ 1965 found = B_FALSE; 1966 for (j = i + 1; j <= n; j++) { 1967 if (node_memory[j].exists) { 1968 found = B_TRUE; 1969 break; 1970 } 1971 } 1972 1973 /* 1974 * Done if no more existing entries to compare against 1975 */ 1976 if (found == B_FALSE) 1977 break; 1978 1979 if (node_memory[i].start > node_memory[j].start) { 1980 node_phys_addr_map_t save_addr; 1981 node_domain_map_t save_node; 1982 1983 /* 1984 * Swap node to proxmity domain ID assignments 1985 */ 1986 bcopy(&node_domain[i], &save_node, 1987 sizeof (node_domain_map_t)); 1988 bcopy(&node_domain[j], &node_domain[i], 1989 sizeof (node_domain_map_t)); 1990 bcopy(&save_node, &node_domain[j], 1991 sizeof (node_domain_map_t)); 1992 1993 /* 1994 * Swap node to physical memory assignments 1995 */ 1996 bcopy(&node_memory[i], &save_addr, 1997 sizeof (node_phys_addr_map_t)); 1998 bcopy(&node_memory[j], &node_memory[i], 1999 sizeof (node_phys_addr_map_t)); 2000 bcopy(&save_addr, &node_memory[j], 2001 sizeof (node_phys_addr_map_t)); 2002 swapped = B_TRUE; 2003 } 2004 } 2005 } while (swapped == B_TRUE); 2006 2007 /* 2008 * Check to make sure that CPUs assigned to correct node IDs now since 2009 * node to proximity domain ID assignments may have been changed above 2010 */ 2011 if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1) 2012 return; 2013 for (i = 0; i < cpu_count; i++) { 2014 int node; 2015 2016 node = lgrp_plat_domain_to_node(node_domain, node_cnt, 2017 cpu_node[i].prox_domain); 2018 if (cpu_node[i].node != node) 2019 cpu_node[i].node = node; 2020 } 2021 2022 } 2023 2024 2025 /* 2026 * Return time needed to probe from current CPU to memory in given node 2027 */ 2028 static hrtime_t 2029 lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 2030 lgrp_plat_probe_mem_config_t *probe_mem_config, 2031 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 2032 { 2033 caddr_t buf; 2034 hrtime_t elapsed; 2035 hrtime_t end; 2036 int from; 2037 int i; 2038 int ipl; 2039 hrtime_t max; 2040 hrtime_t min; 2041 hrtime_t start; 2042 extern int use_sse_pagecopy; 2043 2044 /* 2045 * Determine ID of node containing current CPU 2046 */ 2047 from = lgrp_plat_cpu_to_node(CPU, cpu_node); 2048 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 2049 2050 /* 2051 * Do common work for probing main memory 2052 */ 2053 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { 2054 /* 2055 * Skip probing any nodes without memory and 2056 * set probe time to 0 2057 */ 2058 if (probe_mem_config->probe_va[to] == NULL) { 2059 lat_stats->latencies[from][to] = 0; 2060 return (0); 2061 } 2062 2063 /* 2064 * Invalidate caches once instead of once every sample 2065 * which should cut cost of probing by a lot 2066 */ 2067 probe_stats->flush_cost = gethrtime(); 2068 invalidate_cache(); 2069 probe_stats->flush_cost = gethrtime() - 2070 probe_stats->flush_cost; 2071 probe_stats->probe_cost_total += probe_stats->flush_cost; 2072 } 2073 2074 /* 2075 * Probe from current CPU to given memory using specified operation 2076 * and take specified number of samples 2077 */ 2078 max = 0; 2079 min = -1; 2080 for (i = 0; i < lgrp_plat_probe_nsamples; i++) { 2081 probe_stats->probe_cost = gethrtime(); 2082 2083 /* 2084 * Can't measure probe time if gethrtime() isn't working yet 2085 */ 2086 if (probe_stats->probe_cost == 0 && gethrtime() == 0) 2087 return (0); 2088 2089 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 2090 /* 2091 * Measure how long it takes to read vendor ID from 2092 * Northbridge 2093 */ 2094 elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); 2095 } else { 2096 /* 2097 * Measure how long it takes to copy page 2098 * on top of itself 2099 */ 2100 buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); 2101 2102 kpreempt_disable(); 2103 ipl = splhigh(); 2104 start = gethrtime(); 2105 if (use_sse_pagecopy) 2106 hwblkpagecopy(buf, buf); 2107 else 2108 bcopy(buf, buf, PAGESIZE); 2109 end = gethrtime(); 2110 elapsed = end - start; 2111 splx(ipl); 2112 kpreempt_enable(); 2113 } 2114 2115 probe_stats->probe_cost = gethrtime() - 2116 probe_stats->probe_cost; 2117 probe_stats->probe_cost_total += probe_stats->probe_cost; 2118 2119 if (min == -1 || elapsed < min) 2120 min = elapsed; 2121 if (elapsed > max) 2122 max = elapsed; 2123 } 2124 2125 /* 2126 * Update minimum and maximum probe times between 2127 * these two nodes 2128 */ 2129 if (min < probe_stats->probe_min[from][to] || 2130 probe_stats->probe_min[from][to] == 0) 2131 probe_stats->probe_min[from][to] = min; 2132 2133 if (max > probe_stats->probe_max[from][to]) 2134 probe_stats->probe_max[from][to] = max; 2135 2136 return (min); 2137 } 2138 2139 2140 /* 2141 * Read boot property with CPU to APIC ID array, fill in CPU to node ID 2142 * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs. 2143 * 2144 * NOTE: This code assumes that CPU IDs are assigned in order that they appear 2145 * in in cpu_apicid_array boot property which is based on and follows 2146 * same ordering as processor list in ACPI MADT. If the code in 2147 * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns 2148 * CPU IDs ever changes, then this code will need to change too.... 2149 */ 2150 static int 2151 lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) 2152 { 2153 int boot_prop_len; 2154 char *boot_prop_name = BP_CPU_APICID_ARRAY; 2155 uint8_t cpu_apicid_array[UINT8_MAX + 1]; 2156 int i; 2157 int n; 2158 2159 /* 2160 * Nothing to do when no array to fill in or not enough CPUs 2161 */ 2162 if (cpu_node == NULL) 2163 return (-1); 2164 2165 /* 2166 * Check length of property value 2167 */ 2168 boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); 2169 if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array)) 2170 return (-2); 2171 2172 /* 2173 * Calculate number of entries in array and return when there's just 2174 * one CPU since that's not very interesting for NUMA 2175 */ 2176 n = boot_prop_len / sizeof (uint8_t); 2177 if (n == 1) 2178 return (-3); 2179 2180 /* 2181 * Get CPU to APIC ID property value 2182 */ 2183 if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) 2184 return (-4); 2185 2186 /* 2187 * Fill in CPU to node ID mapping table with APIC ID for each CPU 2188 */ 2189 for (i = 0; i < n; i++) { 2190 cpu_node[i].exists = 1; 2191 cpu_node[i].apicid = cpu_apicid_array[i]; 2192 } 2193 2194 /* 2195 * Return number of CPUs based on number of APIC IDs 2196 */ 2197 return (n); 2198 } 2199 2200 2201 /* 2202 * Read ACPI System Locality Information Table (SLIT) to determine how far each 2203 * NUMA node is from each other 2204 */ 2205 static int 2206 lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 2207 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats) 2208 { 2209 int i; 2210 int j; 2211 int localities; 2212 hrtime_t max; 2213 hrtime_t min; 2214 int retval; 2215 uint8_t *slit_entries; 2216 2217 if (tp == NULL || !lgrp_plat_slit_enable) 2218 return (1); 2219 2220 if (lat_stats == NULL) 2221 return (2); 2222 2223 localities = tp->number; 2224 if (localities != node_cnt) 2225 return (3); 2226 2227 min = lat_stats->latency_min; 2228 max = lat_stats->latency_max; 2229 2230 /* 2231 * Fill in latency matrix based on SLIT entries 2232 */ 2233 slit_entries = tp->entry; 2234 for (i = 0; i < localities; i++) { 2235 for (j = 0; j < localities; j++) { 2236 uint8_t latency; 2237 2238 latency = slit_entries[(i * localities) + j]; 2239 lat_stats->latencies[i][j] = latency; 2240 if (latency < min || min == -1) 2241 min = latency; 2242 if (latency > max) 2243 max = latency; 2244 } 2245 } 2246 2247 /* 2248 * Verify that latencies/distances given in SLIT look reasonable 2249 */ 2250 retval = lgrp_plat_latency_verify(node_memory, lat_stats); 2251 2252 if (retval) { 2253 /* 2254 * Reinitialize (zero) latency table since SLIT doesn't look 2255 * right 2256 */ 2257 for (i = 0; i < localities; i++) { 2258 for (j = 0; j < localities; j++) 2259 lat_stats->latencies[i][j] = 0; 2260 } 2261 } else { 2262 /* 2263 * Update min and max latencies seen since SLIT looks valid 2264 */ 2265 lat_stats->latency_min = min; 2266 lat_stats->latency_max = max; 2267 } 2268 2269 return (retval); 2270 } 2271 2272 2273 /* 2274 * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs 2275 * and memory are local to each other in the same NUMA node and return number 2276 * of nodes 2277 */ 2278 static int 2279 lgrp_plat_process_srat(struct srat *tp, uint32_t *prox_domain_min, 2280 node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, 2281 node_phys_addr_map_t *node_memory) 2282 { 2283 struct srat_item *srat_end; 2284 int i; 2285 struct srat_item *item; 2286 int node_cnt; 2287 int proc_entry_count; 2288 2289 /* 2290 * Nothing to do when no SRAT or disabled 2291 */ 2292 if (tp == NULL || !lgrp_plat_srat_enable) 2293 return (-1); 2294 2295 /* 2296 * Determine number of nodes by counting number of proximity domains in 2297 * SRAT and return if number of nodes is 1 or less since don't need to 2298 * read SRAT then 2299 */ 2300 node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min); 2301 if (node_cnt == 1) 2302 return (1); 2303 else if (node_cnt <= 0) 2304 return (-2); 2305 2306 /* 2307 * Walk through SRAT, examining each CPU and memory entry to determine 2308 * which CPUs and memory belong to which node. 2309 */ 2310 item = tp->list; 2311 srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2312 proc_entry_count = 0; 2313 while (item < srat_end) { 2314 uint32_t apic_id; 2315 uint32_t domain; 2316 uint64_t end; 2317 uint64_t length; 2318 uint64_t start; 2319 2320 switch (item->type) { 2321 case SRAT_PROCESSOR: /* CPU entry */ 2322 if (!(item->i.p.flags & SRAT_ENABLED) || 2323 cpu_node == NULL) 2324 break; 2325 2326 /* 2327 * Calculate domain (node) ID and fill in APIC ID to 2328 * domain/node mapping table 2329 */ 2330 domain = item->i.p.domain1; 2331 for (i = 0; i < 3; i++) { 2332 domain += item->i.p.domain2[i] << 2333 ((i + 1) * 8); 2334 } 2335 apic_id = item->i.p.apic_id; 2336 2337 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2338 cpu_node, cpu_count, apic_id, domain) < 0) 2339 return (-3); 2340 2341 proc_entry_count++; 2342 break; 2343 2344 case SRAT_MEMORY: /* memory entry */ 2345 if (!(item->i.m.flags & SRAT_ENABLED) || 2346 node_memory == NULL) 2347 break; 2348 2349 /* 2350 * Get domain (node) ID and fill in domain/node 2351 * to memory mapping table 2352 */ 2353 domain = item->i.m.domain; 2354 start = item->i.m.base_addr; 2355 length = item->i.m.len; 2356 end = start + length - 1; 2357 2358 if (lgrp_plat_node_memory_update(node_domain, node_cnt, 2359 node_memory, start, end, domain) < 0) 2360 return (-4); 2361 break; 2362 case SRAT_X2APIC: /* x2apic CPU entry */ 2363 if (!(item->i.xp.flags & SRAT_ENABLED) || 2364 cpu_node == NULL) 2365 break; 2366 2367 /* 2368 * Calculate domain (node) ID and fill in APIC ID to 2369 * domain/node mapping table 2370 */ 2371 domain = item->i.xp.domain; 2372 apic_id = item->i.xp.x2apic_id; 2373 2374 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2375 cpu_node, cpu_count, apic_id, domain) < 0) 2376 return (-3); 2377 2378 proc_entry_count++; 2379 break; 2380 2381 default: 2382 break; 2383 } 2384 2385 item = (struct srat_item *)((uintptr_t)item + item->len); 2386 } 2387 2388 /* 2389 * Should have seen at least as many SRAT processor entries as CPUs 2390 */ 2391 if (proc_entry_count < cpu_count) 2392 return (-5); 2393 2394 /* 2395 * Need to sort nodes by starting physical address since VM system 2396 * assumes and expects memnodes to be sorted in ascending order by 2397 * physical address 2398 */ 2399 lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count, 2400 node_memory); 2401 2402 return (node_cnt); 2403 } 2404 2405 2406 /* 2407 * Return number of proximity domains given in ACPI SRAT 2408 */ 2409 static int 2410 lgrp_plat_srat_domains(struct srat *tp, uint32_t *prox_domain_min) 2411 { 2412 int domain_cnt; 2413 uint32_t domain_min; 2414 struct srat_item *end; 2415 int i; 2416 struct srat_item *item; 2417 node_domain_map_t node_domain[MAX_NODES]; 2418 2419 2420 if (tp == NULL || !lgrp_plat_srat_enable) 2421 return (1); 2422 2423 /* 2424 * Walk through SRAT to find minimum proximity domain ID 2425 */ 2426 domain_min = UINT32_MAX; 2427 item = tp->list; 2428 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2429 while (item < end) { 2430 uint32_t domain; 2431 2432 switch (item->type) { 2433 case SRAT_PROCESSOR: /* CPU entry */ 2434 if (!(item->i.p.flags & SRAT_ENABLED)) { 2435 item = (struct srat_item *)((uintptr_t)item + 2436 item->len); 2437 continue; 2438 } 2439 domain = item->i.p.domain1; 2440 for (i = 0; i < 3; i++) { 2441 domain += item->i.p.domain2[i] << 2442 ((i + 1) * 8); 2443 } 2444 break; 2445 2446 case SRAT_MEMORY: /* memory entry */ 2447 if (!(item->i.m.flags & SRAT_ENABLED)) { 2448 item = (struct srat_item *)((uintptr_t)item + 2449 item->len); 2450 continue; 2451 } 2452 domain = item->i.m.domain; 2453 break; 2454 2455 case SRAT_X2APIC: /* x2apic CPU entry */ 2456 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2457 item = (struct srat_item *)((uintptr_t)item + 2458 item->len); 2459 continue; 2460 } 2461 domain = item->i.xp.domain; 2462 break; 2463 2464 default: 2465 item = (struct srat_item *)((uintptr_t)item + 2466 item->len); 2467 continue; 2468 } 2469 2470 /* 2471 * Keep track of minimum proximity domain ID 2472 */ 2473 if (domain < domain_min) 2474 domain_min = domain; 2475 2476 item = (struct srat_item *)((uintptr_t)item + item->len); 2477 } 2478 if (lgrp_plat_domain_min_enable && prox_domain_min != NULL) 2479 *prox_domain_min = domain_min; 2480 2481 /* 2482 * Walk through SRAT, examining each CPU and memory entry to determine 2483 * proximity domain ID for each. 2484 */ 2485 domain_cnt = 0; 2486 item = tp->list; 2487 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2488 bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); 2489 while (item < end) { 2490 uint32_t domain; 2491 boolean_t overflow; 2492 uint_t start; 2493 2494 switch (item->type) { 2495 case SRAT_PROCESSOR: /* CPU entry */ 2496 if (!(item->i.p.flags & SRAT_ENABLED)) { 2497 item = (struct srat_item *)((uintptr_t)item + 2498 item->len); 2499 continue; 2500 } 2501 domain = item->i.p.domain1; 2502 for (i = 0; i < 3; i++) { 2503 domain += item->i.p.domain2[i] << 2504 ((i + 1) * 8); 2505 } 2506 break; 2507 2508 case SRAT_MEMORY: /* memory entry */ 2509 if (!(item->i.m.flags & SRAT_ENABLED)) { 2510 item = (struct srat_item *)((uintptr_t)item + 2511 item->len); 2512 continue; 2513 } 2514 domain = item->i.m.domain; 2515 break; 2516 2517 case SRAT_X2APIC: /* x2apic CPU entry */ 2518 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2519 item = (struct srat_item *)((uintptr_t)item + 2520 item->len); 2521 continue; 2522 } 2523 domain = item->i.xp.domain; 2524 break; 2525 2526 default: 2527 item = (struct srat_item *)((uintptr_t)item + 2528 item->len); 2529 continue; 2530 } 2531 2532 /* 2533 * Count and keep track of which proximity domain IDs seen 2534 */ 2535 start = i = domain % MAX_NODES; 2536 overflow = B_TRUE; 2537 do { 2538 /* 2539 * Create entry for proximity domain and increment 2540 * count when no entry exists where proximity domain 2541 * hashed 2542 */ 2543 if (!node_domain[i].exists) { 2544 node_domain[i].exists = 1; 2545 node_domain[i].prox_domain = domain; 2546 domain_cnt++; 2547 overflow = B_FALSE; 2548 break; 2549 } 2550 2551 /* 2552 * Nothing to do when proximity domain seen already 2553 * and its entry exists 2554 */ 2555 if (node_domain[i].prox_domain == domain) { 2556 overflow = B_FALSE; 2557 break; 2558 } 2559 2560 /* 2561 * Entry exists where proximity domain hashed, but for 2562 * different proximity domain so keep search for empty 2563 * slot to put it or matching entry whichever comes 2564 * first. 2565 */ 2566 i = (i + 1) % MAX_NODES; 2567 } while (i != start); 2568 2569 /* 2570 * Didn't find empty or matching entry which means have more 2571 * proximity domains than supported nodes (:-( 2572 */ 2573 ASSERT(overflow != B_TRUE); 2574 if (overflow == B_TRUE) 2575 return (-1); 2576 2577 item = (struct srat_item *)((uintptr_t)item + item->len); 2578 } 2579 return (domain_cnt); 2580 } 2581 2582 2583 /* 2584 * Set lgroup latencies for 2 level lgroup topology 2585 */ 2586 static void 2587 lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 2588 lgrp_plat_latency_stats_t *lat_stats) 2589 { 2590 int i; 2591 2592 ASSERT(node_memory != NULL && lat_stats != NULL); 2593 2594 if (lgrp_plat_node_cnt >= 4) 2595 cmn_err(CE_NOTE, 2596 "MPO only optimizing for local and remote\n"); 2597 for (i = 0; i < lgrp_plat_node_cnt; i++) { 2598 int j; 2599 2600 if (!node_memory[i].exists) 2601 continue; 2602 for (j = 0; j < lgrp_plat_node_cnt; j++) { 2603 if (!node_memory[j].exists) 2604 continue; 2605 if (i == j) 2606 lat_stats->latencies[i][j] = 2; 2607 else 2608 lat_stats->latencies[i][j] = 3; 2609 } 2610 } 2611 lat_stats->latency_min = 2; 2612 lat_stats->latency_max = 3; 2613 lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); 2614 } 2615 2616 2617 /* 2618 * The following Opteron specific constants, macros, types, and routines define 2619 * PCI configuration space registers and how to read them to determine the NUMA 2620 * configuration of *supported* Opteron processors. They provide the same 2621 * information that may be gotten from the ACPI System Resource Affinity Table 2622 * (SRAT) if it exists on the machine of interest. 2623 * 2624 * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family 2625 * of interest describes all of these registers and their contents. The main 2626 * registers used by this code to determine the NUMA configuration of the 2627 * machine are the node ID register for the number of NUMA nodes and the DRAM 2628 * address map registers for the physical address range of each node. 2629 * 2630 * NOTE: The format and how to determine the NUMA configuration using PCI 2631 * config space registers may change or may not be supported in future 2632 * Opteron processor families. 2633 */ 2634 2635 /* 2636 * How many bits to shift Opteron DRAM Address Map base and limit registers 2637 * to get actual value 2638 */ 2639 #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ 2640 #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ 2641 2642 #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ 2643 #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ 2644 2645 #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ 2646 2647 /* 2648 * Macros to derive addresses from Opteron DRAM Address Map registers 2649 */ 2650 #define OPT_DRAMADDR_HI(reg) \ 2651 (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ 2652 OPT_DRAMADDR_HI_LSHIFT_ADDR) 2653 2654 #define OPT_DRAMADDR_LO(reg) \ 2655 (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ 2656 OPT_DRAMADDR_LO_LSHIFT_ADDR) 2657 2658 #define OPT_DRAMADDR(high, low) \ 2659 (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) 2660 2661 /* 2662 * Bit masks defining what's in Opteron DRAM Address Map base register 2663 */ 2664 #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ 2665 #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ 2666 #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ 2667 2668 /* 2669 * Bit masks defining what's in Opteron DRAM Address Map limit register 2670 */ 2671 #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ 2672 #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ 2673 2674 2675 /* 2676 * Opteron Node ID register in PCI configuration space contains 2677 * number of nodes in system, etc. for Opteron K8. The following 2678 * constants and macros define its contents, structure, and access. 2679 */ 2680 2681 /* 2682 * Bit masks defining what's in Opteron Node ID register 2683 */ 2684 #define OPT_NODE_MASK_ID 0x7 /* node ID */ 2685 #define OPT_NODE_MASK_CNT 0x70 /* node count */ 2686 #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ 2687 #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ 2688 #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ 2689 2690 /* 2691 * How many bits in Opteron Node ID register to shift right to get actual value 2692 */ 2693 #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ 2694 2695 /* 2696 * Macros to get values from Opteron Node ID register 2697 */ 2698 #define OPT_NODE_CNT(reg) \ 2699 ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) 2700 2701 /* 2702 * Macro to setup PCI Extended Configuration Space (ECS) address to give to 2703 * "in/out" instructions 2704 * 2705 * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any 2706 * other uses should just do MMIO to access PCI ECS. 2707 * Must enable special bit in Northbridge Configuration Register on 2708 * Greyhound for extended CF8 space access to be able to access PCI ECS 2709 * using "in/out" instructions and restore special bit after done 2710 * accessing PCI ECS. 2711 */ 2712 #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ 2713 (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ 2714 (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ 2715 ((((reg) >> 8) & 0xf) << 24)) 2716 2717 /* 2718 * PCI configuration space registers accessed by specifying 2719 * a bus, device, function, and offset. The following constants 2720 * define the values needed to access Opteron K8 configuration 2721 * info to determine its node topology 2722 */ 2723 2724 #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ 2725 2726 /* 2727 * Opteron PCI configuration space register function values 2728 */ 2729 #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ 2730 #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ 2731 #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ 2732 #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ 2733 2734 /* 2735 * PCI Configuration Space register offsets 2736 */ 2737 #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ 2738 #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ 2739 #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ 2740 #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ 2741 2742 /* 2743 * Opteron PCI Configuration Space device IDs for nodes 2744 */ 2745 #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ 2746 2747 2748 /* 2749 * Opteron DRAM address map gives base and limit for physical memory in a node 2750 */ 2751 typedef struct opt_dram_addr_map { 2752 uint32_t base_hi; 2753 uint32_t base_lo; 2754 uint32_t limit_hi; 2755 uint32_t limit_lo; 2756 } opt_dram_addr_map_t; 2757 2758 2759 /* 2760 * Supported AMD processor families 2761 */ 2762 #define AMD_FAMILY_HAMMER 15 2763 #define AMD_FAMILY_GREYHOUND 16 2764 2765 /* 2766 * Whether to have is_opteron() return 1 even when processor isn't supported 2767 */ 2768 uint_t is_opteron_override = 0; 2769 2770 /* 2771 * AMD processor family for current CPU 2772 */ 2773 uint_t opt_family = 0; 2774 2775 2776 /* 2777 * Determine whether we're running on a supported AMD Opteron since reading 2778 * node count and DRAM address map registers may have different format or 2779 * may not be supported across processor families 2780 */ 2781 static int 2782 is_opteron(void) 2783 { 2784 2785 if (x86_vendor != X86_VENDOR_AMD) 2786 return (0); 2787 2788 opt_family = cpuid_getfamily(CPU); 2789 if (opt_family == AMD_FAMILY_HAMMER || 2790 opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) 2791 return (1); 2792 else 2793 return (0); 2794 } 2795 2796 2797 /* 2798 * Determine NUMA configuration for Opteron from registers that live in PCI 2799 * configuration space 2800 */ 2801 static void 2802 opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 2803 node_phys_addr_map_t *node_memory) 2804 { 2805 uint_t bus; 2806 uint_t dev; 2807 struct opt_dram_addr_map dram_map[MAX_NODES]; 2808 uint_t node; 2809 uint_t node_info[MAX_NODES]; 2810 uint_t off_hi; 2811 uint_t off_lo; 2812 uint64_t nb_cfg_reg; 2813 2814 /* 2815 * Read configuration registers from PCI configuration space to 2816 * determine node information, which memory is in each node, etc. 2817 * 2818 * Write to PCI configuration space address register to specify 2819 * which configuration register to read and read/write PCI 2820 * configuration space data register to get/set contents 2821 */ 2822 bus = OPT_PCS_BUS_CONFIG; 2823 dev = OPT_PCS_DEV_NODE0; 2824 off_hi = OPT_PCS_OFF_DRAMBASE_HI; 2825 off_lo = OPT_PCS_OFF_DRAMBASE_LO; 2826 2827 /* 2828 * Read node ID register for node 0 to get node count 2829 */ 2830 node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, 2831 OPT_PCS_OFF_NODEID); 2832 *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; 2833 2834 /* 2835 * If number of nodes is more than maximum supported, then set node 2836 * count to 1 and treat system as UMA instead of NUMA. 2837 */ 2838 if (*node_cnt > MAX_NODES) { 2839 *node_cnt = 1; 2840 return; 2841 } 2842 2843 /* 2844 * For Greyhound, PCI Extended Configuration Space must be enabled to 2845 * read high DRAM address map base and limit registers 2846 */ 2847 if (opt_family == AMD_FAMILY_GREYHOUND) { 2848 nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); 2849 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2850 wrmsr(MSR_AMD_NB_CFG, 2851 nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); 2852 } 2853 2854 for (node = 0; node < *node_cnt; node++) { 2855 uint32_t base_hi; 2856 uint32_t base_lo; 2857 uint32_t limit_hi; 2858 uint32_t limit_lo; 2859 2860 /* 2861 * Read node ID register (except for node 0 which we just read) 2862 */ 2863 if (node > 0) { 2864 node_info[node] = pci_getl_func(bus, dev, 2865 OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); 2866 } 2867 2868 /* 2869 * Read DRAM base and limit registers which specify 2870 * physical memory range of each node 2871 */ 2872 if (opt_family != AMD_FAMILY_GREYHOUND) 2873 base_hi = 0; 2874 else { 2875 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2876 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2877 base_hi = dram_map[node].base_hi = 2878 inl(PCI_CONFDATA); 2879 } 2880 base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, 2881 OPT_PCS_FUNC_ADDRMAP, off_lo); 2882 2883 if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && 2884 mem_intrlv) 2885 *mem_intrlv = *mem_intrlv + 1; 2886 2887 off_hi += 4; /* high limit register offset */ 2888 if (opt_family != AMD_FAMILY_GREYHOUND) 2889 limit_hi = 0; 2890 else { 2891 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2892 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2893 limit_hi = dram_map[node].limit_hi = 2894 inl(PCI_CONFDATA); 2895 } 2896 2897 off_lo += 4; /* low limit register offset */ 2898 limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, 2899 dev, OPT_PCS_FUNC_ADDRMAP, off_lo); 2900 2901 /* 2902 * Increment device number to next node and register offsets 2903 * for DRAM base register of next node 2904 */ 2905 off_hi += 4; 2906 off_lo += 4; 2907 dev++; 2908 2909 /* 2910 * Both read and write enable bits must be enabled in DRAM 2911 * address map base register for physical memory to exist in 2912 * node 2913 */ 2914 if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || 2915 (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { 2916 /* 2917 * Mark node memory as non-existent and set start and 2918 * end addresses to be same in node_memory[] 2919 */ 2920 node_memory[node].exists = 0; 2921 node_memory[node].start = node_memory[node].end = 2922 (pfn_t)-1; 2923 continue; 2924 } 2925 2926 /* 2927 * Mark node memory as existing and remember physical address 2928 * range of each node for use later 2929 */ 2930 node_memory[node].exists = 1; 2931 2932 node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); 2933 2934 node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | 2935 OPT_DRAMADDR_LO_MASK_OFF); 2936 } 2937 2938 /* 2939 * Restore PCI Extended Configuration Space enable bit 2940 */ 2941 if (opt_family == AMD_FAMILY_GREYHOUND) { 2942 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2943 wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); 2944 } 2945 } 2946 2947 2948 /* 2949 * Return average amount of time to read vendor ID register on Northbridge 2950 * N times on specified destination node from current CPU 2951 */ 2952 static hrtime_t 2953 opt_probe_vendor(int dest_node, int nreads) 2954 { 2955 int cnt; 2956 uint_t dev; 2957 /* LINTED: set but not used in function */ 2958 volatile uint_t dev_vendor; 2959 hrtime_t elapsed; 2960 hrtime_t end; 2961 int ipl; 2962 hrtime_t start; 2963 2964 dev = OPT_PCS_DEV_NODE0 + dest_node; 2965 kpreempt_disable(); 2966 ipl = spl8(); 2967 outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, 2968 OPT_PCS_OFF_VENDOR)); 2969 start = gethrtime(); 2970 for (cnt = 0; cnt < nreads; cnt++) 2971 dev_vendor = inl(PCI_CONFDATA); 2972 end = gethrtime(); 2973 elapsed = (end - start) / nreads; 2974 splx(ipl); 2975 kpreempt_enable(); 2976 return (elapsed); 2977 } 2978