1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS 29 * ================================================================ 30 * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access 31 * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of 32 * one or more CPUs and some local memory. The CPUs in each node can access 33 * the memory in the other nodes but at a higher latency than accessing their 34 * local memory. Typically, a system with only one node has Uniform Memory 35 * Access (UMA), but it may be possible to have a one node system that has 36 * some global memory outside of the node which is higher latency. 37 * 38 * Module Description 39 * ------------------ 40 * This module provides a platform interface for determining which CPUs and 41 * which memory (and how much) are in a NUMA node and how far each node is from 42 * each other. The interface is used by the Virtual Memory (VM) system and the 43 * common lgroup framework. The VM system uses the plat_*() routines to fill 44 * in its memory node (memnode) array with the physical address range spanned 45 * by each NUMA node to know which memory belongs to which node, so it can 46 * build and manage a physical page free list for each NUMA node and allocate 47 * local memory from each node as needed. The common lgroup framework uses the 48 * exported lgrp_plat_*() routines to figure out which CPUs and memory belong 49 * to each node (leaf lgroup) and how far each node is from each other, so it 50 * can build the latency (lgroup) topology for the machine in order to optimize 51 * for locality. Also, an lgroup platform handle instead of lgroups are used 52 * in the interface with this module, so this module shouldn't need to know 53 * anything about lgroups. Instead, it just needs to know which CPUs, memory, 54 * etc. are in each NUMA node, how far each node is from each other, and to use 55 * a unique lgroup platform handle to refer to each node through the interface. 56 * 57 * Determining NUMA Configuration 58 * ------------------------------ 59 * By default, this module will try to determine the NUMA configuration of the 60 * machine by reading the ACPI System Resource Affinity Table (SRAT) and System 61 * Locality Information Table (SLIT). The SRAT contains info to tell which 62 * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT 63 * is a matrix that gives the distance between each system locality (which is 64 * a NUMA node and should correspond to proximity domains in the SRAT). For 65 * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer 66 * specification. 67 * 68 * If the SRAT doesn't exist on a system with AMD Opteron processors, we 69 * examine registers in PCI configuration space to determine how many nodes are 70 * in the system and which CPUs and memory are in each node. 71 * do while booting the kernel. 72 * 73 * NOTE: Using these PCI configuration space registers to determine this 74 * locality info is not guaranteed to work or be compatible across all 75 * Opteron processor families. 76 * 77 * If the SLIT does not exist or look right, the kernel will probe to determine 78 * the distance between nodes as long as the NUMA CPU and memory configuration 79 * has been determined (see lgrp_plat_probe() for details). 80 * 81 * Data Structures 82 * --------------- 83 * The main data structures used by this code are the following: 84 * 85 * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by 86 * CPU ID (only used for SRAT) 87 * 88 * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and 89 * different nodes indexed by node ID 90 * 91 * - lgrp_plat_node_cnt Number of NUMA nodes in system 92 * 93 * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping 94 * table indexed by node ID (only used 95 * for SRAT) 96 * 97 * - lgrp_plat_node_memory[] Table with physical address range for 98 * each node indexed by node ID 99 * 100 * The code is implemented to make the following always be true: 101 * 102 * lgroup platform handle == node ID == memnode ID 103 * 104 * Moreover, it allows for the proximity domain ID to be equal to all of the 105 * above as long as the proximity domains IDs are numbered from 0 to <number of 106 * nodes - 1>. This is done by hashing each proximity domain ID into the range 107 * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID 108 * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] 109 * and be assigned node ID N. If the proximity domain IDs aren't numbered 110 * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into 111 * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs 112 * to node IDs. However, the proximity domain IDs may not map to the 113 * equivalent node ID since we want to keep the node IDs numbered from 0 to 114 * <number of nodes - 1> to minimize cost of searching and potentially space. 115 * 116 * The code below really tries to do the above. However, the virtual memory 117 * system expects the memnodes which describe the physical address range for 118 * each NUMA node to be arranged in ascending order by physical address. (:-( 119 * Otherwise, the kernel will panic in different semi-random places in the VM 120 * system. 121 * 122 * Consequently, this module has to try to sort the nodes in ascending order by 123 * each node's starting physical address to try to meet this "constraint" in 124 * the VM system (see lgrp_plat_node_sort()). Also, the lowest numbered 125 * proximity domain ID in the system is deteremined and used to make the lowest 126 * numbered proximity domain map to node 0 in hopes that the proximity domains 127 * are sorted in ascending order by physical address already even if their IDs 128 * don't start at 0 (see NODE_DOMAIN_HASH() and lgrp_plat_srat_domains()). 129 * Finally, it is important to note that these workarounds may not be 130 * sufficient if/when memory hotplugging is supported and the VM system may 131 * ultimately need to be fixed to handle this.... 132 */ 133 134 135 #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */ 136 #include <sys/bootconf.h> 137 #include <sys/cmn_err.h> 138 #include <sys/controlregs.h> 139 #include <sys/cpupart.h> 140 #include <sys/cpuvar.h> 141 #include <sys/lgrp.h> 142 #include <sys/machsystm.h> 143 #include <sys/memlist.h> 144 #include <sys/memnode.h> 145 #include <sys/mman.h> 146 #include <sys/pci_cfgspace.h> 147 #include <sys/pci_impl.h> 148 #include <sys/param.h> 149 #include <sys/pghw.h> 150 #include <sys/promif.h> /* for prom_printf() */ 151 #include <sys/sysmacros.h> 152 #include <sys/systm.h> 153 #include <sys/thread.h> 154 #include <sys/types.h> 155 #include <sys/var.h> 156 #include <sys/x86_archext.h> /* for x86_feature and X86_AMD */ 157 #include <vm/hat_i86.h> 158 #include <vm/seg_kmem.h> 159 #include <vm/vm_dep.h> 160 161 #include "acpi_fw.h" /* for SRAT and SLIT */ 162 163 164 #define MAX_NODES 8 165 #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) 166 167 /* 168 * Constants for configuring probing 169 */ 170 #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ 171 #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ 172 #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ 173 174 /* 175 * Flags for probing 176 */ 177 #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ 178 #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ 179 #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ 180 181 /* 182 * Hash proximity domain ID into node to domain mapping table "mod" number of 183 * nodes to minimize span of entries used and try to have lowest numbered 184 * proximity domain be node 0 185 */ 186 #define NODE_DOMAIN_HASH(domain, node_cnt) \ 187 ((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \ 188 ((domain) - lgrp_plat_prox_domain_min) % node_cnt) 189 190 191 /* 192 * CPU to node ID mapping structure (only used with SRAT) 193 */ 194 typedef struct cpu_node_map { 195 int exists; 196 uint_t node; 197 uint32_t apicid; 198 uint32_t prox_domain; 199 } cpu_node_map_t; 200 201 /* 202 * Latency statistics 203 */ 204 typedef struct lgrp_plat_latency_stats { 205 hrtime_t latencies[MAX_NODES][MAX_NODES]; 206 hrtime_t latency_max; 207 hrtime_t latency_min; 208 } lgrp_plat_latency_stats_t; 209 210 /* 211 * Memory configuration for probing 212 */ 213 typedef struct lgrp_plat_probe_mem_config { 214 size_t probe_memsize; /* how much memory to probe per node */ 215 caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ 216 pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ 217 } lgrp_plat_probe_mem_config_t; 218 219 /* 220 * Statistics kept for probing 221 */ 222 typedef struct lgrp_plat_probe_stats { 223 hrtime_t flush_cost; 224 hrtime_t probe_cost; 225 hrtime_t probe_cost_total; 226 hrtime_t probe_error_code; 227 hrtime_t probe_errors[MAX_NODES][MAX_NODES]; 228 int probe_suspect[MAX_NODES][MAX_NODES]; 229 hrtime_t probe_max[MAX_NODES][MAX_NODES]; 230 hrtime_t probe_min[MAX_NODES][MAX_NODES]; 231 } lgrp_plat_probe_stats_t; 232 233 /* 234 * Node to proximity domain ID mapping structure (only used with SRAT) 235 */ 236 typedef struct node_domain_map { 237 int exists; 238 uint32_t prox_domain; 239 } node_domain_map_t; 240 241 /* 242 * Node ID and starting and ending page for physical memory in node 243 */ 244 typedef struct node_phys_addr_map { 245 pfn_t start; 246 pfn_t end; 247 int exists; 248 uint32_t prox_domain; 249 } node_phys_addr_map_t; 250 251 /* 252 * Number of CPUs for which we got APIC IDs 253 */ 254 static int lgrp_plat_apic_ncpus = 0; 255 256 /* 257 * CPU to node ID mapping table (only used for SRAT) 258 */ 259 static cpu_node_map_t lgrp_plat_cpu_node[NCPU]; 260 261 /* 262 * Latency statistics 263 */ 264 lgrp_plat_latency_stats_t lgrp_plat_lat_stats; 265 266 /* 267 * Whether memory is interleaved across nodes causing MPO to be disabled 268 */ 269 static int lgrp_plat_mem_intrlv = 0; 270 271 /* 272 * Node ID to proximity domain ID mapping table (only used for SRAT) 273 */ 274 static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; 275 276 /* 277 * Physical address range for memory in each node 278 */ 279 static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES]; 280 281 /* 282 * Statistics gotten from probing 283 */ 284 static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; 285 286 /* 287 * Memory configuration for probing 288 */ 289 static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; 290 291 /* 292 * Lowest proximity domain ID seen in ACPI SRAT 293 */ 294 static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX; 295 296 /* 297 * Error code from processing ACPI SRAT 298 */ 299 static int lgrp_plat_srat_error = 0; 300 301 /* 302 * Error code from processing ACPI SLIT 303 */ 304 static int lgrp_plat_slit_error = 0; 305 306 /* 307 * Allocate lgroup array statically 308 */ 309 static lgrp_t lgrp_space[NLGRP]; 310 static int nlgrps_alloc; 311 312 313 /* 314 * Enable finding and using minimum proximity domain ID when hashing 315 */ 316 int lgrp_plat_domain_min_enable = 1; 317 318 /* 319 * Number of nodes in system 320 */ 321 uint_t lgrp_plat_node_cnt = 1; 322 323 /* 324 * Enable sorting nodes in ascending order by starting physical address 325 */ 326 int lgrp_plat_node_sort_enable = 1; 327 328 /* 329 * Configuration Parameters for Probing 330 * - lgrp_plat_probe_flags Flags to specify enabling probing, probe 331 * operation, etc. 332 * - lgrp_plat_probe_nrounds How many rounds of probing to do 333 * - lgrp_plat_probe_nsamples Number of samples to take when probing each 334 * node 335 * - lgrp_plat_probe_nreads Number of times to read vendor ID from 336 * Northbridge for each probe 337 */ 338 uint_t lgrp_plat_probe_flags = 0; 339 int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; 340 int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; 341 int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; 342 343 /* 344 * Enable use of ACPI System Resource Affinity Table (SRAT) and System 345 * Locality Information Table (SLIT) 346 */ 347 int lgrp_plat_srat_enable = 1; 348 int lgrp_plat_slit_enable = 1; 349 350 /* 351 * mnode_xwa: set to non-zero value to initiate workaround if large pages are 352 * found to be crossing memory node boundaries. The workaround will eliminate 353 * a base size page at the end of each memory node boundary to ensure that 354 * a large page with constituent pages that span more than 1 memory node 355 * can never be formed. 356 * 357 */ 358 int mnode_xwa = 1; 359 360 /* 361 * Static array to hold lgroup statistics 362 */ 363 struct lgrp_stats lgrp_stats[NLGRP]; 364 365 366 /* 367 * Forward declarations of platform interface routines 368 */ 369 void plat_build_mem_nodes(struct memlist *list); 370 371 int plat_lgrphand_to_mem_node(lgrp_handle_t hand); 372 373 lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); 374 375 int plat_mnode_xcheck(pfn_t pfncnt); 376 377 int plat_pfn_to_mem_node(pfn_t pfn); 378 379 /* 380 * Forward declarations of lgroup platform interface routines 381 */ 382 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 383 384 void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); 385 386 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); 387 388 void lgrp_plat_init(void); 389 390 int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); 391 392 void lgrp_plat_main_init(void); 393 394 int lgrp_plat_max_lgrps(void); 395 396 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, 397 lgrp_mem_query_t query); 398 399 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); 400 401 void lgrp_plat_probe(void); 402 403 lgrp_handle_t lgrp_plat_root_hand(void); 404 405 406 /* 407 * Forward declarations of local routines 408 */ 409 static int is_opteron(void); 410 411 static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, 412 int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, 413 uint32_t domain); 414 415 static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node); 416 417 static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, 418 int node_cnt, uint32_t domain); 419 420 static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 421 lgrp_plat_latency_stats_t *lat_stats, 422 lgrp_plat_probe_stats_t *probe_stats); 423 424 static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 425 lgrp_plat_latency_stats_t *lat_stats); 426 427 static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); 428 429 static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, 430 int node_cnt, uint32_t domain); 431 432 static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain, 433 int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start, 434 uint64_t end, uint32_t domain); 435 436 static void lgrp_plat_node_sort(node_domain_map_t *node_domain, 437 int node_cnt, cpu_node_map_t *cpu_node, int cpu_count, 438 node_phys_addr_map_t *node_memory); 439 440 static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 441 lgrp_plat_probe_mem_config_t *probe_mem_config, 442 lgrp_plat_latency_stats_t *lat_stats, 443 lgrp_plat_probe_stats_t *probe_stats); 444 445 static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); 446 447 static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 448 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats); 449 450 static int lgrp_plat_process_srat(struct srat *tp, 451 uint32_t *prox_domain_min, node_domain_map_t *node_domain, 452 cpu_node_map_t *cpu_node, int cpu_count, 453 node_phys_addr_map_t *node_memory); 454 455 static int lgrp_plat_srat_domains(struct srat *tp, 456 uint32_t *prox_domain_min); 457 458 static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 459 lgrp_plat_latency_stats_t *lat_stats); 460 461 static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 462 node_phys_addr_map_t *node_memory); 463 464 static hrtime_t opt_probe_vendor(int dest_node, int nreads); 465 466 467 /* 468 * PLATFORM INTERFACE ROUTINES 469 */ 470 471 /* 472 * Configure memory nodes for machines with more than one node (ie NUMA) 473 */ 474 void 475 plat_build_mem_nodes(struct memlist *list) 476 { 477 pfn_t cur_start; /* start addr of subrange */ 478 pfn_t cur_end; /* end addr of subrange */ 479 pfn_t start; /* start addr of whole range */ 480 pfn_t end; /* end addr of whole range */ 481 pgcnt_t endcnt; /* pages to sacrifice */ 482 483 /* 484 * Boot install lists are arranged <addr, len>, ... 485 */ 486 while (list) { 487 int node; 488 489 start = list->address >> PAGESHIFT; 490 end = (list->address + list->size - 1) >> PAGESHIFT; 491 492 if (start > physmax) { 493 list = list->next; 494 continue; 495 } 496 if (end > physmax) 497 end = physmax; 498 499 /* 500 * When there is only one memnode, just add memory to memnode 501 */ 502 if (max_mem_nodes == 1) { 503 mem_node_add_slice(start, end); 504 list = list->next; 505 continue; 506 } 507 508 /* 509 * mem_node_add_slice() expects to get a memory range that 510 * is within one memnode, so need to split any memory range 511 * that spans multiple memnodes into subranges that are each 512 * contained within one memnode when feeding them to 513 * mem_node_add_slice() 514 */ 515 cur_start = start; 516 do { 517 node = plat_pfn_to_mem_node(cur_start); 518 519 /* 520 * Panic if DRAM address map registers or SRAT say 521 * memory in node doesn't exist or address from 522 * boot installed memory list entry isn't in this node. 523 * This shouldn't happen and rest of code can't deal 524 * with this if it does. 525 */ 526 if (node < 0 || node >= lgrp_plat_node_cnt || 527 !lgrp_plat_node_memory[node].exists || 528 cur_start < lgrp_plat_node_memory[node].start || 529 cur_start > lgrp_plat_node_memory[node].end) { 530 cmn_err(CE_PANIC, "Don't know which memnode " 531 "to add installed memory address 0x%lx\n", 532 cur_start); 533 } 534 535 /* 536 * End of current subrange should not span memnodes 537 */ 538 cur_end = end; 539 endcnt = 0; 540 if (lgrp_plat_node_memory[node].exists && 541 cur_end > lgrp_plat_node_memory[node].end) { 542 cur_end = lgrp_plat_node_memory[node].end; 543 if (mnode_xwa > 1) { 544 /* 545 * sacrifice the last page in each 546 * node to eliminate large pages 547 * that span more than 1 memory node. 548 */ 549 endcnt = 1; 550 } 551 } 552 553 mem_node_add_slice(cur_start, cur_end - endcnt); 554 555 /* 556 * Next subrange starts after end of current one 557 */ 558 cur_start = cur_end + 1; 559 } while (cur_end < end); 560 561 list = list->next; 562 } 563 mem_node_physalign = 0; 564 mem_node_pfn_shift = 0; 565 } 566 567 568 int 569 plat_lgrphand_to_mem_node(lgrp_handle_t hand) 570 { 571 if (max_mem_nodes == 1) 572 return (0); 573 574 return ((int)hand); 575 } 576 577 578 /* 579 * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt 580 * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if 581 * a crossing is found and returns 0 otherwise. 582 */ 583 int 584 plat_mnode_xcheck(pfn_t pfncnt) 585 { 586 int node, prevnode = -1, basenode; 587 pfn_t ea, sa; 588 589 for (node = 0; node < lgrp_plat_node_cnt; node++) { 590 591 if (lgrp_plat_node_memory[node].exists == 0) 592 continue; 593 594 if (prevnode == -1) { 595 prevnode = node; 596 basenode = node; 597 continue; 598 } 599 600 /* assume x86 node pfn ranges are in increasing order */ 601 ASSERT(lgrp_plat_node_memory[node].start > 602 lgrp_plat_node_memory[prevnode].end); 603 604 /* 605 * continue if the starting address of node is not contiguous 606 * with the previous node. 607 */ 608 609 if (lgrp_plat_node_memory[node].start != 610 (lgrp_plat_node_memory[prevnode].end + 1)) { 611 basenode = node; 612 prevnode = node; 613 continue; 614 } 615 616 /* check if the starting address of node is pfncnt aligned */ 617 if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) { 618 619 /* 620 * at this point, node starts at an unaligned boundary 621 * and is contiguous with the previous node(s) to 622 * basenode. Check if there is an aligned contiguous 623 * range of length pfncnt that crosses this boundary. 624 */ 625 626 sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end, 627 pfncnt); 628 ea = P2ROUNDUP((lgrp_plat_node_memory[node].start), 629 pfncnt); 630 631 ASSERT((ea - sa) == pfncnt); 632 if (sa >= lgrp_plat_node_memory[basenode].start && 633 ea <= (lgrp_plat_node_memory[node].end + 1)) { 634 /* 635 * large page found to cross mnode boundary. 636 * Return Failure if workaround not enabled. 637 */ 638 if (mnode_xwa == 0) 639 return (1); 640 mnode_xwa++; 641 } 642 } 643 prevnode = node; 644 } 645 return (0); 646 } 647 648 649 lgrp_handle_t 650 plat_mem_node_to_lgrphand(int mnode) 651 { 652 if (max_mem_nodes == 1) 653 return (LGRP_DEFAULT_HANDLE); 654 655 return ((lgrp_handle_t)mnode); 656 } 657 658 659 int 660 plat_pfn_to_mem_node(pfn_t pfn) 661 { 662 int node; 663 664 if (max_mem_nodes == 1) 665 return (0); 666 667 for (node = 0; node < lgrp_plat_node_cnt; node++) { 668 /* 669 * Skip nodes with no memory 670 */ 671 if (!lgrp_plat_node_memory[node].exists) 672 continue; 673 674 if (pfn >= lgrp_plat_node_memory[node].start && 675 pfn <= lgrp_plat_node_memory[node].end) 676 return (node); 677 } 678 679 /* 680 * Didn't find memnode where this PFN lives which should never happen 681 */ 682 ASSERT(node < lgrp_plat_node_cnt); 683 return (-1); 684 } 685 686 687 /* 688 * LGROUP PLATFORM INTERFACE ROUTINES 689 */ 690 691 /* 692 * Allocate additional space for an lgroup. 693 */ 694 /* ARGSUSED */ 695 lgrp_t * 696 lgrp_plat_alloc(lgrp_id_t lgrpid) 697 { 698 lgrp_t *lgrp; 699 700 lgrp = &lgrp_space[nlgrps_alloc++]; 701 if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) 702 return (NULL); 703 return (lgrp); 704 } 705 706 707 /* 708 * Platform handling for (re)configuration changes 709 */ 710 /* ARGSUSED */ 711 void 712 lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) 713 { 714 } 715 716 717 /* 718 * Return the platform handle for the lgroup containing the given CPU 719 */ 720 /* ARGSUSED */ 721 lgrp_handle_t 722 lgrp_plat_cpu_to_hand(processorid_t id) 723 { 724 lgrp_handle_t hand; 725 726 if (lgrp_plat_node_cnt == 1) 727 return (LGRP_DEFAULT_HANDLE); 728 729 hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], 730 lgrp_plat_cpu_node); 731 732 ASSERT(hand != (lgrp_handle_t)-1); 733 if (hand == (lgrp_handle_t)-1) 734 return (LGRP_NULL_HANDLE); 735 736 return (hand); 737 } 738 739 740 /* 741 * Platform-specific initialization of lgroups 742 */ 743 void 744 lgrp_plat_init(void) 745 { 746 #if defined(__xpv) 747 /* 748 * XXPV For now, the hypervisor treats all memory equally. 749 */ 750 lgrp_plat_node_cnt = max_mem_nodes = 1; 751 #else /* __xpv */ 752 uint_t probe_op; 753 u_longlong_t value; 754 755 /* 756 * Get boot property for lgroup topology height limit 757 */ 758 if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0) 759 (void) lgrp_topo_ht_limit_set((int)value); 760 761 /* 762 * Get boot property for enabling/disabling SRAT 763 */ 764 if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0) 765 lgrp_plat_srat_enable = (int)value; 766 767 /* 768 * Get boot property for enabling/disabling SLIT 769 */ 770 if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0) 771 lgrp_plat_slit_enable = (int)value; 772 773 /* 774 * Initialize as a UMA machine 775 */ 776 if (lgrp_topo_ht_limit() == 1) { 777 lgrp_plat_node_cnt = max_mem_nodes = 1; 778 return; 779 } 780 781 /* 782 * Read boot property with CPU to APIC ID mapping table/array and fill 783 * in CPU to node ID mapping table with APIC ID for each CPU 784 */ 785 lgrp_plat_apic_ncpus = 786 lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); 787 788 /* 789 * Determine which CPUs and memory are local to each other and number 790 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) 791 */ 792 if (lgrp_plat_apic_ncpus > 0) { 793 int retval; 794 795 retval = lgrp_plat_process_srat(srat_ptr, 796 &lgrp_plat_prox_domain_min, 797 lgrp_plat_node_domain, lgrp_plat_cpu_node, 798 lgrp_plat_apic_ncpus, lgrp_plat_node_memory); 799 if (retval <= 0) { 800 lgrp_plat_srat_error = retval; 801 lgrp_plat_node_cnt = 1; 802 } else { 803 lgrp_plat_srat_error = 0; 804 lgrp_plat_node_cnt = retval; 805 } 806 } 807 808 /* 809 * Try to use PCI config space registers on Opteron if there's an error 810 * processing CPU to APIC ID mapping or SRAT 811 */ 812 if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && 813 is_opteron()) 814 opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, 815 lgrp_plat_node_memory); 816 817 /* 818 * Don't bother to setup system for multiple lgroups and only use one 819 * memory node when memory is interleaved between any nodes or there is 820 * only one NUMA node 821 * 822 * NOTE: May need to change this for Dynamic Reconfiguration (DR) 823 * when and if it happens for x86/x64 824 */ 825 if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { 826 lgrp_plat_node_cnt = max_mem_nodes = 1; 827 (void) lgrp_topo_ht_limit_set(1); 828 return; 829 } 830 831 /* 832 * Leaf lgroups on x86/x64 architectures contain one physical 833 * processor chip. Tune lgrp_expand_proc_thresh and 834 * lgrp_expand_proc_diff so that lgrp_choose() will spread 835 * things out aggressively. 836 */ 837 lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; 838 lgrp_expand_proc_diff = 0; 839 840 /* 841 * There should be one memnode (physical page free list(s)) for 842 * each node 843 */ 844 max_mem_nodes = lgrp_plat_node_cnt; 845 846 /* 847 * Initialize min and max latency before reading SLIT or probing 848 */ 849 lgrp_plat_lat_stats.latency_min = -1; 850 lgrp_plat_lat_stats.latency_max = 0; 851 852 /* 853 * Determine how far each NUMA node is from each other by 854 * reading ACPI System Locality Information Table (SLIT) if it 855 * exists 856 */ 857 lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, 858 lgrp_plat_node_cnt, lgrp_plat_node_memory, 859 &lgrp_plat_lat_stats); 860 if (lgrp_plat_slit_error == 0) 861 return; 862 863 /* 864 * Probe to determine latency between NUMA nodes when SLIT 865 * doesn't exist or make sense 866 */ 867 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; 868 869 /* 870 * Specify whether to probe using vendor ID register or page copy 871 * if hasn't been specified already or is overspecified 872 */ 873 probe_op = lgrp_plat_probe_flags & 874 (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 875 876 if (probe_op == 0 || 877 probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { 878 lgrp_plat_probe_flags &= 879 ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 880 if (is_opteron()) 881 lgrp_plat_probe_flags |= 882 LGRP_PLAT_PROBE_VENDOR; 883 else 884 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; 885 } 886 887 /* 888 * Probing errors can mess up the lgroup topology and 889 * force us fall back to a 2 level lgroup topology. 890 * Here we bound how tall the lgroup topology can grow 891 * in hopes of avoiding any anamolies in probing from 892 * messing up the lgroup topology by limiting the 893 * accuracy of the latency topology. 894 * 895 * Assume that nodes will at least be configured in a 896 * ring, so limit height of lgroup topology to be less 897 * than number of nodes on a system with 4 or more 898 * nodes 899 */ 900 if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == 901 lgrp_topo_ht_limit_default()) 902 (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); 903 #endif /* __xpv */ 904 } 905 906 907 /* 908 * Return latency between "from" and "to" lgroups 909 * 910 * This latency number can only be used for relative comparison 911 * between lgroups on the running system, cannot be used across platforms, 912 * and may not reflect the actual latency. It is platform and implementation 913 * specific, so platform gets to decide its value. It would be nice if the 914 * number was at least proportional to make comparisons more meaningful though. 915 */ 916 /* ARGSUSED */ 917 int 918 lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) 919 { 920 lgrp_handle_t src, dest; 921 int node; 922 923 if (max_mem_nodes == 1) 924 return (0); 925 926 /* 927 * Return max latency for root lgroup 928 */ 929 if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) 930 return (lgrp_plat_lat_stats.latency_max); 931 932 src = from; 933 dest = to; 934 935 /* 936 * Return 0 for nodes (lgroup platform handles) out of range 937 */ 938 if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES) 939 return (0); 940 941 /* 942 * Probe from current CPU if its lgroup latencies haven't been set yet 943 * and we are trying to get latency from current CPU to some node 944 */ 945 node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 946 ASSERT(node >= 0 && node < lgrp_plat_node_cnt); 947 if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src) 948 lgrp_plat_probe(); 949 950 return (lgrp_plat_lat_stats.latencies[src][dest]); 951 } 952 953 954 /* 955 * Platform-specific initialization 956 */ 957 void 958 lgrp_plat_main_init(void) 959 { 960 int curnode; 961 int ht_limit; 962 int i; 963 964 /* 965 * Print a notice that MPO is disabled when memory is interleaved 966 * across nodes....Would do this when it is discovered, but can't 967 * because it happens way too early during boot.... 968 */ 969 if (lgrp_plat_mem_intrlv) 970 cmn_err(CE_NOTE, 971 "MPO disabled because memory is interleaved\n"); 972 973 /* 974 * Don't bother to do any probing if it is disabled, there is only one 975 * node, or the height of the lgroup topology less than or equal to 2 976 */ 977 ht_limit = lgrp_topo_ht_limit(); 978 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 979 max_mem_nodes == 1 || ht_limit <= 2) { 980 /* 981 * Setup lgroup latencies for 2 level lgroup topology 982 * (ie. local and remote only) if they haven't been set yet 983 */ 984 if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && 985 lgrp_plat_lat_stats.latency_max == 0) 986 lgrp_plat_2level_setup(lgrp_plat_node_memory, 987 &lgrp_plat_lat_stats); 988 return; 989 } 990 991 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 992 /* 993 * Should have been able to probe from CPU 0 when it was added 994 * to lgroup hierarchy, but may not have been able to then 995 * because it happens so early in boot that gethrtime() hasn't 996 * been initialized. (:-( 997 */ 998 curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 999 ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt); 1000 if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) 1001 lgrp_plat_probe(); 1002 1003 return; 1004 } 1005 1006 /* 1007 * When probing memory, use one page for every sample to determine 1008 * lgroup topology and taking multiple samples 1009 */ 1010 if (lgrp_plat_probe_mem_config.probe_memsize == 0) 1011 lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * 1012 lgrp_plat_probe_nsamples; 1013 1014 /* 1015 * Map memory in each node needed for probing to determine latency 1016 * topology 1017 */ 1018 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1019 int mnode; 1020 1021 /* 1022 * Skip this node and leave its probe page NULL 1023 * if it doesn't have any memory 1024 */ 1025 mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i); 1026 if (!mem_node_config[mnode].exists) { 1027 lgrp_plat_probe_mem_config.probe_va[i] = NULL; 1028 continue; 1029 } 1030 1031 /* 1032 * Allocate one kernel virtual page 1033 */ 1034 lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, 1035 lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); 1036 if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { 1037 cmn_err(CE_WARN, 1038 "lgrp_plat_main_init: couldn't allocate memory"); 1039 return; 1040 } 1041 1042 /* 1043 * Get PFN for first page in each node 1044 */ 1045 lgrp_plat_probe_mem_config.probe_pfn[i] = 1046 mem_node_config[mnode].physbase; 1047 1048 /* 1049 * Map virtual page to first page in node 1050 */ 1051 hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], 1052 lgrp_plat_probe_mem_config.probe_memsize, 1053 lgrp_plat_probe_mem_config.probe_pfn[i], 1054 PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, 1055 HAT_LOAD_NOCONSIST); 1056 } 1057 1058 /* 1059 * Probe from current CPU 1060 */ 1061 lgrp_plat_probe(); 1062 } 1063 1064 1065 /* 1066 * Return the maximum number of lgrps supported by the platform. 1067 * Before lgrp topology is known it returns an estimate based on the number of 1068 * nodes. Once topology is known it returns the actual maximim number of lgrps 1069 * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and 1070 * dynamic addition of new nodes, this number may not grow during system 1071 * lifetime (yet). 1072 */ 1073 int 1074 lgrp_plat_max_lgrps(void) 1075 { 1076 return (lgrp_topo_initialized ? 1077 lgrp_alloc_max + 1 : 1078 lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); 1079 } 1080 1081 1082 /* 1083 * Return the number of free pages in an lgroup. 1084 * 1085 * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize 1086 * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the 1087 * number of allocatable base pagesize pages corresponding to the 1088 * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) 1089 * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical 1090 * memory installed, regardless of whether or not it's usable. 1091 */ 1092 pgcnt_t 1093 lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) 1094 { 1095 int mnode; 1096 pgcnt_t npgs = (pgcnt_t)0; 1097 extern struct memlist *phys_avail; 1098 extern struct memlist *phys_install; 1099 1100 1101 if (plathand == LGRP_DEFAULT_HANDLE) 1102 return (lgrp_plat_mem_size_default(plathand, query)); 1103 1104 if (plathand != LGRP_NULL_HANDLE) { 1105 mnode = plat_lgrphand_to_mem_node(plathand); 1106 if (mnode >= 0 && mem_node_config[mnode].exists) { 1107 switch (query) { 1108 case LGRP_MEM_SIZE_FREE: 1109 npgs = MNODE_PGCNT(mnode); 1110 break; 1111 case LGRP_MEM_SIZE_AVAIL: 1112 npgs = mem_node_memlist_pages(mnode, 1113 phys_avail); 1114 break; 1115 case LGRP_MEM_SIZE_INSTALL: 1116 npgs = mem_node_memlist_pages(mnode, 1117 phys_install); 1118 break; 1119 default: 1120 break; 1121 } 1122 } 1123 } 1124 return (npgs); 1125 } 1126 1127 1128 /* 1129 * Return the platform handle of the lgroup that contains the physical memory 1130 * corresponding to the given page frame number 1131 */ 1132 /* ARGSUSED */ 1133 lgrp_handle_t 1134 lgrp_plat_pfn_to_hand(pfn_t pfn) 1135 { 1136 int mnode; 1137 1138 if (max_mem_nodes == 1) 1139 return (LGRP_DEFAULT_HANDLE); 1140 1141 if (pfn > physmax) 1142 return (LGRP_NULL_HANDLE); 1143 1144 mnode = plat_pfn_to_mem_node(pfn); 1145 if (mnode < 0) 1146 return (LGRP_NULL_HANDLE); 1147 1148 return (MEM_NODE_2_LGRPHAND(mnode)); 1149 } 1150 1151 1152 /* 1153 * Probe memory in each node from current CPU to determine latency topology 1154 * 1155 * The probing code will probe the vendor ID register on the Northbridge of 1156 * Opteron processors and probe memory for other processors by default. 1157 * 1158 * Since probing is inherently error prone, the code takes laps across all the 1159 * nodes probing from each node to each of the other nodes some number of 1160 * times. Furthermore, each node is probed some number of times before moving 1161 * onto the next one during each lap. The minimum latency gotten between nodes 1162 * is kept as the latency between the nodes. 1163 * 1164 * After all that, the probe times are adjusted by normalizing values that are 1165 * close to each other and local latencies are made the same. Lastly, the 1166 * latencies are verified to make sure that certain conditions are met (eg. 1167 * local < remote, latency(a, b) == latency(b, a), etc.). 1168 * 1169 * If any of the conditions aren't met, the code will export a NUMA 1170 * configuration with the local CPUs and memory given by the SRAT or PCI config 1171 * space registers and one remote memory latency since it can't tell exactly 1172 * how far each node is from each other. 1173 */ 1174 void 1175 lgrp_plat_probe(void) 1176 { 1177 int from; 1178 int i; 1179 lgrp_plat_latency_stats_t *lat_stats; 1180 hrtime_t probe_time; 1181 int to; 1182 1183 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 1184 max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) 1185 return; 1186 1187 /* 1188 * Determine ID of node containing current CPU 1189 */ 1190 from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 1191 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 1192 if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) 1193 ASSERT(lgrp_plat_node_domain[from].exists); 1194 1195 /* 1196 * Don't need to probe if got times already 1197 */ 1198 lat_stats = &lgrp_plat_lat_stats; 1199 if (lat_stats->latencies[from][from] != 0) 1200 return; 1201 1202 /* 1203 * Read vendor ID in Northbridge or read and write page(s) 1204 * in each node from current CPU and remember how long it takes, 1205 * so we can build latency topology of machine later. 1206 * This should approximate the memory latency between each node. 1207 */ 1208 for (i = 0; i < lgrp_plat_probe_nrounds; i++) { 1209 for (to = 0; to < lgrp_plat_node_cnt; to++) { 1210 /* 1211 * Get probe time and bail out if can't get it yet 1212 */ 1213 probe_time = lgrp_plat_probe_time(to, 1214 lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config, 1215 &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); 1216 if (probe_time == 0) 1217 return; 1218 1219 /* 1220 * Keep lowest probe time as latency between nodes 1221 */ 1222 if (lat_stats->latencies[from][to] == 0 || 1223 probe_time < lat_stats->latencies[from][to]) 1224 lat_stats->latencies[from][to] = probe_time; 1225 1226 /* 1227 * Update overall minimum and maximum probe times 1228 * across all nodes 1229 */ 1230 if (probe_time < lat_stats->latency_min || 1231 lat_stats->latency_min == -1) 1232 lat_stats->latency_min = probe_time; 1233 if (probe_time > lat_stats->latency_max) 1234 lat_stats->latency_max = probe_time; 1235 } 1236 } 1237 1238 /* 1239 * - Fix up latencies such that local latencies are same, 1240 * latency(i, j) == latency(j, i), etc. (if possible) 1241 * 1242 * - Verify that latencies look ok 1243 * 1244 * - Fallback to just optimizing for local and remote if 1245 * latencies didn't look right 1246 */ 1247 lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats, 1248 &lgrp_plat_probe_stats); 1249 lgrp_plat_probe_stats.probe_error_code = 1250 lgrp_plat_latency_verify(lgrp_plat_node_memory, 1251 &lgrp_plat_lat_stats); 1252 if (lgrp_plat_probe_stats.probe_error_code) 1253 lgrp_plat_2level_setup(lgrp_plat_node_memory, 1254 &lgrp_plat_lat_stats); 1255 } 1256 1257 1258 /* 1259 * Return platform handle for root lgroup 1260 */ 1261 lgrp_handle_t 1262 lgrp_plat_root_hand(void) 1263 { 1264 return (LGRP_DEFAULT_HANDLE); 1265 } 1266 1267 1268 /* 1269 * INTERNAL ROUTINES 1270 */ 1271 1272 1273 /* 1274 * Update CPU to node mapping for given CPU and proximity domain (and returns 1275 * negative numbers for errors and positive ones for success) 1276 */ 1277 static int 1278 lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, 1279 cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) 1280 { 1281 uint_t i; 1282 int node; 1283 1284 /* 1285 * Get node number for proximity domain 1286 */ 1287 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1288 if (node == -1) { 1289 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1290 domain); 1291 if (node == -1) 1292 return (-1); 1293 } 1294 1295 /* 1296 * Search for entry with given APIC ID and fill in its node and 1297 * proximity domain IDs (if they haven't been set already) 1298 */ 1299 for (i = 0; i < nentries; i++) { 1300 /* 1301 * Skip nonexistent entries and ones without matching APIC ID 1302 */ 1303 if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) 1304 continue; 1305 1306 /* 1307 * Just return if entry completely and correctly filled in 1308 * already 1309 */ 1310 if (cpu_node[i].prox_domain == domain && 1311 cpu_node[i].node == node) 1312 return (1); 1313 1314 /* 1315 * Fill in node and proximity domain IDs 1316 */ 1317 cpu_node[i].prox_domain = domain; 1318 cpu_node[i].node = node; 1319 1320 return (0); 1321 } 1322 1323 /* 1324 * Return error when entry for APIC ID wasn't found in table 1325 */ 1326 return (-2); 1327 } 1328 1329 1330 /* 1331 * Get node ID for given CPU 1332 */ 1333 static int 1334 lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node) 1335 { 1336 processorid_t cpuid; 1337 1338 if (cp == NULL) 1339 return (-1); 1340 1341 cpuid = cp->cpu_id; 1342 if (cpuid < 0 || cpuid >= max_ncpus) 1343 return (-1); 1344 1345 /* 1346 * SRAT doesn't exist, isn't enabled, or there was an error processing 1347 * it, so return chip ID for Opteron and -1 otherwise. 1348 */ 1349 if (srat_ptr == NULL || !lgrp_plat_srat_enable || 1350 lgrp_plat_srat_error) { 1351 if (is_opteron()) 1352 return (pg_plat_hw_instance_id(cp, PGHW_CHIP)); 1353 return (-1); 1354 } 1355 1356 /* 1357 * Return -1 when CPU to node ID mapping entry doesn't exist for given 1358 * CPU 1359 */ 1360 if (!cpu_node[cpuid].exists) 1361 return (-1); 1362 1363 return (cpu_node[cpuid].node); 1364 } 1365 1366 1367 /* 1368 * Return node number for given proximity domain/system locality 1369 */ 1370 static int 1371 lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, 1372 uint32_t domain) 1373 { 1374 uint_t node; 1375 uint_t start; 1376 1377 /* 1378 * Hash proximity domain ID into node to domain mapping table (array), 1379 * search for entry with matching proximity domain ID, and return index 1380 * of matching entry as node ID. 1381 */ 1382 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1383 do { 1384 if (node_domain[node].prox_domain == domain && 1385 node_domain[node].exists) 1386 return (node); 1387 node = (node + 1) % node_cnt; 1388 } while (node != start); 1389 return (-1); 1390 } 1391 1392 1393 /* 1394 * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to 1395 * be considered same 1396 */ 1397 #define LGRP_LAT_TOLERANCE_SHIFT 4 1398 1399 int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; 1400 1401 1402 /* 1403 * Adjust latencies between nodes to be symmetric, normalize latencies between 1404 * any nodes that are within some tolerance to be same, and make local 1405 * latencies be same 1406 */ 1407 static void 1408 lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 1409 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 1410 { 1411 int i; 1412 int j; 1413 int k; 1414 int l; 1415 u_longlong_t max; 1416 u_longlong_t min; 1417 u_longlong_t t; 1418 u_longlong_t t1; 1419 u_longlong_t t2; 1420 const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; 1421 int lat_corrected[MAX_NODES][MAX_NODES]; 1422 1423 /* 1424 * Nothing to do when this is an UMA machine or don't have args needed 1425 */ 1426 if (max_mem_nodes == 1) 1427 return; 1428 1429 ASSERT(node_memory != NULL && lat_stats != NULL && 1430 probe_stats != NULL); 1431 1432 /* 1433 * Make sure that latencies are symmetric between any two nodes 1434 * (ie. latency(node0, node1) == latency(node1, node0)) 1435 */ 1436 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1437 if (!node_memory[i].exists) 1438 continue; 1439 1440 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1441 if (!node_memory[j].exists) 1442 continue; 1443 1444 t1 = lat_stats->latencies[i][j]; 1445 t2 = lat_stats->latencies[j][i]; 1446 1447 if (t1 == 0 || t2 == 0 || t1 == t2) 1448 continue; 1449 1450 /* 1451 * Latencies should be same 1452 * - Use minimum of two latencies which should be same 1453 * - Track suspect probe times not within tolerance of 1454 * min value 1455 * - Remember how much values are corrected by 1456 */ 1457 if (t1 > t2) { 1458 t = t2; 1459 probe_stats->probe_errors[i][j] += t1 - t2; 1460 if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { 1461 probe_stats->probe_suspect[i][j]++; 1462 probe_stats->probe_suspect[j][i]++; 1463 } 1464 } else if (t2 > t1) { 1465 t = t1; 1466 probe_stats->probe_errors[j][i] += t2 - t1; 1467 if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { 1468 probe_stats->probe_suspect[i][j]++; 1469 probe_stats->probe_suspect[j][i]++; 1470 } 1471 } 1472 1473 lat_stats->latencies[i][j] = 1474 lat_stats->latencies[j][i] = t; 1475 lgrp_config(cflag, t1, t); 1476 lgrp_config(cflag, t2, t); 1477 } 1478 } 1479 1480 /* 1481 * Keep track of which latencies get corrected 1482 */ 1483 for (i = 0; i < MAX_NODES; i++) 1484 for (j = 0; j < MAX_NODES; j++) 1485 lat_corrected[i][j] = 0; 1486 1487 /* 1488 * For every two nodes, see whether there is another pair of nodes which 1489 * are about the same distance apart and make the latencies be the same 1490 * if they are close enough together 1491 */ 1492 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1493 if (!node_memory[i].exists) 1494 continue; 1495 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1496 if (!node_memory[j].exists) 1497 continue; 1498 /* 1499 * Pick one pair of nodes (i, j) 1500 * and get latency between them 1501 */ 1502 t1 = lat_stats->latencies[i][j]; 1503 1504 /* 1505 * Skip this pair of nodes if there isn't a latency 1506 * for it yet 1507 */ 1508 if (t1 == 0) 1509 continue; 1510 1511 for (k = 0; k < lgrp_plat_node_cnt; k++) { 1512 if (!node_memory[k].exists) 1513 continue; 1514 for (l = 0; l < lgrp_plat_node_cnt; l++) { 1515 if (!node_memory[l].exists) 1516 continue; 1517 /* 1518 * Pick another pair of nodes (k, l) 1519 * not same as (i, j) and get latency 1520 * between them 1521 */ 1522 if (k == i && l == j) 1523 continue; 1524 1525 t2 = lat_stats->latencies[k][l]; 1526 1527 /* 1528 * Skip this pair of nodes if there 1529 * isn't a latency for it yet 1530 */ 1531 1532 if (t2 == 0) 1533 continue; 1534 1535 /* 1536 * Skip nodes (k, l) if they already 1537 * have same latency as (i, j) or 1538 * their latency isn't close enough to 1539 * be considered/made the same 1540 */ 1541 if (t1 == t2 || (t1 > t2 && t1 - t2 > 1542 t1 >> lgrp_plat_probe_lt_shift) || 1543 (t2 > t1 && t2 - t1 > 1544 t2 >> lgrp_plat_probe_lt_shift)) 1545 continue; 1546 1547 /* 1548 * Make latency(i, j) same as 1549 * latency(k, l), try to use latency 1550 * that has been adjusted already to get 1551 * more consistency (if possible), and 1552 * remember which latencies were 1553 * adjusted for next time 1554 */ 1555 if (lat_corrected[i][j]) { 1556 t = t1; 1557 lgrp_config(cflag, t2, t); 1558 t2 = t; 1559 } else if (lat_corrected[k][l]) { 1560 t = t2; 1561 lgrp_config(cflag, t1, t); 1562 t1 = t; 1563 } else { 1564 if (t1 > t2) 1565 t = t2; 1566 else 1567 t = t1; 1568 lgrp_config(cflag, t1, t); 1569 lgrp_config(cflag, t2, t); 1570 t1 = t2 = t; 1571 } 1572 1573 lat_stats->latencies[i][j] = 1574 lat_stats->latencies[k][l] = t; 1575 1576 lat_corrected[i][j] = 1577 lat_corrected[k][l] = 1; 1578 } 1579 } 1580 } 1581 } 1582 1583 /* 1584 * Local latencies should be same 1585 * - Find min and max local latencies 1586 * - Make all local latencies be minimum 1587 */ 1588 min = -1; 1589 max = 0; 1590 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1591 if (!node_memory[i].exists) 1592 continue; 1593 t = lat_stats->latencies[i][i]; 1594 if (t == 0) 1595 continue; 1596 if (min == -1 || t < min) 1597 min = t; 1598 if (t > max) 1599 max = t; 1600 } 1601 if (min != max) { 1602 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1603 int local; 1604 1605 if (!node_memory[i].exists) 1606 continue; 1607 1608 local = lat_stats->latencies[i][i]; 1609 if (local == 0) 1610 continue; 1611 1612 /* 1613 * Track suspect probe times that aren't within 1614 * tolerance of minimum local latency and how much 1615 * probe times are corrected by 1616 */ 1617 if (local - min > min >> lgrp_plat_probe_lt_shift) 1618 probe_stats->probe_suspect[i][i]++; 1619 1620 probe_stats->probe_errors[i][i] += local - min; 1621 1622 /* 1623 * Make local latencies be minimum 1624 */ 1625 lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); 1626 lat_stats->latencies[i][i] = min; 1627 } 1628 } 1629 1630 /* 1631 * Determine max probe time again since just adjusted latencies 1632 */ 1633 lat_stats->latency_max = 0; 1634 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1635 if (!node_memory[i].exists) 1636 continue; 1637 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1638 if (!node_memory[j].exists) 1639 continue; 1640 t = lat_stats->latencies[i][j]; 1641 if (t > lat_stats->latency_max) 1642 lat_stats->latency_max = t; 1643 } 1644 } 1645 } 1646 1647 1648 /* 1649 * Verify following about latencies between nodes: 1650 * 1651 * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) 1652 * - Local latencies same 1653 * - Local < remote 1654 * - Number of latencies seen is reasonable 1655 * - Number of occurrences of a given latency should be more than 1 1656 * 1657 * Returns: 1658 * 0 Success 1659 * -1 Not symmetric 1660 * -2 Local latencies not same 1661 * -3 Local >= remote 1662 */ 1663 static int 1664 lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 1665 lgrp_plat_latency_stats_t *lat_stats) 1666 { 1667 int i; 1668 int j; 1669 u_longlong_t t1; 1670 u_longlong_t t2; 1671 1672 ASSERT(node_memory != NULL && lat_stats != NULL); 1673 1674 /* 1675 * Nothing to do when this is an UMA machine, lgroup topology is 1676 * limited to 2 levels, or there aren't any probe times yet 1677 */ 1678 if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || 1679 lat_stats->latencies[0][0] == 0) 1680 return (0); 1681 1682 /* 1683 * Make sure that latencies are symmetric between any two nodes 1684 * (ie. latency(node0, node1) == latency(node1, node0)) 1685 */ 1686 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1687 if (!node_memory[i].exists) 1688 continue; 1689 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1690 if (!node_memory[j].exists) 1691 continue; 1692 t1 = lat_stats->latencies[i][j]; 1693 t2 = lat_stats->latencies[j][i]; 1694 1695 if (t1 == 0 || t2 == 0 || t1 == t2) 1696 continue; 1697 1698 return (-1); 1699 } 1700 } 1701 1702 /* 1703 * Local latencies should be same 1704 */ 1705 t1 = lat_stats->latencies[0][0]; 1706 for (i = 1; i < lgrp_plat_node_cnt; i++) { 1707 if (!node_memory[i].exists) 1708 continue; 1709 1710 t2 = lat_stats->latencies[i][i]; 1711 if (t2 == 0) 1712 continue; 1713 1714 if (t1 == 0) { 1715 t1 = t2; 1716 continue; 1717 } 1718 1719 if (t1 != t2) 1720 return (-2); 1721 } 1722 1723 /* 1724 * Local latencies should be less than remote 1725 */ 1726 if (t1) { 1727 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1728 if (!node_memory[i].exists) 1729 continue; 1730 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1731 if (!node_memory[j].exists) 1732 continue; 1733 t2 = lat_stats->latencies[i][j]; 1734 if (i == j || t2 == 0) 1735 continue; 1736 1737 if (t1 >= t2) 1738 return (-3); 1739 } 1740 } 1741 } 1742 1743 return (0); 1744 } 1745 1746 1747 /* 1748 * Return the number of free, allocatable, or installed 1749 * pages in an lgroup 1750 * This is a copy of the MAX_MEM_NODES == 1 version of the routine 1751 * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup 1752 */ 1753 /* ARGSUSED */ 1754 static pgcnt_t 1755 lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) 1756 { 1757 struct memlist *mlist; 1758 pgcnt_t npgs = 0; 1759 extern struct memlist *phys_avail; 1760 extern struct memlist *phys_install; 1761 1762 switch (query) { 1763 case LGRP_MEM_SIZE_FREE: 1764 return ((pgcnt_t)freemem); 1765 case LGRP_MEM_SIZE_AVAIL: 1766 memlist_read_lock(); 1767 for (mlist = phys_avail; mlist; mlist = mlist->next) 1768 npgs += btop(mlist->size); 1769 memlist_read_unlock(); 1770 return (npgs); 1771 case LGRP_MEM_SIZE_INSTALL: 1772 memlist_read_lock(); 1773 for (mlist = phys_install; mlist; mlist = mlist->next) 1774 npgs += btop(mlist->size); 1775 memlist_read_unlock(); 1776 return (npgs); 1777 default: 1778 return ((pgcnt_t)0); 1779 } 1780 } 1781 1782 1783 /* 1784 * Update node to proximity domain mappings for given domain and return node ID 1785 */ 1786 static int 1787 lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, 1788 uint32_t domain) 1789 { 1790 uint_t node; 1791 uint_t start; 1792 1793 /* 1794 * Hash proximity domain ID into node to domain mapping table (array) 1795 * and add entry for it into first non-existent or matching entry found 1796 */ 1797 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1798 do { 1799 /* 1800 * Entry doesn't exist yet, so create one for this proximity 1801 * domain and return node ID which is index into mapping table. 1802 */ 1803 if (!node_domain[node].exists) { 1804 node_domain[node].exists = 1; 1805 node_domain[node].prox_domain = domain; 1806 return (node); 1807 } 1808 1809 /* 1810 * Entry exists for this proximity domain already, so just 1811 * return node ID (index into table). 1812 */ 1813 if (node_domain[node].prox_domain == domain) 1814 return (node); 1815 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1816 } while (node != start); 1817 1818 /* 1819 * Ran out of supported number of entries which shouldn't happen.... 1820 */ 1821 ASSERT(node != start); 1822 return (-1); 1823 } 1824 1825 1826 /* 1827 * Update node memory information for given proximity domain with specified 1828 * starting and ending physical address range (and return positive numbers for 1829 * success and negative ones for errors) 1830 */ 1831 static int 1832 lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt, 1833 node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end, 1834 uint32_t domain) 1835 { 1836 int node; 1837 1838 /* 1839 * Get node number for proximity domain 1840 */ 1841 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1842 if (node == -1) { 1843 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1844 domain); 1845 if (node == -1) 1846 return (-1); 1847 } 1848 1849 /* 1850 * Create entry in table for node if it doesn't exist 1851 */ 1852 if (!node_memory[node].exists) { 1853 node_memory[node].exists = 1; 1854 node_memory[node].start = btop(start); 1855 node_memory[node].end = btop(end); 1856 node_memory[node].prox_domain = domain; 1857 return (0); 1858 } 1859 1860 /* 1861 * Entry already exists for this proximity domain 1862 * 1863 * There may be more than one SRAT memory entry for a domain, so we may 1864 * need to update existing start or end address for the node. 1865 */ 1866 if (node_memory[node].prox_domain == domain) { 1867 if (btop(start) < node_memory[node].start) 1868 node_memory[node].start = btop(start); 1869 if (btop(end) > node_memory[node].end) 1870 node_memory[node].end = btop(end); 1871 return (1); 1872 } 1873 return (-2); 1874 } 1875 1876 1877 /* 1878 * Have to sort node by starting physical address because VM system (physical 1879 * page free list management) assumes and expects memnodes to be sorted in 1880 * ascending order by physical address. If not, the kernel will panic in 1881 * potentially a number of different places. (:-( 1882 * NOTE: This workaround will not be sufficient if/when hotplugging memory is 1883 * supported on x86/x64. 1884 */ 1885 static void 1886 lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt, 1887 cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory) 1888 { 1889 boolean_t found; 1890 int i; 1891 int j; 1892 int n; 1893 boolean_t sorted; 1894 boolean_t swapped; 1895 1896 if (!lgrp_plat_node_sort_enable || node_cnt <= 1 || 1897 node_domain == NULL || node_memory == NULL) 1898 return; 1899 1900 /* 1901 * Sorted already? 1902 */ 1903 sorted = B_TRUE; 1904 for (i = 0; i < node_cnt - 1; i++) { 1905 /* 1906 * Skip entries that don't exist 1907 */ 1908 if (!node_memory[i].exists) 1909 continue; 1910 1911 /* 1912 * Try to find next existing entry to compare against 1913 */ 1914 found = B_FALSE; 1915 for (j = i + 1; j < node_cnt; j++) { 1916 if (node_memory[j].exists) { 1917 found = B_TRUE; 1918 break; 1919 } 1920 } 1921 1922 /* 1923 * Done if no more existing entries to compare against 1924 */ 1925 if (found == B_FALSE) 1926 break; 1927 1928 /* 1929 * Not sorted if starting address of current entry is bigger 1930 * than starting address of next existing entry 1931 */ 1932 if (node_memory[i].start > node_memory[j].start) { 1933 sorted = B_FALSE; 1934 break; 1935 } 1936 } 1937 1938 /* 1939 * Don't need to sort if sorted already 1940 */ 1941 if (sorted == B_TRUE) 1942 return; 1943 1944 /* 1945 * Just use bubble sort since number of nodes is small 1946 */ 1947 n = node_cnt; 1948 do { 1949 swapped = B_FALSE; 1950 n--; 1951 for (i = 0; i < n; i++) { 1952 /* 1953 * Skip entries that don't exist 1954 */ 1955 if (!node_memory[i].exists) 1956 continue; 1957 1958 /* 1959 * Try to find next existing entry to compare against 1960 */ 1961 found = B_FALSE; 1962 for (j = i + 1; j <= n; j++) { 1963 if (node_memory[j].exists) { 1964 found = B_TRUE; 1965 break; 1966 } 1967 } 1968 1969 /* 1970 * Done if no more existing entries to compare against 1971 */ 1972 if (found == B_FALSE) 1973 break; 1974 1975 if (node_memory[i].start > node_memory[j].start) { 1976 node_phys_addr_map_t save_addr; 1977 node_domain_map_t save_node; 1978 1979 /* 1980 * Swap node to proxmity domain ID assignments 1981 */ 1982 bcopy(&node_domain[i], &save_node, 1983 sizeof (node_domain_map_t)); 1984 bcopy(&node_domain[j], &node_domain[i], 1985 sizeof (node_domain_map_t)); 1986 bcopy(&save_node, &node_domain[j], 1987 sizeof (node_domain_map_t)); 1988 1989 /* 1990 * Swap node to physical memory assignments 1991 */ 1992 bcopy(&node_memory[i], &save_addr, 1993 sizeof (node_phys_addr_map_t)); 1994 bcopy(&node_memory[j], &node_memory[i], 1995 sizeof (node_phys_addr_map_t)); 1996 bcopy(&save_addr, &node_memory[j], 1997 sizeof (node_phys_addr_map_t)); 1998 swapped = B_TRUE; 1999 } 2000 } 2001 } while (swapped == B_TRUE); 2002 2003 /* 2004 * Check to make sure that CPUs assigned to correct node IDs now since 2005 * node to proximity domain ID assignments may have been changed above 2006 */ 2007 if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1) 2008 return; 2009 for (i = 0; i < cpu_count; i++) { 2010 int node; 2011 2012 node = lgrp_plat_domain_to_node(node_domain, node_cnt, 2013 cpu_node[i].prox_domain); 2014 if (cpu_node[i].node != node) 2015 cpu_node[i].node = node; 2016 } 2017 2018 } 2019 2020 2021 /* 2022 * Return time needed to probe from current CPU to memory in given node 2023 */ 2024 static hrtime_t 2025 lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 2026 lgrp_plat_probe_mem_config_t *probe_mem_config, 2027 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 2028 { 2029 caddr_t buf; 2030 hrtime_t elapsed; 2031 hrtime_t end; 2032 int from; 2033 int i; 2034 int ipl; 2035 hrtime_t max; 2036 hrtime_t min; 2037 hrtime_t start; 2038 extern int use_sse_pagecopy; 2039 2040 /* 2041 * Determine ID of node containing current CPU 2042 */ 2043 from = lgrp_plat_cpu_to_node(CPU, cpu_node); 2044 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 2045 2046 /* 2047 * Do common work for probing main memory 2048 */ 2049 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { 2050 /* 2051 * Skip probing any nodes without memory and 2052 * set probe time to 0 2053 */ 2054 if (probe_mem_config->probe_va[to] == NULL) { 2055 lat_stats->latencies[from][to] = 0; 2056 return (0); 2057 } 2058 2059 /* 2060 * Invalidate caches once instead of once every sample 2061 * which should cut cost of probing by a lot 2062 */ 2063 probe_stats->flush_cost = gethrtime(); 2064 invalidate_cache(); 2065 probe_stats->flush_cost = gethrtime() - 2066 probe_stats->flush_cost; 2067 probe_stats->probe_cost_total += probe_stats->flush_cost; 2068 } 2069 2070 /* 2071 * Probe from current CPU to given memory using specified operation 2072 * and take specified number of samples 2073 */ 2074 max = 0; 2075 min = -1; 2076 for (i = 0; i < lgrp_plat_probe_nsamples; i++) { 2077 probe_stats->probe_cost = gethrtime(); 2078 2079 /* 2080 * Can't measure probe time if gethrtime() isn't working yet 2081 */ 2082 if (probe_stats->probe_cost == 0 && gethrtime() == 0) 2083 return (0); 2084 2085 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 2086 /* 2087 * Measure how long it takes to read vendor ID from 2088 * Northbridge 2089 */ 2090 elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); 2091 } else { 2092 /* 2093 * Measure how long it takes to copy page 2094 * on top of itself 2095 */ 2096 buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); 2097 2098 kpreempt_disable(); 2099 ipl = splhigh(); 2100 start = gethrtime(); 2101 if (use_sse_pagecopy) 2102 hwblkpagecopy(buf, buf); 2103 else 2104 bcopy(buf, buf, PAGESIZE); 2105 end = gethrtime(); 2106 elapsed = end - start; 2107 splx(ipl); 2108 kpreempt_enable(); 2109 } 2110 2111 probe_stats->probe_cost = gethrtime() - 2112 probe_stats->probe_cost; 2113 probe_stats->probe_cost_total += probe_stats->probe_cost; 2114 2115 if (min == -1 || elapsed < min) 2116 min = elapsed; 2117 if (elapsed > max) 2118 max = elapsed; 2119 } 2120 2121 /* 2122 * Update minimum and maximum probe times between 2123 * these two nodes 2124 */ 2125 if (min < probe_stats->probe_min[from][to] || 2126 probe_stats->probe_min[from][to] == 0) 2127 probe_stats->probe_min[from][to] = min; 2128 2129 if (max > probe_stats->probe_max[from][to]) 2130 probe_stats->probe_max[from][to] = max; 2131 2132 return (min); 2133 } 2134 2135 2136 /* 2137 * Read boot property with CPU to APIC ID array, fill in CPU to node ID 2138 * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs. 2139 * 2140 * NOTE: This code assumes that CPU IDs are assigned in order that they appear 2141 * in in cpu_apicid_array boot property which is based on and follows 2142 * same ordering as processor list in ACPI MADT. If the code in 2143 * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns 2144 * CPU IDs ever changes, then this code will need to change too.... 2145 */ 2146 static int 2147 lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) 2148 { 2149 int boot_prop_len; 2150 char *boot_prop_name = BP_CPU_APICID_ARRAY; 2151 uint8_t cpu_apicid_array[UINT8_MAX + 1]; 2152 int i; 2153 int n; 2154 2155 /* 2156 * Nothing to do when no array to fill in or not enough CPUs 2157 */ 2158 if (cpu_node == NULL) 2159 return (-1); 2160 2161 /* 2162 * Check length of property value 2163 */ 2164 boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); 2165 if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array)) 2166 return (-2); 2167 2168 /* 2169 * Calculate number of entries in array and return when there's just 2170 * one CPU since that's not very interesting for NUMA 2171 */ 2172 n = boot_prop_len / sizeof (uint8_t); 2173 if (n == 1) 2174 return (-3); 2175 2176 /* 2177 * Get CPU to APIC ID property value 2178 */ 2179 if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) 2180 return (-4); 2181 2182 /* 2183 * Fill in CPU to node ID mapping table with APIC ID for each CPU 2184 */ 2185 for (i = 0; i < n; i++) { 2186 cpu_node[i].exists = 1; 2187 cpu_node[i].apicid = cpu_apicid_array[i]; 2188 } 2189 2190 /* 2191 * Return number of CPUs based on number of APIC IDs 2192 */ 2193 return (n); 2194 } 2195 2196 2197 /* 2198 * Read ACPI System Locality Information Table (SLIT) to determine how far each 2199 * NUMA node is from each other 2200 */ 2201 static int 2202 lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 2203 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats) 2204 { 2205 int i; 2206 int j; 2207 int localities; 2208 hrtime_t max; 2209 hrtime_t min; 2210 int retval; 2211 uint8_t *slit_entries; 2212 2213 if (tp == NULL || !lgrp_plat_slit_enable) 2214 return (1); 2215 2216 if (lat_stats == NULL) 2217 return (2); 2218 2219 localities = tp->number; 2220 if (localities != node_cnt) 2221 return (3); 2222 2223 min = lat_stats->latency_min; 2224 max = lat_stats->latency_max; 2225 2226 /* 2227 * Fill in latency matrix based on SLIT entries 2228 */ 2229 slit_entries = tp->entry; 2230 for (i = 0; i < localities; i++) { 2231 for (j = 0; j < localities; j++) { 2232 uint8_t latency; 2233 2234 latency = slit_entries[(i * localities) + j]; 2235 lat_stats->latencies[i][j] = latency; 2236 if (latency < min || min == -1) 2237 min = latency; 2238 if (latency > max) 2239 max = latency; 2240 } 2241 } 2242 2243 /* 2244 * Verify that latencies/distances given in SLIT look reasonable 2245 */ 2246 retval = lgrp_plat_latency_verify(node_memory, lat_stats); 2247 2248 if (retval) { 2249 /* 2250 * Reinitialize (zero) latency table since SLIT doesn't look 2251 * right 2252 */ 2253 for (i = 0; i < localities; i++) { 2254 for (j = 0; j < localities; j++) 2255 lat_stats->latencies[i][j] = 0; 2256 } 2257 } else { 2258 /* 2259 * Update min and max latencies seen since SLIT looks valid 2260 */ 2261 lat_stats->latency_min = min; 2262 lat_stats->latency_max = max; 2263 } 2264 2265 return (retval); 2266 } 2267 2268 2269 /* 2270 * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs 2271 * and memory are local to each other in the same NUMA node and return number 2272 * of nodes 2273 */ 2274 static int 2275 lgrp_plat_process_srat(struct srat *tp, uint32_t *prox_domain_min, 2276 node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, 2277 node_phys_addr_map_t *node_memory) 2278 { 2279 struct srat_item *srat_end; 2280 int i; 2281 struct srat_item *item; 2282 int node_cnt; 2283 int proc_entry_count; 2284 2285 /* 2286 * Nothing to do when no SRAT or disabled 2287 */ 2288 if (tp == NULL || !lgrp_plat_srat_enable) 2289 return (-1); 2290 2291 /* 2292 * Determine number of nodes by counting number of proximity domains in 2293 * SRAT and return if number of nodes is 1 or less since don't need to 2294 * read SRAT then 2295 */ 2296 node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min); 2297 if (node_cnt == 1) 2298 return (1); 2299 else if (node_cnt <= 0) 2300 return (-2); 2301 2302 /* 2303 * Walk through SRAT, examining each CPU and memory entry to determine 2304 * which CPUs and memory belong to which node. 2305 */ 2306 item = tp->list; 2307 srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2308 proc_entry_count = 0; 2309 while (item < srat_end) { 2310 uint32_t apic_id; 2311 uint32_t domain; 2312 uint64_t end; 2313 uint64_t length; 2314 uint64_t start; 2315 2316 switch (item->type) { 2317 case SRAT_PROCESSOR: /* CPU entry */ 2318 if (!(item->i.p.flags & SRAT_ENABLED) || 2319 cpu_node == NULL) 2320 break; 2321 2322 /* 2323 * Calculate domain (node) ID and fill in APIC ID to 2324 * domain/node mapping table 2325 */ 2326 domain = item->i.p.domain1; 2327 for (i = 0; i < 3; i++) { 2328 domain += item->i.p.domain2[i] << 2329 ((i + 1) * 8); 2330 } 2331 apic_id = item->i.p.apic_id; 2332 2333 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2334 cpu_node, cpu_count, apic_id, domain) < 0) 2335 return (-3); 2336 2337 proc_entry_count++; 2338 break; 2339 2340 case SRAT_MEMORY: /* memory entry */ 2341 if (!(item->i.m.flags & SRAT_ENABLED) || 2342 node_memory == NULL) 2343 break; 2344 2345 /* 2346 * Get domain (node) ID and fill in domain/node 2347 * to memory mapping table 2348 */ 2349 domain = item->i.m.domain; 2350 start = item->i.m.base_addr; 2351 length = item->i.m.len; 2352 end = start + length - 1; 2353 2354 if (lgrp_plat_node_memory_update(node_domain, node_cnt, 2355 node_memory, start, end, domain) < 0) 2356 return (-4); 2357 break; 2358 case SRAT_X2APIC: /* x2apic CPU entry */ 2359 if (!(item->i.xp.flags & SRAT_ENABLED) || 2360 cpu_node == NULL) 2361 break; 2362 2363 /* 2364 * Calculate domain (node) ID and fill in APIC ID to 2365 * domain/node mapping table 2366 */ 2367 domain = item->i.xp.domain; 2368 apic_id = item->i.xp.x2apic_id; 2369 2370 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2371 cpu_node, cpu_count, apic_id, domain) < 0) 2372 return (-3); 2373 2374 proc_entry_count++; 2375 break; 2376 2377 default: 2378 break; 2379 } 2380 2381 item = (struct srat_item *)((uintptr_t)item + item->len); 2382 } 2383 2384 /* 2385 * Should have seen at least as many SRAT processor entries as CPUs 2386 */ 2387 if (proc_entry_count < cpu_count) 2388 return (-5); 2389 2390 /* 2391 * Need to sort nodes by starting physical address since VM system 2392 * assumes and expects memnodes to be sorted in ascending order by 2393 * physical address 2394 */ 2395 lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count, 2396 node_memory); 2397 2398 return (node_cnt); 2399 } 2400 2401 2402 /* 2403 * Return number of proximity domains given in ACPI SRAT 2404 */ 2405 static int 2406 lgrp_plat_srat_domains(struct srat *tp, uint32_t *prox_domain_min) 2407 { 2408 int domain_cnt; 2409 uint32_t domain_min; 2410 struct srat_item *end; 2411 int i; 2412 struct srat_item *item; 2413 node_domain_map_t node_domain[MAX_NODES]; 2414 2415 2416 if (tp == NULL || !lgrp_plat_srat_enable) 2417 return (1); 2418 2419 /* 2420 * Walk through SRAT to find minimum proximity domain ID 2421 */ 2422 domain_min = UINT32_MAX; 2423 item = tp->list; 2424 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2425 while (item < end) { 2426 uint32_t domain; 2427 2428 switch (item->type) { 2429 case SRAT_PROCESSOR: /* CPU entry */ 2430 if (!(item->i.p.flags & SRAT_ENABLED)) { 2431 item = (struct srat_item *)((uintptr_t)item + 2432 item->len); 2433 continue; 2434 } 2435 domain = item->i.p.domain1; 2436 for (i = 0; i < 3; i++) { 2437 domain += item->i.p.domain2[i] << 2438 ((i + 1) * 8); 2439 } 2440 break; 2441 2442 case SRAT_MEMORY: /* memory entry */ 2443 if (!(item->i.m.flags & SRAT_ENABLED)) { 2444 item = (struct srat_item *)((uintptr_t)item + 2445 item->len); 2446 continue; 2447 } 2448 domain = item->i.m.domain; 2449 break; 2450 2451 case SRAT_X2APIC: /* x2apic CPU entry */ 2452 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2453 item = (struct srat_item *)((uintptr_t)item + 2454 item->len); 2455 continue; 2456 } 2457 domain = item->i.xp.domain; 2458 break; 2459 2460 default: 2461 item = (struct srat_item *)((uintptr_t)item + 2462 item->len); 2463 continue; 2464 } 2465 2466 /* 2467 * Keep track of minimum proximity domain ID 2468 */ 2469 if (domain < domain_min) 2470 domain_min = domain; 2471 2472 item = (struct srat_item *)((uintptr_t)item + item->len); 2473 } 2474 if (lgrp_plat_domain_min_enable && prox_domain_min != NULL) 2475 *prox_domain_min = domain_min; 2476 2477 /* 2478 * Walk through SRAT, examining each CPU and memory entry to determine 2479 * proximity domain ID for each. 2480 */ 2481 domain_cnt = 0; 2482 item = tp->list; 2483 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2484 bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); 2485 while (item < end) { 2486 uint32_t domain; 2487 boolean_t overflow; 2488 uint_t start; 2489 2490 switch (item->type) { 2491 case SRAT_PROCESSOR: /* CPU entry */ 2492 if (!(item->i.p.flags & SRAT_ENABLED)) { 2493 item = (struct srat_item *)((uintptr_t)item + 2494 item->len); 2495 continue; 2496 } 2497 domain = item->i.p.domain1; 2498 for (i = 0; i < 3; i++) { 2499 domain += item->i.p.domain2[i] << 2500 ((i + 1) * 8); 2501 } 2502 break; 2503 2504 case SRAT_MEMORY: /* memory entry */ 2505 if (!(item->i.m.flags & SRAT_ENABLED)) { 2506 item = (struct srat_item *)((uintptr_t)item + 2507 item->len); 2508 continue; 2509 } 2510 domain = item->i.m.domain; 2511 break; 2512 2513 case SRAT_X2APIC: /* x2apic CPU entry */ 2514 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2515 item = (struct srat_item *)((uintptr_t)item + 2516 item->len); 2517 continue; 2518 } 2519 domain = item->i.xp.domain; 2520 break; 2521 2522 default: 2523 item = (struct srat_item *)((uintptr_t)item + 2524 item->len); 2525 continue; 2526 } 2527 2528 /* 2529 * Count and keep track of which proximity domain IDs seen 2530 */ 2531 start = i = domain % MAX_NODES; 2532 overflow = B_TRUE; 2533 do { 2534 /* 2535 * Create entry for proximity domain and increment 2536 * count when no entry exists where proximity domain 2537 * hashed 2538 */ 2539 if (!node_domain[i].exists) { 2540 node_domain[i].exists = 1; 2541 node_domain[i].prox_domain = domain; 2542 domain_cnt++; 2543 overflow = B_FALSE; 2544 break; 2545 } 2546 2547 /* 2548 * Nothing to do when proximity domain seen already 2549 * and its entry exists 2550 */ 2551 if (node_domain[i].prox_domain == domain) { 2552 overflow = B_FALSE; 2553 break; 2554 } 2555 2556 /* 2557 * Entry exists where proximity domain hashed, but for 2558 * different proximity domain so keep search for empty 2559 * slot to put it or matching entry whichever comes 2560 * first. 2561 */ 2562 i = (i + 1) % MAX_NODES; 2563 } while (i != start); 2564 2565 /* 2566 * Didn't find empty or matching entry which means have more 2567 * proximity domains than supported nodes (:-( 2568 */ 2569 ASSERT(overflow != B_TRUE); 2570 if (overflow == B_TRUE) 2571 return (-1); 2572 2573 item = (struct srat_item *)((uintptr_t)item + item->len); 2574 } 2575 return (domain_cnt); 2576 } 2577 2578 2579 /* 2580 * Set lgroup latencies for 2 level lgroup topology 2581 */ 2582 static void 2583 lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 2584 lgrp_plat_latency_stats_t *lat_stats) 2585 { 2586 int i; 2587 2588 ASSERT(node_memory != NULL && lat_stats != NULL); 2589 2590 if (lgrp_plat_node_cnt >= 4) 2591 cmn_err(CE_NOTE, 2592 "MPO only optimizing for local and remote\n"); 2593 for (i = 0; i < lgrp_plat_node_cnt; i++) { 2594 int j; 2595 2596 if (!node_memory[i].exists) 2597 continue; 2598 for (j = 0; j < lgrp_plat_node_cnt; j++) { 2599 if (!node_memory[j].exists) 2600 continue; 2601 if (i == j) 2602 lat_stats->latencies[i][j] = 2; 2603 else 2604 lat_stats->latencies[i][j] = 3; 2605 } 2606 } 2607 lat_stats->latency_min = 2; 2608 lat_stats->latency_max = 3; 2609 lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); 2610 } 2611 2612 2613 /* 2614 * The following Opteron specific constants, macros, types, and routines define 2615 * PCI configuration space registers and how to read them to determine the NUMA 2616 * configuration of *supported* Opteron processors. They provide the same 2617 * information that may be gotten from the ACPI System Resource Affinity Table 2618 * (SRAT) if it exists on the machine of interest. 2619 * 2620 * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family 2621 * of interest describes all of these registers and their contents. The main 2622 * registers used by this code to determine the NUMA configuration of the 2623 * machine are the node ID register for the number of NUMA nodes and the DRAM 2624 * address map registers for the physical address range of each node. 2625 * 2626 * NOTE: The format and how to determine the NUMA configuration using PCI 2627 * config space registers may change or may not be supported in future 2628 * Opteron processor families. 2629 */ 2630 2631 /* 2632 * How many bits to shift Opteron DRAM Address Map base and limit registers 2633 * to get actual value 2634 */ 2635 #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ 2636 #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ 2637 2638 #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ 2639 #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ 2640 2641 #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ 2642 2643 /* 2644 * Macros to derive addresses from Opteron DRAM Address Map registers 2645 */ 2646 #define OPT_DRAMADDR_HI(reg) \ 2647 (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ 2648 OPT_DRAMADDR_HI_LSHIFT_ADDR) 2649 2650 #define OPT_DRAMADDR_LO(reg) \ 2651 (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ 2652 OPT_DRAMADDR_LO_LSHIFT_ADDR) 2653 2654 #define OPT_DRAMADDR(high, low) \ 2655 (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) 2656 2657 /* 2658 * Bit masks defining what's in Opteron DRAM Address Map base register 2659 */ 2660 #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ 2661 #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ 2662 #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ 2663 2664 /* 2665 * Bit masks defining what's in Opteron DRAM Address Map limit register 2666 */ 2667 #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ 2668 #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ 2669 2670 2671 /* 2672 * Opteron Node ID register in PCI configuration space contains 2673 * number of nodes in system, etc. for Opteron K8. The following 2674 * constants and macros define its contents, structure, and access. 2675 */ 2676 2677 /* 2678 * Bit masks defining what's in Opteron Node ID register 2679 */ 2680 #define OPT_NODE_MASK_ID 0x7 /* node ID */ 2681 #define OPT_NODE_MASK_CNT 0x70 /* node count */ 2682 #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ 2683 #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ 2684 #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ 2685 2686 /* 2687 * How many bits in Opteron Node ID register to shift right to get actual value 2688 */ 2689 #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ 2690 2691 /* 2692 * Macros to get values from Opteron Node ID register 2693 */ 2694 #define OPT_NODE_CNT(reg) \ 2695 ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) 2696 2697 /* 2698 * Macro to setup PCI Extended Configuration Space (ECS) address to give to 2699 * "in/out" instructions 2700 * 2701 * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any 2702 * other uses should just do MMIO to access PCI ECS. 2703 * Must enable special bit in Northbridge Configuration Register on 2704 * Greyhound for extended CF8 space access to be able to access PCI ECS 2705 * using "in/out" instructions and restore special bit after done 2706 * accessing PCI ECS. 2707 */ 2708 #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ 2709 (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ 2710 (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ 2711 ((((reg) >> 8) & 0xf) << 24)) 2712 2713 /* 2714 * PCI configuration space registers accessed by specifying 2715 * a bus, device, function, and offset. The following constants 2716 * define the values needed to access Opteron K8 configuration 2717 * info to determine its node topology 2718 */ 2719 2720 #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ 2721 2722 /* 2723 * Opteron PCI configuration space register function values 2724 */ 2725 #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ 2726 #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ 2727 #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ 2728 #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ 2729 2730 /* 2731 * PCI Configuration Space register offsets 2732 */ 2733 #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ 2734 #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ 2735 #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ 2736 #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ 2737 2738 /* 2739 * Opteron PCI Configuration Space device IDs for nodes 2740 */ 2741 #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ 2742 2743 2744 /* 2745 * Opteron DRAM address map gives base and limit for physical memory in a node 2746 */ 2747 typedef struct opt_dram_addr_map { 2748 uint32_t base_hi; 2749 uint32_t base_lo; 2750 uint32_t limit_hi; 2751 uint32_t limit_lo; 2752 } opt_dram_addr_map_t; 2753 2754 2755 /* 2756 * Supported AMD processor families 2757 */ 2758 #define AMD_FAMILY_HAMMER 15 2759 #define AMD_FAMILY_GREYHOUND 16 2760 2761 /* 2762 * Whether to have is_opteron() return 1 even when processor isn't supported 2763 */ 2764 uint_t is_opteron_override = 0; 2765 2766 /* 2767 * AMD processor family for current CPU 2768 */ 2769 uint_t opt_family = 0; 2770 2771 2772 /* 2773 * Determine whether we're running on a supported AMD Opteron since reading 2774 * node count and DRAM address map registers may have different format or 2775 * may not be supported across processor families 2776 */ 2777 static int 2778 is_opteron(void) 2779 { 2780 2781 if (x86_vendor != X86_VENDOR_AMD) 2782 return (0); 2783 2784 opt_family = cpuid_getfamily(CPU); 2785 if (opt_family == AMD_FAMILY_HAMMER || 2786 opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) 2787 return (1); 2788 else 2789 return (0); 2790 } 2791 2792 2793 /* 2794 * Determine NUMA configuration for Opteron from registers that live in PCI 2795 * configuration space 2796 */ 2797 static void 2798 opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 2799 node_phys_addr_map_t *node_memory) 2800 { 2801 uint_t bus; 2802 uint_t dev; 2803 struct opt_dram_addr_map dram_map[MAX_NODES]; 2804 uint_t node; 2805 uint_t node_info[MAX_NODES]; 2806 uint_t off_hi; 2807 uint_t off_lo; 2808 uint64_t nb_cfg_reg; 2809 2810 /* 2811 * Read configuration registers from PCI configuration space to 2812 * determine node information, which memory is in each node, etc. 2813 * 2814 * Write to PCI configuration space address register to specify 2815 * which configuration register to read and read/write PCI 2816 * configuration space data register to get/set contents 2817 */ 2818 bus = OPT_PCS_BUS_CONFIG; 2819 dev = OPT_PCS_DEV_NODE0; 2820 off_hi = OPT_PCS_OFF_DRAMBASE_HI; 2821 off_lo = OPT_PCS_OFF_DRAMBASE_LO; 2822 2823 /* 2824 * Read node ID register for node 0 to get node count 2825 */ 2826 node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, 2827 OPT_PCS_OFF_NODEID); 2828 *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; 2829 2830 /* 2831 * If number of nodes is more than maximum supported, then set node 2832 * count to 1 and treat system as UMA instead of NUMA. 2833 */ 2834 if (*node_cnt > MAX_NODES) { 2835 *node_cnt = 1; 2836 return; 2837 } 2838 2839 /* 2840 * For Greyhound, PCI Extended Configuration Space must be enabled to 2841 * read high DRAM address map base and limit registers 2842 */ 2843 if (opt_family == AMD_FAMILY_GREYHOUND) { 2844 nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); 2845 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2846 wrmsr(MSR_AMD_NB_CFG, 2847 nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); 2848 } 2849 2850 for (node = 0; node < *node_cnt; node++) { 2851 uint32_t base_hi; 2852 uint32_t base_lo; 2853 uint32_t limit_hi; 2854 uint32_t limit_lo; 2855 2856 /* 2857 * Read node ID register (except for node 0 which we just read) 2858 */ 2859 if (node > 0) { 2860 node_info[node] = pci_getl_func(bus, dev, 2861 OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); 2862 } 2863 2864 /* 2865 * Read DRAM base and limit registers which specify 2866 * physical memory range of each node 2867 */ 2868 if (opt_family != AMD_FAMILY_GREYHOUND) 2869 base_hi = 0; 2870 else { 2871 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2872 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2873 base_hi = dram_map[node].base_hi = 2874 inl(PCI_CONFDATA); 2875 } 2876 base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, 2877 OPT_PCS_FUNC_ADDRMAP, off_lo); 2878 2879 if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && 2880 mem_intrlv) 2881 *mem_intrlv = *mem_intrlv + 1; 2882 2883 off_hi += 4; /* high limit register offset */ 2884 if (opt_family != AMD_FAMILY_GREYHOUND) 2885 limit_hi = 0; 2886 else { 2887 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2888 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2889 limit_hi = dram_map[node].limit_hi = 2890 inl(PCI_CONFDATA); 2891 } 2892 2893 off_lo += 4; /* low limit register offset */ 2894 limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, 2895 dev, OPT_PCS_FUNC_ADDRMAP, off_lo); 2896 2897 /* 2898 * Increment device number to next node and register offsets 2899 * for DRAM base register of next node 2900 */ 2901 off_hi += 4; 2902 off_lo += 4; 2903 dev++; 2904 2905 /* 2906 * Both read and write enable bits must be enabled in DRAM 2907 * address map base register for physical memory to exist in 2908 * node 2909 */ 2910 if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || 2911 (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { 2912 /* 2913 * Mark node memory as non-existent and set start and 2914 * end addresses to be same in node_memory[] 2915 */ 2916 node_memory[node].exists = 0; 2917 node_memory[node].start = node_memory[node].end = 2918 (pfn_t)-1; 2919 continue; 2920 } 2921 2922 /* 2923 * Mark node memory as existing and remember physical address 2924 * range of each node for use later 2925 */ 2926 node_memory[node].exists = 1; 2927 2928 node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); 2929 2930 node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | 2931 OPT_DRAMADDR_LO_MASK_OFF); 2932 } 2933 2934 /* 2935 * Restore PCI Extended Configuration Space enable bit 2936 */ 2937 if (opt_family == AMD_FAMILY_GREYHOUND) { 2938 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2939 wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); 2940 } 2941 } 2942 2943 2944 /* 2945 * Return average amount of time to read vendor ID register on Northbridge 2946 * N times on specified destination node from current CPU 2947 */ 2948 static hrtime_t 2949 opt_probe_vendor(int dest_node, int nreads) 2950 { 2951 int cnt; 2952 uint_t dev; 2953 /* LINTED: set but not used in function */ 2954 volatile uint_t dev_vendor; 2955 hrtime_t elapsed; 2956 hrtime_t end; 2957 int ipl; 2958 hrtime_t start; 2959 2960 dev = OPT_PCS_DEV_NODE0 + dest_node; 2961 kpreempt_disable(); 2962 ipl = spl8(); 2963 outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, 2964 OPT_PCS_OFF_VENDOR)); 2965 start = gethrtime(); 2966 for (cnt = 0; cnt < nreads; cnt++) 2967 dev_vendor = inl(PCI_CONFDATA); 2968 end = gethrtime(); 2969 elapsed = (end - start) / nreads; 2970 splx(ipl); 2971 kpreempt_enable(); 2972 return (elapsed); 2973 } 2974