1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS 29 * ================================================================ 30 * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access 31 * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of 32 * one or more CPUs and some local memory. The CPUs in each node can access 33 * the memory in the other nodes but at a higher latency than accessing their 34 * local memory. Typically, a system with only one node has Uniform Memory 35 * Access (UMA), but it may be possible to have a one node system that has 36 * some global memory outside of the node which is higher latency. 37 * 38 * Module Description 39 * ------------------ 40 * This module provides a platform interface for determining which CPUs and 41 * which memory (and how much) are in a NUMA node and how far each node is from 42 * each other. The interface is used by the Virtual Memory (VM) system and the 43 * common lgroup framework. The VM system uses the plat_*() routines to fill 44 * in its memory node (memnode) array with the physical address range spanned 45 * by each NUMA node to know which memory belongs to which node, so it can 46 * build and manage a physical page free list for each NUMA node and allocate 47 * local memory from each node as needed. The common lgroup framework uses the 48 * exported lgrp_plat_*() routines to figure out which CPUs and memory belong 49 * to each node (leaf lgroup) and how far each node is from each other, so it 50 * can build the latency (lgroup) topology for the machine in order to optimize 51 * for locality. Also, an lgroup platform handle instead of lgroups are used 52 * in the interface with this module, so this module shouldn't need to know 53 * anything about lgroups. Instead, it just needs to know which CPUs, memory, 54 * etc. are in each NUMA node, how far each node is from each other, and to use 55 * a unique lgroup platform handle to refer to each node through the interface. 56 * 57 * Determining NUMA Configuration 58 * ------------------------------ 59 * By default, this module will try to determine the NUMA configuration of the 60 * machine by reading the ACPI System Resource Affinity Table (SRAT) and System 61 * Locality Information Table (SLIT). The SRAT contains info to tell which 62 * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT 63 * is a matrix that gives the distance between each system locality (which is 64 * a NUMA node and should correspond to proximity domains in the SRAT). For 65 * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer 66 * specification. 67 * 68 * If the SRAT doesn't exist on a system with AMD Opteron processors, we 69 * examine registers in PCI configuration space to determine how many nodes are 70 * in the system and which CPUs and memory are in each node. 71 * do while booting the kernel. 72 * 73 * NOTE: Using these PCI configuration space registers to determine this 74 * locality info is not guaranteed to work or be compatible across all 75 * Opteron processor families. 76 * 77 * If the SLIT does not exist or look right, the kernel will probe to determine 78 * the distance between nodes as long as the NUMA CPU and memory configuration 79 * has been determined (see lgrp_plat_probe() for details). 80 * 81 * Data Structures 82 * --------------- 83 * The main data structures used by this code are the following: 84 * 85 * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by 86 * CPU ID (only used for SRAT) 87 * 88 * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and 89 * different nodes indexed by node ID 90 * 91 * - lgrp_plat_node_cnt Number of NUMA nodes in system 92 * 93 * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping 94 * table indexed by node ID (only used 95 * for SRAT) 96 * 97 * - lgrp_plat_node_memory[] Table with physical address range for 98 * each node indexed by node ID 99 * 100 * The code is implemented to make the following always be true: 101 * 102 * lgroup platform handle == node ID == memnode ID 103 * 104 * Moreover, it allows for the proximity domain ID to be equal to all of the 105 * above as long as the proximity domains IDs are numbered from 0 to <number of 106 * nodes - 1>. This is done by hashing each proximity domain ID into the range 107 * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID 108 * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] 109 * and be assigned node ID N. If the proximity domain IDs aren't numbered 110 * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into 111 * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs 112 * to node IDs. However, the proximity domain IDs may not map to the 113 * equivalent node ID since we want to keep the node IDs numbered from 0 to 114 * <number of nodes - 1> to minimize cost of searching and potentially space. 115 * 116 * The code below really tries to do the above. However, the virtual memory 117 * system expects the memnodes which describe the physical address range for 118 * each NUMA node to be arranged in ascending order by physical address. (:-( 119 * Otherwise, the kernel will panic in different semi-random places in the VM 120 * system (see CR#6816963). 121 * 122 * Consequently, this module has to try to sort the nodes in ascending order by 123 * each node's starting physical address to try to meet this "constraint" in 124 * the VM system (see lgrp_plat_node_sort()). Also, the lowest numbered 125 * proximity domain ID in the system is deteremined and used to make the lowest 126 * numbered proximity domain map to node 0 in hopes that the proximity domains 127 * are sorted in ascending order by physical address already even if their IDs 128 * don't start at 0 (see NODE_DOMAIN_HASH() and lgrp_plat_srat_domains()). 129 * Finally, it is important to note that these workarounds may not be 130 * sufficient if/when memory hotplugging is supported and the VM system may 131 * ultimately need to be fixed to handle this.... 132 */ 133 134 135 #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */ 136 #include <sys/bootconf.h> 137 #include <sys/cmn_err.h> 138 #include <sys/controlregs.h> 139 #include <sys/cpupart.h> 140 #include <sys/cpuvar.h> 141 #include <sys/lgrp.h> 142 #include <sys/machsystm.h> 143 #include <sys/memlist.h> 144 #include <sys/memnode.h> 145 #include <sys/mman.h> 146 #include <sys/pci_cfgspace.h> 147 #include <sys/pci_impl.h> 148 #include <sys/param.h> 149 #include <sys/pghw.h> 150 #include <sys/promif.h> /* for prom_printf() */ 151 #include <sys/sysmacros.h> 152 #include <sys/systm.h> 153 #include <sys/thread.h> 154 #include <sys/types.h> 155 #include <sys/var.h> 156 #include <sys/x86_archext.h> /* for x86_feature and X86_AMD */ 157 #include <vm/hat_i86.h> 158 #include <vm/seg_kmem.h> 159 #include <vm/vm_dep.h> 160 161 #include "acpi_fw.h" /* for SRAT and SLIT */ 162 163 164 #define MAX_NODES 8 165 #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) 166 167 /* 168 * Constants for configuring probing 169 */ 170 #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ 171 #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ 172 #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ 173 174 /* 175 * Flags for probing 176 */ 177 #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ 178 #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ 179 #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ 180 181 /* 182 * Hash proximity domain ID into node to domain mapping table "mod" number of 183 * nodes to minimize span of entries used and try to have lowest numbered 184 * proximity domain be node 0 185 */ 186 #define NODE_DOMAIN_HASH(domain, node_cnt) \ 187 ((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \ 188 ((domain) - lgrp_plat_prox_domain_min) % node_cnt) 189 190 191 /* 192 * CPU to node ID mapping structure (only used with SRAT) 193 */ 194 typedef struct cpu_node_map { 195 int exists; 196 uint_t node; 197 uint32_t apicid; 198 uint32_t prox_domain; 199 } cpu_node_map_t; 200 201 /* 202 * Latency statistics 203 */ 204 typedef struct lgrp_plat_latency_stats { 205 hrtime_t latencies[MAX_NODES][MAX_NODES]; 206 hrtime_t latency_max; 207 hrtime_t latency_min; 208 } lgrp_plat_latency_stats_t; 209 210 /* 211 * Memory configuration for probing 212 */ 213 typedef struct lgrp_plat_probe_mem_config { 214 size_t probe_memsize; /* how much memory to probe per node */ 215 caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ 216 pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ 217 } lgrp_plat_probe_mem_config_t; 218 219 /* 220 * Statistics kept for probing 221 */ 222 typedef struct lgrp_plat_probe_stats { 223 hrtime_t flush_cost; 224 hrtime_t probe_cost; 225 hrtime_t probe_cost_total; 226 hrtime_t probe_error_code; 227 hrtime_t probe_errors[MAX_NODES][MAX_NODES]; 228 int probe_suspect[MAX_NODES][MAX_NODES]; 229 hrtime_t probe_max[MAX_NODES][MAX_NODES]; 230 hrtime_t probe_min[MAX_NODES][MAX_NODES]; 231 } lgrp_plat_probe_stats_t; 232 233 /* 234 * Node to proximity domain ID mapping structure (only used with SRAT) 235 */ 236 typedef struct node_domain_map { 237 int exists; 238 uint32_t prox_domain; 239 } node_domain_map_t; 240 241 /* 242 * Node ID and starting and ending page for physical memory in node 243 */ 244 typedef struct node_phys_addr_map { 245 pfn_t start; 246 pfn_t end; 247 int exists; 248 uint32_t prox_domain; 249 } node_phys_addr_map_t; 250 251 /* 252 * Number of CPUs for which we got APIC IDs 253 */ 254 static int lgrp_plat_apic_ncpus = 0; 255 256 /* 257 * CPU to node ID mapping table (only used for SRAT) 258 */ 259 static cpu_node_map_t lgrp_plat_cpu_node[NCPU]; 260 261 /* 262 * Latency statistics 263 */ 264 lgrp_plat_latency_stats_t lgrp_plat_lat_stats; 265 266 /* 267 * Whether memory is interleaved across nodes causing MPO to be disabled 268 */ 269 static int lgrp_plat_mem_intrlv = 0; 270 271 /* 272 * Node ID to proximity domain ID mapping table (only used for SRAT) 273 */ 274 static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; 275 276 /* 277 * Physical address range for memory in each node 278 */ 279 static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES]; 280 281 /* 282 * Statistics gotten from probing 283 */ 284 static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; 285 286 /* 287 * Memory configuration for probing 288 */ 289 static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; 290 291 /* 292 * Lowest proximity domain ID seen in ACPI SRAT 293 */ 294 static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX; 295 296 /* 297 * Error code from processing ACPI SRAT 298 */ 299 static int lgrp_plat_srat_error = 0; 300 301 /* 302 * Error code from processing ACPI SLIT 303 */ 304 static int lgrp_plat_slit_error = 0; 305 306 /* 307 * Allocate lgroup array statically 308 */ 309 static lgrp_t lgrp_space[NLGRP]; 310 static int nlgrps_alloc; 311 312 313 /* 314 * Enable finding and using minimum proximity domain ID when hashing 315 */ 316 int lgrp_plat_domain_min_enable = 1; 317 318 /* 319 * Number of nodes in system 320 */ 321 uint_t lgrp_plat_node_cnt = 1; 322 323 /* 324 * Enable sorting nodes in ascending order by starting physical address 325 */ 326 int lgrp_plat_node_sort_enable = 1; 327 328 /* 329 * Configuration Parameters for Probing 330 * - lgrp_plat_probe_flags Flags to specify enabling probing, probe 331 * operation, etc. 332 * - lgrp_plat_probe_nrounds How many rounds of probing to do 333 * - lgrp_plat_probe_nsamples Number of samples to take when probing each 334 * node 335 * - lgrp_plat_probe_nreads Number of times to read vendor ID from 336 * Northbridge for each probe 337 */ 338 uint_t lgrp_plat_probe_flags = 0; 339 int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; 340 int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; 341 int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; 342 343 /* 344 * Enable use of ACPI System Resource Affinity Table (SRAT) and System 345 * Locality Information Table (SLIT) 346 */ 347 int lgrp_plat_srat_enable = 1; 348 int lgrp_plat_slit_enable = 1; 349 350 /* 351 * Static array to hold lgroup statistics 352 */ 353 struct lgrp_stats lgrp_stats[NLGRP]; 354 355 356 /* 357 * Forward declarations of platform interface routines 358 */ 359 void plat_build_mem_nodes(struct memlist *list); 360 361 int plat_lgrphand_to_mem_node(lgrp_handle_t hand); 362 363 lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); 364 365 int plat_mnode_xcheck(pfn_t pfncnt); 366 367 int plat_pfn_to_mem_node(pfn_t pfn); 368 369 /* 370 * Forward declarations of lgroup platform interface routines 371 */ 372 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 373 374 void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); 375 376 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); 377 378 void lgrp_plat_init(void); 379 380 int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); 381 382 void lgrp_plat_main_init(void); 383 384 int lgrp_plat_max_lgrps(void); 385 386 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, 387 lgrp_mem_query_t query); 388 389 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); 390 391 void lgrp_plat_probe(void); 392 393 lgrp_handle_t lgrp_plat_root_hand(void); 394 395 396 /* 397 * Forward declarations of local routines 398 */ 399 static int is_opteron(void); 400 401 static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, 402 int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, 403 uint32_t domain); 404 405 static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node); 406 407 static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, 408 int node_cnt, uint32_t domain); 409 410 static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 411 lgrp_plat_latency_stats_t *lat_stats, 412 lgrp_plat_probe_stats_t *probe_stats); 413 414 static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 415 lgrp_plat_latency_stats_t *lat_stats); 416 417 static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); 418 419 static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, 420 int node_cnt, uint32_t domain); 421 422 static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain, 423 int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start, 424 uint64_t end, uint32_t domain); 425 426 static void lgrp_plat_node_sort(node_domain_map_t *node_domain, 427 int node_cnt, cpu_node_map_t *cpu_node, int cpu_count, 428 node_phys_addr_map_t *node_memory); 429 430 static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 431 lgrp_plat_probe_mem_config_t *probe_mem_config, 432 lgrp_plat_latency_stats_t *lat_stats, 433 lgrp_plat_probe_stats_t *probe_stats); 434 435 static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); 436 437 static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 438 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats); 439 440 static int lgrp_plat_process_srat(struct srat *tp, 441 uint32_t *prox_domain_min, node_domain_map_t *node_domain, 442 cpu_node_map_t *cpu_node, int cpu_count, 443 node_phys_addr_map_t *node_memory); 444 445 static int lgrp_plat_srat_domains(struct srat *tp, 446 uint32_t *prox_domain_min); 447 448 static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 449 lgrp_plat_latency_stats_t *lat_stats); 450 451 static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 452 node_phys_addr_map_t *node_memory); 453 454 static hrtime_t opt_probe_vendor(int dest_node, int nreads); 455 456 457 /* 458 * PLATFORM INTERFACE ROUTINES 459 */ 460 461 /* 462 * Configure memory nodes for machines with more than one node (ie NUMA) 463 */ 464 void 465 plat_build_mem_nodes(struct memlist *list) 466 { 467 pfn_t cur_start; /* start addr of subrange */ 468 pfn_t cur_end; /* end addr of subrange */ 469 pfn_t start; /* start addr of whole range */ 470 pfn_t end; /* end addr of whole range */ 471 472 /* 473 * Boot install lists are arranged <addr, len>, ... 474 */ 475 while (list) { 476 int node; 477 478 start = list->address >> PAGESHIFT; 479 end = (list->address + list->size - 1) >> PAGESHIFT; 480 481 if (start > physmax) { 482 list = list->next; 483 continue; 484 } 485 if (end > physmax) 486 end = physmax; 487 488 /* 489 * When there is only one memnode, just add memory to memnode 490 */ 491 if (max_mem_nodes == 1) { 492 mem_node_add_slice(start, end); 493 list = list->next; 494 continue; 495 } 496 497 /* 498 * mem_node_add_slice() expects to get a memory range that 499 * is within one memnode, so need to split any memory range 500 * that spans multiple memnodes into subranges that are each 501 * contained within one memnode when feeding them to 502 * mem_node_add_slice() 503 */ 504 cur_start = start; 505 do { 506 node = plat_pfn_to_mem_node(cur_start); 507 508 /* 509 * Panic if DRAM address map registers or SRAT say 510 * memory in node doesn't exist or address from 511 * boot installed memory list entry isn't in this node. 512 * This shouldn't happen and rest of code can't deal 513 * with this if it does. 514 */ 515 if (node < 0 || node >= lgrp_plat_node_cnt || 516 !lgrp_plat_node_memory[node].exists || 517 cur_start < lgrp_plat_node_memory[node].start || 518 cur_start > lgrp_plat_node_memory[node].end) { 519 cmn_err(CE_PANIC, "Don't know which memnode " 520 "to add installed memory address 0x%lx\n", 521 cur_start); 522 } 523 524 /* 525 * End of current subrange should not span memnodes 526 */ 527 cur_end = end; 528 if (lgrp_plat_node_memory[node].exists && 529 cur_end > lgrp_plat_node_memory[node].end) 530 cur_end = lgrp_plat_node_memory[node].end; 531 532 mem_node_add_slice(cur_start, cur_end); 533 534 /* 535 * Next subrange starts after end of current one 536 */ 537 cur_start = cur_end + 1; 538 } while (cur_end < end); 539 540 list = list->next; 541 } 542 mem_node_physalign = 0; 543 mem_node_pfn_shift = 0; 544 } 545 546 547 int 548 plat_lgrphand_to_mem_node(lgrp_handle_t hand) 549 { 550 if (max_mem_nodes == 1) 551 return (0); 552 553 return ((int)hand); 554 } 555 556 557 /* 558 * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt 559 * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if 560 * a crossing is found and returns 0 otherwise. 561 */ 562 int 563 plat_mnode_xcheck(pfn_t pfncnt) 564 { 565 int node, prevnode = -1, basenode; 566 pfn_t ea, sa; 567 568 for (node = 0; node < lgrp_plat_node_cnt; node++) { 569 570 if (lgrp_plat_node_memory[node].exists == 0) 571 continue; 572 573 if (prevnode == -1) { 574 prevnode = node; 575 basenode = node; 576 continue; 577 } 578 579 /* assume x86 node pfn ranges are in increasing order */ 580 ASSERT(lgrp_plat_node_memory[node].start > 581 lgrp_plat_node_memory[prevnode].end); 582 583 /* 584 * continue if the starting address of node is not contiguous 585 * with the previous node. 586 */ 587 588 if (lgrp_plat_node_memory[node].start != 589 (lgrp_plat_node_memory[prevnode].end + 1)) { 590 basenode = node; 591 prevnode = node; 592 continue; 593 } 594 595 /* check if the starting address of node is pfncnt aligned */ 596 if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) { 597 598 /* 599 * at this point, node starts at an unaligned boundary 600 * and is contiguous with the previous node(s) to 601 * basenode. Check if there is an aligned contiguous 602 * range of length pfncnt that crosses this boundary. 603 */ 604 605 sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end, 606 pfncnt); 607 ea = P2ROUNDUP((lgrp_plat_node_memory[node].start), 608 pfncnt); 609 610 ASSERT((ea - sa) == pfncnt); 611 if (sa >= lgrp_plat_node_memory[basenode].start && 612 ea <= (lgrp_plat_node_memory[node].end + 1)) 613 return (1); 614 } 615 prevnode = node; 616 } 617 return (0); 618 } 619 620 621 lgrp_handle_t 622 plat_mem_node_to_lgrphand(int mnode) 623 { 624 if (max_mem_nodes == 1) 625 return (LGRP_DEFAULT_HANDLE); 626 627 return ((lgrp_handle_t)mnode); 628 } 629 630 631 int 632 plat_pfn_to_mem_node(pfn_t pfn) 633 { 634 int node; 635 636 if (max_mem_nodes == 1) 637 return (0); 638 639 for (node = 0; node < lgrp_plat_node_cnt; node++) { 640 /* 641 * Skip nodes with no memory 642 */ 643 if (!lgrp_plat_node_memory[node].exists) 644 continue; 645 646 if (pfn >= lgrp_plat_node_memory[node].start && 647 pfn <= lgrp_plat_node_memory[node].end) 648 return (node); 649 } 650 651 /* 652 * Didn't find memnode where this PFN lives which should never happen 653 */ 654 ASSERT(node < lgrp_plat_node_cnt); 655 return (-1); 656 } 657 658 659 /* 660 * LGROUP PLATFORM INTERFACE ROUTINES 661 */ 662 663 /* 664 * Allocate additional space for an lgroup. 665 */ 666 /* ARGSUSED */ 667 lgrp_t * 668 lgrp_plat_alloc(lgrp_id_t lgrpid) 669 { 670 lgrp_t *lgrp; 671 672 lgrp = &lgrp_space[nlgrps_alloc++]; 673 if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) 674 return (NULL); 675 return (lgrp); 676 } 677 678 679 /* 680 * Platform handling for (re)configuration changes 681 */ 682 /* ARGSUSED */ 683 void 684 lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) 685 { 686 } 687 688 689 /* 690 * Return the platform handle for the lgroup containing the given CPU 691 */ 692 /* ARGSUSED */ 693 lgrp_handle_t 694 lgrp_plat_cpu_to_hand(processorid_t id) 695 { 696 lgrp_handle_t hand; 697 698 if (lgrp_plat_node_cnt == 1) 699 return (LGRP_DEFAULT_HANDLE); 700 701 hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], 702 lgrp_plat_cpu_node); 703 704 ASSERT(hand != (lgrp_handle_t)-1); 705 if (hand == (lgrp_handle_t)-1) 706 return (LGRP_NULL_HANDLE); 707 708 return (hand); 709 } 710 711 712 /* 713 * Platform-specific initialization of lgroups 714 */ 715 void 716 lgrp_plat_init(void) 717 { 718 #if defined(__xpv) 719 /* 720 * XXPV For now, the hypervisor treats all memory equally. 721 */ 722 lgrp_plat_node_cnt = max_mem_nodes = 1; 723 #else /* __xpv */ 724 uint_t probe_op; 725 u_longlong_t value; 726 727 /* 728 * Get boot property for lgroup topology height limit 729 */ 730 if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0) 731 (void) lgrp_topo_ht_limit_set((int)value); 732 733 /* 734 * Get boot property for enabling/disabling SRAT 735 */ 736 if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0) 737 lgrp_plat_srat_enable = (int)value; 738 739 /* 740 * Get boot property for enabling/disabling SLIT 741 */ 742 if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0) 743 lgrp_plat_slit_enable = (int)value; 744 745 /* 746 * Initialize as a UMA machine 747 */ 748 if (lgrp_topo_ht_limit() == 1) { 749 lgrp_plat_node_cnt = max_mem_nodes = 1; 750 return; 751 } 752 753 /* 754 * Read boot property with CPU to APIC ID mapping table/array and fill 755 * in CPU to node ID mapping table with APIC ID for each CPU 756 */ 757 lgrp_plat_apic_ncpus = 758 lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); 759 760 /* 761 * Determine which CPUs and memory are local to each other and number 762 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) 763 */ 764 if (lgrp_plat_apic_ncpus > 0) { 765 int retval; 766 767 retval = lgrp_plat_process_srat(srat_ptr, 768 &lgrp_plat_prox_domain_min, 769 lgrp_plat_node_domain, lgrp_plat_cpu_node, 770 lgrp_plat_apic_ncpus, lgrp_plat_node_memory); 771 if (retval <= 0) { 772 lgrp_plat_srat_error = retval; 773 lgrp_plat_node_cnt = 1; 774 } else { 775 lgrp_plat_srat_error = 0; 776 lgrp_plat_node_cnt = retval; 777 } 778 } 779 780 /* 781 * Try to use PCI config space registers on Opteron if there's an error 782 * processing CPU to APIC ID mapping or SRAT 783 */ 784 if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && 785 is_opteron()) 786 opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, 787 lgrp_plat_node_memory); 788 789 /* 790 * Don't bother to setup system for multiple lgroups and only use one 791 * memory node when memory is interleaved between any nodes or there is 792 * only one NUMA node 793 * 794 * NOTE: May need to change this for Dynamic Reconfiguration (DR) 795 * when and if it happens for x86/x64 796 */ 797 if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { 798 lgrp_plat_node_cnt = max_mem_nodes = 1; 799 (void) lgrp_topo_ht_limit_set(1); 800 return; 801 } 802 803 /* 804 * Leaf lgroups on x86/x64 architectures contain one physical 805 * processor chip. Tune lgrp_expand_proc_thresh and 806 * lgrp_expand_proc_diff so that lgrp_choose() will spread 807 * things out aggressively. 808 */ 809 lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; 810 lgrp_expand_proc_diff = 0; 811 812 /* 813 * There should be one memnode (physical page free list(s)) for 814 * each node 815 */ 816 max_mem_nodes = lgrp_plat_node_cnt; 817 818 /* 819 * Initialize min and max latency before reading SLIT or probing 820 */ 821 lgrp_plat_lat_stats.latency_min = -1; 822 lgrp_plat_lat_stats.latency_max = 0; 823 824 /* 825 * Determine how far each NUMA node is from each other by 826 * reading ACPI System Locality Information Table (SLIT) if it 827 * exists 828 */ 829 lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, 830 lgrp_plat_node_cnt, lgrp_plat_node_memory, 831 &lgrp_plat_lat_stats); 832 if (lgrp_plat_slit_error == 0) 833 return; 834 835 /* 836 * Probe to determine latency between NUMA nodes when SLIT 837 * doesn't exist or make sense 838 */ 839 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; 840 841 /* 842 * Specify whether to probe using vendor ID register or page copy 843 * if hasn't been specified already or is overspecified 844 */ 845 probe_op = lgrp_plat_probe_flags & 846 (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 847 848 if (probe_op == 0 || 849 probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { 850 lgrp_plat_probe_flags &= 851 ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 852 if (is_opteron()) 853 lgrp_plat_probe_flags |= 854 LGRP_PLAT_PROBE_VENDOR; 855 else 856 lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; 857 } 858 859 /* 860 * Probing errors can mess up the lgroup topology and 861 * force us fall back to a 2 level lgroup topology. 862 * Here we bound how tall the lgroup topology can grow 863 * in hopes of avoiding any anamolies in probing from 864 * messing up the lgroup topology by limiting the 865 * accuracy of the latency topology. 866 * 867 * Assume that nodes will at least be configured in a 868 * ring, so limit height of lgroup topology to be less 869 * than number of nodes on a system with 4 or more 870 * nodes 871 */ 872 if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == 873 lgrp_topo_ht_limit_default()) 874 (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); 875 #endif /* __xpv */ 876 } 877 878 879 /* 880 * Return latency between "from" and "to" lgroups 881 * 882 * This latency number can only be used for relative comparison 883 * between lgroups on the running system, cannot be used across platforms, 884 * and may not reflect the actual latency. It is platform and implementation 885 * specific, so platform gets to decide its value. It would be nice if the 886 * number was at least proportional to make comparisons more meaningful though. 887 */ 888 /* ARGSUSED */ 889 int 890 lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) 891 { 892 lgrp_handle_t src, dest; 893 int node; 894 895 if (max_mem_nodes == 1) 896 return (0); 897 898 /* 899 * Return max latency for root lgroup 900 */ 901 if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) 902 return (lgrp_plat_lat_stats.latency_max); 903 904 src = from; 905 dest = to; 906 907 /* 908 * Return 0 for nodes (lgroup platform handles) out of range 909 */ 910 if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES) 911 return (0); 912 913 /* 914 * Probe from current CPU if its lgroup latencies haven't been set yet 915 * and we are trying to get latency from current CPU to some node 916 */ 917 node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 918 ASSERT(node >= 0 && node < lgrp_plat_node_cnt); 919 if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src) 920 lgrp_plat_probe(); 921 922 return (lgrp_plat_lat_stats.latencies[src][dest]); 923 } 924 925 926 /* 927 * Platform-specific initialization 928 */ 929 void 930 lgrp_plat_main_init(void) 931 { 932 int curnode; 933 int ht_limit; 934 int i; 935 936 /* 937 * Print a notice that MPO is disabled when memory is interleaved 938 * across nodes....Would do this when it is discovered, but can't 939 * because it happens way too early during boot.... 940 */ 941 if (lgrp_plat_mem_intrlv) 942 cmn_err(CE_NOTE, 943 "MPO disabled because memory is interleaved\n"); 944 945 /* 946 * Don't bother to do any probing if it is disabled, there is only one 947 * node, or the height of the lgroup topology less than or equal to 2 948 */ 949 ht_limit = lgrp_topo_ht_limit(); 950 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 951 max_mem_nodes == 1 || ht_limit <= 2) { 952 /* 953 * Setup lgroup latencies for 2 level lgroup topology 954 * (ie. local and remote only) if they haven't been set yet 955 */ 956 if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && 957 lgrp_plat_lat_stats.latency_max == 0) 958 lgrp_plat_2level_setup(lgrp_plat_node_memory, 959 &lgrp_plat_lat_stats); 960 return; 961 } 962 963 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 964 /* 965 * Should have been able to probe from CPU 0 when it was added 966 * to lgroup hierarchy, but may not have been able to then 967 * because it happens so early in boot that gethrtime() hasn't 968 * been initialized. (:-( 969 */ 970 curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 971 ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt); 972 if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) 973 lgrp_plat_probe(); 974 975 return; 976 } 977 978 /* 979 * When probing memory, use one page for every sample to determine 980 * lgroup topology and taking multiple samples 981 */ 982 if (lgrp_plat_probe_mem_config.probe_memsize == 0) 983 lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * 984 lgrp_plat_probe_nsamples; 985 986 /* 987 * Map memory in each node needed for probing to determine latency 988 * topology 989 */ 990 for (i = 0; i < lgrp_plat_node_cnt; i++) { 991 int mnode; 992 993 /* 994 * Skip this node and leave its probe page NULL 995 * if it doesn't have any memory 996 */ 997 mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i); 998 if (!mem_node_config[mnode].exists) { 999 lgrp_plat_probe_mem_config.probe_va[i] = NULL; 1000 continue; 1001 } 1002 1003 /* 1004 * Allocate one kernel virtual page 1005 */ 1006 lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, 1007 lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); 1008 if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { 1009 cmn_err(CE_WARN, 1010 "lgrp_plat_main_init: couldn't allocate memory"); 1011 return; 1012 } 1013 1014 /* 1015 * Get PFN for first page in each node 1016 */ 1017 lgrp_plat_probe_mem_config.probe_pfn[i] = 1018 mem_node_config[mnode].physbase; 1019 1020 /* 1021 * Map virtual page to first page in node 1022 */ 1023 hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], 1024 lgrp_plat_probe_mem_config.probe_memsize, 1025 lgrp_plat_probe_mem_config.probe_pfn[i], 1026 PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, 1027 HAT_LOAD_NOCONSIST); 1028 } 1029 1030 /* 1031 * Probe from current CPU 1032 */ 1033 lgrp_plat_probe(); 1034 } 1035 1036 1037 /* 1038 * Return the maximum number of lgrps supported by the platform. 1039 * Before lgrp topology is known it returns an estimate based on the number of 1040 * nodes. Once topology is known it returns the actual maximim number of lgrps 1041 * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and 1042 * dynamic addition of new nodes, this number may not grow during system 1043 * lifetime (yet). 1044 */ 1045 int 1046 lgrp_plat_max_lgrps(void) 1047 { 1048 return (lgrp_topo_initialized ? 1049 lgrp_alloc_max + 1 : 1050 lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); 1051 } 1052 1053 1054 /* 1055 * Return the number of free pages in an lgroup. 1056 * 1057 * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize 1058 * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the 1059 * number of allocatable base pagesize pages corresponding to the 1060 * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) 1061 * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical 1062 * memory installed, regardless of whether or not it's usable. 1063 */ 1064 pgcnt_t 1065 lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) 1066 { 1067 int mnode; 1068 pgcnt_t npgs = (pgcnt_t)0; 1069 extern struct memlist *phys_avail; 1070 extern struct memlist *phys_install; 1071 1072 1073 if (plathand == LGRP_DEFAULT_HANDLE) 1074 return (lgrp_plat_mem_size_default(plathand, query)); 1075 1076 if (plathand != LGRP_NULL_HANDLE) { 1077 mnode = plat_lgrphand_to_mem_node(plathand); 1078 if (mnode >= 0 && mem_node_config[mnode].exists) { 1079 switch (query) { 1080 case LGRP_MEM_SIZE_FREE: 1081 npgs = MNODE_PGCNT(mnode); 1082 break; 1083 case LGRP_MEM_SIZE_AVAIL: 1084 npgs = mem_node_memlist_pages(mnode, 1085 phys_avail); 1086 break; 1087 case LGRP_MEM_SIZE_INSTALL: 1088 npgs = mem_node_memlist_pages(mnode, 1089 phys_install); 1090 break; 1091 default: 1092 break; 1093 } 1094 } 1095 } 1096 return (npgs); 1097 } 1098 1099 1100 /* 1101 * Return the platform handle of the lgroup that contains the physical memory 1102 * corresponding to the given page frame number 1103 */ 1104 /* ARGSUSED */ 1105 lgrp_handle_t 1106 lgrp_plat_pfn_to_hand(pfn_t pfn) 1107 { 1108 int mnode; 1109 1110 if (max_mem_nodes == 1) 1111 return (LGRP_DEFAULT_HANDLE); 1112 1113 if (pfn > physmax) 1114 return (LGRP_NULL_HANDLE); 1115 1116 mnode = plat_pfn_to_mem_node(pfn); 1117 if (mnode < 0) 1118 return (LGRP_NULL_HANDLE); 1119 1120 return (MEM_NODE_2_LGRPHAND(mnode)); 1121 } 1122 1123 1124 /* 1125 * Probe memory in each node from current CPU to determine latency topology 1126 * 1127 * The probing code will probe the vendor ID register on the Northbridge of 1128 * Opteron processors and probe memory for other processors by default. 1129 * 1130 * Since probing is inherently error prone, the code takes laps across all the 1131 * nodes probing from each node to each of the other nodes some number of 1132 * times. Furthermore, each node is probed some number of times before moving 1133 * onto the next one during each lap. The minimum latency gotten between nodes 1134 * is kept as the latency between the nodes. 1135 * 1136 * After all that, the probe times are adjusted by normalizing values that are 1137 * close to each other and local latencies are made the same. Lastly, the 1138 * latencies are verified to make sure that certain conditions are met (eg. 1139 * local < remote, latency(a, b) == latency(b, a), etc.). 1140 * 1141 * If any of the conditions aren't met, the code will export a NUMA 1142 * configuration with the local CPUs and memory given by the SRAT or PCI config 1143 * space registers and one remote memory latency since it can't tell exactly 1144 * how far each node is from each other. 1145 */ 1146 void 1147 lgrp_plat_probe(void) 1148 { 1149 int from; 1150 int i; 1151 lgrp_plat_latency_stats_t *lat_stats; 1152 hrtime_t probe_time; 1153 int to; 1154 1155 if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 1156 max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) 1157 return; 1158 1159 /* 1160 * Determine ID of node containing current CPU 1161 */ 1162 from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 1163 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 1164 if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) 1165 ASSERT(lgrp_plat_node_domain[from].exists); 1166 1167 /* 1168 * Don't need to probe if got times already 1169 */ 1170 lat_stats = &lgrp_plat_lat_stats; 1171 if (lat_stats->latencies[from][from] != 0) 1172 return; 1173 1174 /* 1175 * Read vendor ID in Northbridge or read and write page(s) 1176 * in each node from current CPU and remember how long it takes, 1177 * so we can build latency topology of machine later. 1178 * This should approximate the memory latency between each node. 1179 */ 1180 for (i = 0; i < lgrp_plat_probe_nrounds; i++) { 1181 for (to = 0; to < lgrp_plat_node_cnt; to++) { 1182 /* 1183 * Get probe time and bail out if can't get it yet 1184 */ 1185 probe_time = lgrp_plat_probe_time(to, 1186 lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config, 1187 &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); 1188 if (probe_time == 0) 1189 return; 1190 1191 /* 1192 * Keep lowest probe time as latency between nodes 1193 */ 1194 if (lat_stats->latencies[from][to] == 0 || 1195 probe_time < lat_stats->latencies[from][to]) 1196 lat_stats->latencies[from][to] = probe_time; 1197 1198 /* 1199 * Update overall minimum and maximum probe times 1200 * across all nodes 1201 */ 1202 if (probe_time < lat_stats->latency_min || 1203 lat_stats->latency_min == -1) 1204 lat_stats->latency_min = probe_time; 1205 if (probe_time > lat_stats->latency_max) 1206 lat_stats->latency_max = probe_time; 1207 } 1208 } 1209 1210 /* 1211 * - Fix up latencies such that local latencies are same, 1212 * latency(i, j) == latency(j, i), etc. (if possible) 1213 * 1214 * - Verify that latencies look ok 1215 * 1216 * - Fallback to just optimizing for local and remote if 1217 * latencies didn't look right 1218 */ 1219 lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats, 1220 &lgrp_plat_probe_stats); 1221 lgrp_plat_probe_stats.probe_error_code = 1222 lgrp_plat_latency_verify(lgrp_plat_node_memory, 1223 &lgrp_plat_lat_stats); 1224 if (lgrp_plat_probe_stats.probe_error_code) 1225 lgrp_plat_2level_setup(lgrp_plat_node_memory, 1226 &lgrp_plat_lat_stats); 1227 } 1228 1229 1230 /* 1231 * Return platform handle for root lgroup 1232 */ 1233 lgrp_handle_t 1234 lgrp_plat_root_hand(void) 1235 { 1236 return (LGRP_DEFAULT_HANDLE); 1237 } 1238 1239 1240 /* 1241 * INTERNAL ROUTINES 1242 */ 1243 1244 1245 /* 1246 * Update CPU to node mapping for given CPU and proximity domain (and returns 1247 * negative numbers for errors and positive ones for success) 1248 */ 1249 static int 1250 lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, 1251 cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) 1252 { 1253 uint_t i; 1254 int node; 1255 1256 /* 1257 * Get node number for proximity domain 1258 */ 1259 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1260 if (node == -1) { 1261 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1262 domain); 1263 if (node == -1) 1264 return (-1); 1265 } 1266 1267 /* 1268 * Search for entry with given APIC ID and fill in its node and 1269 * proximity domain IDs (if they haven't been set already) 1270 */ 1271 for (i = 0; i < nentries; i++) { 1272 /* 1273 * Skip nonexistent entries and ones without matching APIC ID 1274 */ 1275 if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) 1276 continue; 1277 1278 /* 1279 * Just return if entry completely and correctly filled in 1280 * already 1281 */ 1282 if (cpu_node[i].prox_domain == domain && 1283 cpu_node[i].node == node) 1284 return (1); 1285 1286 /* 1287 * Fill in node and proximity domain IDs 1288 */ 1289 cpu_node[i].prox_domain = domain; 1290 cpu_node[i].node = node; 1291 1292 return (0); 1293 } 1294 1295 /* 1296 * Return error when entry for APIC ID wasn't found in table 1297 */ 1298 return (-2); 1299 } 1300 1301 1302 /* 1303 * Get node ID for given CPU 1304 */ 1305 static int 1306 lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node) 1307 { 1308 processorid_t cpuid; 1309 1310 if (cp == NULL) 1311 return (-1); 1312 1313 cpuid = cp->cpu_id; 1314 if (cpuid < 0 || cpuid >= max_ncpus) 1315 return (-1); 1316 1317 /* 1318 * SRAT doesn't exist, isn't enabled, or there was an error processing 1319 * it, so return chip ID for Opteron and -1 otherwise. 1320 */ 1321 if (srat_ptr == NULL || !lgrp_plat_srat_enable || 1322 lgrp_plat_srat_error) { 1323 if (is_opteron()) 1324 return (pg_plat_hw_instance_id(cp, PGHW_CHIP)); 1325 return (-1); 1326 } 1327 1328 /* 1329 * Return -1 when CPU to node ID mapping entry doesn't exist for given 1330 * CPU 1331 */ 1332 if (!cpu_node[cpuid].exists) 1333 return (-1); 1334 1335 return (cpu_node[cpuid].node); 1336 } 1337 1338 1339 /* 1340 * Return node number for given proximity domain/system locality 1341 */ 1342 static int 1343 lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, 1344 uint32_t domain) 1345 { 1346 uint_t node; 1347 uint_t start; 1348 1349 /* 1350 * Hash proximity domain ID into node to domain mapping table (array), 1351 * search for entry with matching proximity domain ID, and return index 1352 * of matching entry as node ID. 1353 */ 1354 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1355 do { 1356 if (node_domain[node].prox_domain == domain && 1357 node_domain[node].exists) 1358 return (node); 1359 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1360 } while (node != start); 1361 return (-1); 1362 } 1363 1364 1365 /* 1366 * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to 1367 * be considered same 1368 */ 1369 #define LGRP_LAT_TOLERANCE_SHIFT 4 1370 1371 int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; 1372 1373 1374 /* 1375 * Adjust latencies between nodes to be symmetric, normalize latencies between 1376 * any nodes that are within some tolerance to be same, and make local 1377 * latencies be same 1378 */ 1379 static void 1380 lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 1381 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 1382 { 1383 int i; 1384 int j; 1385 int k; 1386 int l; 1387 u_longlong_t max; 1388 u_longlong_t min; 1389 u_longlong_t t; 1390 u_longlong_t t1; 1391 u_longlong_t t2; 1392 const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; 1393 int lat_corrected[MAX_NODES][MAX_NODES]; 1394 1395 /* 1396 * Nothing to do when this is an UMA machine or don't have args needed 1397 */ 1398 if (max_mem_nodes == 1) 1399 return; 1400 1401 ASSERT(node_memory != NULL && lat_stats != NULL && 1402 probe_stats != NULL); 1403 1404 /* 1405 * Make sure that latencies are symmetric between any two nodes 1406 * (ie. latency(node0, node1) == latency(node1, node0)) 1407 */ 1408 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1409 if (!node_memory[i].exists) 1410 continue; 1411 1412 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1413 if (!node_memory[j].exists) 1414 continue; 1415 1416 t1 = lat_stats->latencies[i][j]; 1417 t2 = lat_stats->latencies[j][i]; 1418 1419 if (t1 == 0 || t2 == 0 || t1 == t2) 1420 continue; 1421 1422 /* 1423 * Latencies should be same 1424 * - Use minimum of two latencies which should be same 1425 * - Track suspect probe times not within tolerance of 1426 * min value 1427 * - Remember how much values are corrected by 1428 */ 1429 if (t1 > t2) { 1430 t = t2; 1431 probe_stats->probe_errors[i][j] += t1 - t2; 1432 if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { 1433 probe_stats->probe_suspect[i][j]++; 1434 probe_stats->probe_suspect[j][i]++; 1435 } 1436 } else if (t2 > t1) { 1437 t = t1; 1438 probe_stats->probe_errors[j][i] += t2 - t1; 1439 if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { 1440 probe_stats->probe_suspect[i][j]++; 1441 probe_stats->probe_suspect[j][i]++; 1442 } 1443 } 1444 1445 lat_stats->latencies[i][j] = 1446 lat_stats->latencies[j][i] = t; 1447 lgrp_config(cflag, t1, t); 1448 lgrp_config(cflag, t2, t); 1449 } 1450 } 1451 1452 /* 1453 * Keep track of which latencies get corrected 1454 */ 1455 for (i = 0; i < MAX_NODES; i++) 1456 for (j = 0; j < MAX_NODES; j++) 1457 lat_corrected[i][j] = 0; 1458 1459 /* 1460 * For every two nodes, see whether there is another pair of nodes which 1461 * are about the same distance apart and make the latencies be the same 1462 * if they are close enough together 1463 */ 1464 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1465 if (!node_memory[i].exists) 1466 continue; 1467 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1468 if (!node_memory[j].exists) 1469 continue; 1470 /* 1471 * Pick one pair of nodes (i, j) 1472 * and get latency between them 1473 */ 1474 t1 = lat_stats->latencies[i][j]; 1475 1476 /* 1477 * Skip this pair of nodes if there isn't a latency 1478 * for it yet 1479 */ 1480 if (t1 == 0) 1481 continue; 1482 1483 for (k = 0; k < lgrp_plat_node_cnt; k++) { 1484 if (!node_memory[k].exists) 1485 continue; 1486 for (l = 0; l < lgrp_plat_node_cnt; l++) { 1487 if (!node_memory[l].exists) 1488 continue; 1489 /* 1490 * Pick another pair of nodes (k, l) 1491 * not same as (i, j) and get latency 1492 * between them 1493 */ 1494 if (k == i && l == j) 1495 continue; 1496 1497 t2 = lat_stats->latencies[k][l]; 1498 1499 /* 1500 * Skip this pair of nodes if there 1501 * isn't a latency for it yet 1502 */ 1503 1504 if (t2 == 0) 1505 continue; 1506 1507 /* 1508 * Skip nodes (k, l) if they already 1509 * have same latency as (i, j) or 1510 * their latency isn't close enough to 1511 * be considered/made the same 1512 */ 1513 if (t1 == t2 || (t1 > t2 && t1 - t2 > 1514 t1 >> lgrp_plat_probe_lt_shift) || 1515 (t2 > t1 && t2 - t1 > 1516 t2 >> lgrp_plat_probe_lt_shift)) 1517 continue; 1518 1519 /* 1520 * Make latency(i, j) same as 1521 * latency(k, l), try to use latency 1522 * that has been adjusted already to get 1523 * more consistency (if possible), and 1524 * remember which latencies were 1525 * adjusted for next time 1526 */ 1527 if (lat_corrected[i][j]) { 1528 t = t1; 1529 lgrp_config(cflag, t2, t); 1530 t2 = t; 1531 } else if (lat_corrected[k][l]) { 1532 t = t2; 1533 lgrp_config(cflag, t1, t); 1534 t1 = t; 1535 } else { 1536 if (t1 > t2) 1537 t = t2; 1538 else 1539 t = t1; 1540 lgrp_config(cflag, t1, t); 1541 lgrp_config(cflag, t2, t); 1542 t1 = t2 = t; 1543 } 1544 1545 lat_stats->latencies[i][j] = 1546 lat_stats->latencies[k][l] = t; 1547 1548 lat_corrected[i][j] = 1549 lat_corrected[k][l] = 1; 1550 } 1551 } 1552 } 1553 } 1554 1555 /* 1556 * Local latencies should be same 1557 * - Find min and max local latencies 1558 * - Make all local latencies be minimum 1559 */ 1560 min = -1; 1561 max = 0; 1562 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1563 if (!node_memory[i].exists) 1564 continue; 1565 t = lat_stats->latencies[i][i]; 1566 if (t == 0) 1567 continue; 1568 if (min == -1 || t < min) 1569 min = t; 1570 if (t > max) 1571 max = t; 1572 } 1573 if (min != max) { 1574 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1575 int local; 1576 1577 if (!node_memory[i].exists) 1578 continue; 1579 1580 local = lat_stats->latencies[i][i]; 1581 if (local == 0) 1582 continue; 1583 1584 /* 1585 * Track suspect probe times that aren't within 1586 * tolerance of minimum local latency and how much 1587 * probe times are corrected by 1588 */ 1589 if (local - min > min >> lgrp_plat_probe_lt_shift) 1590 probe_stats->probe_suspect[i][i]++; 1591 1592 probe_stats->probe_errors[i][i] += local - min; 1593 1594 /* 1595 * Make local latencies be minimum 1596 */ 1597 lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); 1598 lat_stats->latencies[i][i] = min; 1599 } 1600 } 1601 1602 /* 1603 * Determine max probe time again since just adjusted latencies 1604 */ 1605 lat_stats->latency_max = 0; 1606 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1607 if (!node_memory[i].exists) 1608 continue; 1609 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1610 if (!node_memory[j].exists) 1611 continue; 1612 t = lat_stats->latencies[i][j]; 1613 if (t > lat_stats->latency_max) 1614 lat_stats->latency_max = t; 1615 } 1616 } 1617 } 1618 1619 1620 /* 1621 * Verify following about latencies between nodes: 1622 * 1623 * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) 1624 * - Local latencies same 1625 * - Local < remote 1626 * - Number of latencies seen is reasonable 1627 * - Number of occurrences of a given latency should be more than 1 1628 * 1629 * Returns: 1630 * 0 Success 1631 * -1 Not symmetric 1632 * -2 Local latencies not same 1633 * -3 Local >= remote 1634 */ 1635 static int 1636 lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 1637 lgrp_plat_latency_stats_t *lat_stats) 1638 { 1639 int i; 1640 int j; 1641 u_longlong_t t1; 1642 u_longlong_t t2; 1643 1644 ASSERT(node_memory != NULL && lat_stats != NULL); 1645 1646 /* 1647 * Nothing to do when this is an UMA machine, lgroup topology is 1648 * limited to 2 levels, or there aren't any probe times yet 1649 */ 1650 if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || 1651 lat_stats->latencies[0][0] == 0) 1652 return (0); 1653 1654 /* 1655 * Make sure that latencies are symmetric between any two nodes 1656 * (ie. latency(node0, node1) == latency(node1, node0)) 1657 */ 1658 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1659 if (!node_memory[i].exists) 1660 continue; 1661 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1662 if (!node_memory[j].exists) 1663 continue; 1664 t1 = lat_stats->latencies[i][j]; 1665 t2 = lat_stats->latencies[j][i]; 1666 1667 if (t1 == 0 || t2 == 0 || t1 == t2) 1668 continue; 1669 1670 return (-1); 1671 } 1672 } 1673 1674 /* 1675 * Local latencies should be same 1676 */ 1677 t1 = lat_stats->latencies[0][0]; 1678 for (i = 1; i < lgrp_plat_node_cnt; i++) { 1679 if (!node_memory[i].exists) 1680 continue; 1681 1682 t2 = lat_stats->latencies[i][i]; 1683 if (t2 == 0) 1684 continue; 1685 1686 if (t1 == 0) { 1687 t1 = t2; 1688 continue; 1689 } 1690 1691 if (t1 != t2) 1692 return (-2); 1693 } 1694 1695 /* 1696 * Local latencies should be less than remote 1697 */ 1698 if (t1) { 1699 for (i = 0; i < lgrp_plat_node_cnt; i++) { 1700 if (!node_memory[i].exists) 1701 continue; 1702 for (j = 0; j < lgrp_plat_node_cnt; j++) { 1703 if (!node_memory[j].exists) 1704 continue; 1705 t2 = lat_stats->latencies[i][j]; 1706 if (i == j || t2 == 0) 1707 continue; 1708 1709 if (t1 >= t2) 1710 return (-3); 1711 } 1712 } 1713 } 1714 1715 return (0); 1716 } 1717 1718 1719 /* 1720 * Return the number of free, allocatable, or installed 1721 * pages in an lgroup 1722 * This is a copy of the MAX_MEM_NODES == 1 version of the routine 1723 * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup 1724 */ 1725 /* ARGSUSED */ 1726 static pgcnt_t 1727 lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) 1728 { 1729 struct memlist *mlist; 1730 pgcnt_t npgs = 0; 1731 extern struct memlist *phys_avail; 1732 extern struct memlist *phys_install; 1733 1734 switch (query) { 1735 case LGRP_MEM_SIZE_FREE: 1736 return ((pgcnt_t)freemem); 1737 case LGRP_MEM_SIZE_AVAIL: 1738 memlist_read_lock(); 1739 for (mlist = phys_avail; mlist; mlist = mlist->next) 1740 npgs += btop(mlist->size); 1741 memlist_read_unlock(); 1742 return (npgs); 1743 case LGRP_MEM_SIZE_INSTALL: 1744 memlist_read_lock(); 1745 for (mlist = phys_install; mlist; mlist = mlist->next) 1746 npgs += btop(mlist->size); 1747 memlist_read_unlock(); 1748 return (npgs); 1749 default: 1750 return ((pgcnt_t)0); 1751 } 1752 } 1753 1754 1755 /* 1756 * Update node to proximity domain mappings for given domain and return node ID 1757 */ 1758 static int 1759 lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, 1760 uint32_t domain) 1761 { 1762 uint_t node; 1763 uint_t start; 1764 1765 /* 1766 * Hash proximity domain ID into node to domain mapping table (array) 1767 * and add entry for it into first non-existent or matching entry found 1768 */ 1769 node = start = NODE_DOMAIN_HASH(domain, node_cnt); 1770 do { 1771 /* 1772 * Entry doesn't exist yet, so create one for this proximity 1773 * domain and return node ID which is index into mapping table. 1774 */ 1775 if (!node_domain[node].exists) { 1776 node_domain[node].exists = 1; 1777 node_domain[node].prox_domain = domain; 1778 return (node); 1779 } 1780 1781 /* 1782 * Entry exists for this proximity domain already, so just 1783 * return node ID (index into table). 1784 */ 1785 if (node_domain[node].prox_domain == domain) 1786 return (node); 1787 node = NODE_DOMAIN_HASH(node + 1, node_cnt); 1788 } while (node != start); 1789 1790 /* 1791 * Ran out of supported number of entries which shouldn't happen.... 1792 */ 1793 ASSERT(node != start); 1794 return (-1); 1795 } 1796 1797 1798 /* 1799 * Update node memory information for given proximity domain with specified 1800 * starting and ending physical address range (and return positive numbers for 1801 * success and negative ones for errors) 1802 */ 1803 static int 1804 lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt, 1805 node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end, 1806 uint32_t domain) 1807 { 1808 int node; 1809 1810 /* 1811 * Get node number for proximity domain 1812 */ 1813 node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 1814 if (node == -1) { 1815 node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1816 domain); 1817 if (node == -1) 1818 return (-1); 1819 } 1820 1821 /* 1822 * Create entry in table for node if it doesn't exist 1823 */ 1824 if (!node_memory[node].exists) { 1825 node_memory[node].exists = 1; 1826 node_memory[node].start = btop(start); 1827 node_memory[node].end = btop(end); 1828 node_memory[node].prox_domain = domain; 1829 return (0); 1830 } 1831 1832 /* 1833 * Entry already exists for this proximity domain 1834 * 1835 * There may be more than one SRAT memory entry for a domain, so we may 1836 * need to update existing start or end address for the node. 1837 */ 1838 if (node_memory[node].prox_domain == domain) { 1839 if (btop(start) < node_memory[node].start) 1840 node_memory[node].start = btop(start); 1841 if (btop(end) > node_memory[node].end) 1842 node_memory[node].end = btop(end); 1843 return (1); 1844 } 1845 return (-2); 1846 } 1847 1848 1849 /* 1850 * Have to sort node by starting physical address because VM system (physical 1851 * page free list management) assumes and expects memnodes to be sorted in 1852 * ascending order by physical address. If not, the kernel will panic in 1853 * potentially a number of different places. (:-( 1854 * NOTE: This workaround will not be sufficient if/when hotplugging memory is 1855 * supported on x86/x64. 1856 */ 1857 static void 1858 lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt, 1859 cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory) 1860 { 1861 boolean_t found; 1862 int i; 1863 int j; 1864 int n; 1865 boolean_t sorted; 1866 boolean_t swapped; 1867 1868 if (!lgrp_plat_node_sort_enable || node_cnt <= 1 || 1869 node_domain == NULL || node_memory == NULL) 1870 return; 1871 1872 /* 1873 * Sorted already? 1874 */ 1875 sorted = B_TRUE; 1876 for (i = 0; i < node_cnt - 1; i++) { 1877 /* 1878 * Skip entries that don't exist 1879 */ 1880 if (!node_memory[i].exists) 1881 continue; 1882 1883 /* 1884 * Try to find next existing entry to compare against 1885 */ 1886 found = B_FALSE; 1887 for (j = i + 1; j < node_cnt; j++) { 1888 if (node_memory[j].exists) { 1889 found = B_TRUE; 1890 break; 1891 } 1892 } 1893 1894 /* 1895 * Done if no more existing entries to compare against 1896 */ 1897 if (found == B_FALSE) 1898 break; 1899 1900 /* 1901 * Not sorted if starting address of current entry is bigger 1902 * than starting address of next existing entry 1903 */ 1904 if (node_memory[i].start > node_memory[j].start) { 1905 sorted = B_FALSE; 1906 break; 1907 } 1908 } 1909 1910 /* 1911 * Don't need to sort if sorted already 1912 */ 1913 if (sorted == B_TRUE) 1914 return; 1915 1916 /* 1917 * Just use bubble sort since number of nodes is small 1918 */ 1919 n = node_cnt; 1920 do { 1921 swapped = B_FALSE; 1922 n--; 1923 for (i = 0; i < n; i++) { 1924 /* 1925 * Skip entries that don't exist 1926 */ 1927 if (!node_memory[i].exists) 1928 continue; 1929 1930 /* 1931 * Try to find next existing entry to compare against 1932 */ 1933 found = B_FALSE; 1934 for (j = i + 1; j <= n; j++) { 1935 if (node_memory[j].exists) { 1936 found = B_TRUE; 1937 break; 1938 } 1939 } 1940 1941 /* 1942 * Done if no more existing entries to compare against 1943 */ 1944 if (found == B_FALSE) 1945 break; 1946 1947 if (node_memory[i].start > node_memory[j].start) { 1948 node_phys_addr_map_t save_addr; 1949 node_domain_map_t save_node; 1950 1951 /* 1952 * Swap node to proxmity domain ID assignments 1953 */ 1954 bcopy(&node_domain[i], &save_node, 1955 sizeof (node_domain_map_t)); 1956 bcopy(&node_domain[j], &node_domain[i], 1957 sizeof (node_domain_map_t)); 1958 bcopy(&save_node, &node_domain[j], 1959 sizeof (node_domain_map_t)); 1960 1961 /* 1962 * Swap node to physical memory assignments 1963 */ 1964 bcopy(&node_memory[i], &save_addr, 1965 sizeof (node_phys_addr_map_t)); 1966 bcopy(&node_memory[j], &node_memory[i], 1967 sizeof (node_phys_addr_map_t)); 1968 bcopy(&save_addr, &node_memory[j], 1969 sizeof (node_phys_addr_map_t)); 1970 swapped = B_TRUE; 1971 } 1972 } 1973 } while (swapped == B_TRUE); 1974 1975 /* 1976 * Check to make sure that CPUs assigned to correct node IDs now since 1977 * node to proximity domain ID assignments may have been changed above 1978 */ 1979 if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1) 1980 return; 1981 for (i = 0; i < cpu_count; i++) { 1982 int node; 1983 1984 node = lgrp_plat_domain_to_node(node_domain, node_cnt, 1985 cpu_node[i].prox_domain); 1986 if (cpu_node[i].node != node) 1987 cpu_node[i].node = node; 1988 } 1989 1990 } 1991 1992 1993 /* 1994 * Return time needed to probe from current CPU to memory in given node 1995 */ 1996 static hrtime_t 1997 lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 1998 lgrp_plat_probe_mem_config_t *probe_mem_config, 1999 lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 2000 { 2001 caddr_t buf; 2002 hrtime_t elapsed; 2003 hrtime_t end; 2004 int from; 2005 int i; 2006 int ipl; 2007 hrtime_t max; 2008 hrtime_t min; 2009 hrtime_t start; 2010 extern int use_sse_pagecopy; 2011 2012 /* 2013 * Determine ID of node containing current CPU 2014 */ 2015 from = lgrp_plat_cpu_to_node(CPU, cpu_node); 2016 ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 2017 2018 /* 2019 * Do common work for probing main memory 2020 */ 2021 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { 2022 /* 2023 * Skip probing any nodes without memory and 2024 * set probe time to 0 2025 */ 2026 if (probe_mem_config->probe_va[to] == NULL) { 2027 lat_stats->latencies[from][to] = 0; 2028 return (0); 2029 } 2030 2031 /* 2032 * Invalidate caches once instead of once every sample 2033 * which should cut cost of probing by a lot 2034 */ 2035 probe_stats->flush_cost = gethrtime(); 2036 invalidate_cache(); 2037 probe_stats->flush_cost = gethrtime() - 2038 probe_stats->flush_cost; 2039 probe_stats->probe_cost_total += probe_stats->flush_cost; 2040 } 2041 2042 /* 2043 * Probe from current CPU to given memory using specified operation 2044 * and take specified number of samples 2045 */ 2046 max = 0; 2047 min = -1; 2048 for (i = 0; i < lgrp_plat_probe_nsamples; i++) { 2049 probe_stats->probe_cost = gethrtime(); 2050 2051 /* 2052 * Can't measure probe time if gethrtime() isn't working yet 2053 */ 2054 if (probe_stats->probe_cost == 0 && gethrtime() == 0) 2055 return (0); 2056 2057 if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 2058 /* 2059 * Measure how long it takes to read vendor ID from 2060 * Northbridge 2061 */ 2062 elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); 2063 } else { 2064 /* 2065 * Measure how long it takes to copy page 2066 * on top of itself 2067 */ 2068 buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); 2069 2070 kpreempt_disable(); 2071 ipl = splhigh(); 2072 start = gethrtime(); 2073 if (use_sse_pagecopy) 2074 hwblkpagecopy(buf, buf); 2075 else 2076 bcopy(buf, buf, PAGESIZE); 2077 end = gethrtime(); 2078 elapsed = end - start; 2079 splx(ipl); 2080 kpreempt_enable(); 2081 } 2082 2083 probe_stats->probe_cost = gethrtime() - 2084 probe_stats->probe_cost; 2085 probe_stats->probe_cost_total += probe_stats->probe_cost; 2086 2087 if (min == -1 || elapsed < min) 2088 min = elapsed; 2089 if (elapsed > max) 2090 max = elapsed; 2091 } 2092 2093 /* 2094 * Update minimum and maximum probe times between 2095 * these two nodes 2096 */ 2097 if (min < probe_stats->probe_min[from][to] || 2098 probe_stats->probe_min[from][to] == 0) 2099 probe_stats->probe_min[from][to] = min; 2100 2101 if (max > probe_stats->probe_max[from][to]) 2102 probe_stats->probe_max[from][to] = max; 2103 2104 return (min); 2105 } 2106 2107 2108 /* 2109 * Read boot property with CPU to APIC ID array, fill in CPU to node ID 2110 * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs. 2111 * 2112 * NOTE: This code assumes that CPU IDs are assigned in order that they appear 2113 * in in cpu_apicid_array boot property which is based on and follows 2114 * same ordering as processor list in ACPI MADT. If the code in 2115 * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns 2116 * CPU IDs ever changes, then this code will need to change too.... 2117 */ 2118 static int 2119 lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) 2120 { 2121 int boot_prop_len; 2122 char *boot_prop_name = BP_CPU_APICID_ARRAY; 2123 uint8_t cpu_apicid_array[UINT8_MAX + 1]; 2124 int i; 2125 int n; 2126 2127 /* 2128 * Nothing to do when no array to fill in or not enough CPUs 2129 */ 2130 if (cpu_node == NULL) 2131 return (-1); 2132 2133 /* 2134 * Check length of property value 2135 */ 2136 boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); 2137 if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array)) 2138 return (-2); 2139 2140 /* 2141 * Calculate number of entries in array and return when there's just 2142 * one CPU since that's not very interesting for NUMA 2143 */ 2144 n = boot_prop_len / sizeof (uint8_t); 2145 if (n == 1) 2146 return (-3); 2147 2148 /* 2149 * Get CPU to APIC ID property value 2150 */ 2151 if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) 2152 return (-4); 2153 2154 /* 2155 * Fill in CPU to node ID mapping table with APIC ID for each CPU 2156 */ 2157 for (i = 0; i < n; i++) { 2158 cpu_node[i].exists = 1; 2159 cpu_node[i].apicid = cpu_apicid_array[i]; 2160 } 2161 2162 /* 2163 * Return number of CPUs based on number of APIC IDs 2164 */ 2165 return (n); 2166 } 2167 2168 2169 /* 2170 * Read ACPI System Locality Information Table (SLIT) to determine how far each 2171 * NUMA node is from each other 2172 */ 2173 static int 2174 lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 2175 node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats) 2176 { 2177 int i; 2178 int j; 2179 int localities; 2180 hrtime_t max; 2181 hrtime_t min; 2182 int retval; 2183 uint8_t *slit_entries; 2184 2185 if (tp == NULL || !lgrp_plat_slit_enable) 2186 return (1); 2187 2188 if (lat_stats == NULL) 2189 return (2); 2190 2191 localities = tp->number; 2192 if (localities != node_cnt) 2193 return (3); 2194 2195 min = lat_stats->latency_min; 2196 max = lat_stats->latency_max; 2197 2198 /* 2199 * Fill in latency matrix based on SLIT entries 2200 */ 2201 slit_entries = tp->entry; 2202 for (i = 0; i < localities; i++) { 2203 for (j = 0; j < localities; j++) { 2204 uint8_t latency; 2205 2206 latency = slit_entries[(i * localities) + j]; 2207 lat_stats->latencies[i][j] = latency; 2208 if (latency < min || min == -1) 2209 min = latency; 2210 if (latency > max) 2211 max = latency; 2212 } 2213 } 2214 2215 /* 2216 * Verify that latencies/distances given in SLIT look reasonable 2217 */ 2218 retval = lgrp_plat_latency_verify(node_memory, lat_stats); 2219 2220 if (retval) { 2221 /* 2222 * Reinitialize (zero) latency table since SLIT doesn't look 2223 * right 2224 */ 2225 for (i = 0; i < localities; i++) { 2226 for (j = 0; j < localities; j++) 2227 lat_stats->latencies[i][j] = 0; 2228 } 2229 } else { 2230 /* 2231 * Update min and max latencies seen since SLIT looks valid 2232 */ 2233 lat_stats->latency_min = min; 2234 lat_stats->latency_max = max; 2235 } 2236 2237 return (retval); 2238 } 2239 2240 2241 /* 2242 * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs 2243 * and memory are local to each other in the same NUMA node and return number 2244 * of nodes 2245 */ 2246 static int 2247 lgrp_plat_process_srat(struct srat *tp, uint32_t *prox_domain_min, 2248 node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, 2249 node_phys_addr_map_t *node_memory) 2250 { 2251 struct srat_item *srat_end; 2252 int i; 2253 struct srat_item *item; 2254 int node_cnt; 2255 int proc_entry_count; 2256 2257 /* 2258 * Nothing to do when no SRAT or disabled 2259 */ 2260 if (tp == NULL || !lgrp_plat_srat_enable) 2261 return (-1); 2262 2263 /* 2264 * Determine number of nodes by counting number of proximity domains in 2265 * SRAT and return if number of nodes is 1 or less since don't need to 2266 * read SRAT then 2267 */ 2268 node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min); 2269 if (node_cnt == 1) 2270 return (1); 2271 else if (node_cnt <= 0) 2272 return (-2); 2273 2274 /* 2275 * Walk through SRAT, examining each CPU and memory entry to determine 2276 * which CPUs and memory belong to which node. 2277 */ 2278 item = tp->list; 2279 srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2280 proc_entry_count = 0; 2281 while (item < srat_end) { 2282 uint32_t apic_id; 2283 uint32_t domain; 2284 uint64_t end; 2285 uint64_t length; 2286 uint64_t start; 2287 2288 switch (item->type) { 2289 case SRAT_PROCESSOR: /* CPU entry */ 2290 if (!(item->i.p.flags & SRAT_ENABLED) || 2291 cpu_node == NULL) 2292 break; 2293 2294 /* 2295 * Calculate domain (node) ID and fill in APIC ID to 2296 * domain/node mapping table 2297 */ 2298 domain = item->i.p.domain1; 2299 for (i = 0; i < 3; i++) { 2300 domain += item->i.p.domain2[i] << 2301 ((i + 1) * 8); 2302 } 2303 apic_id = item->i.p.apic_id; 2304 2305 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2306 cpu_node, cpu_count, apic_id, domain) < 0) 2307 return (-3); 2308 2309 proc_entry_count++; 2310 break; 2311 2312 case SRAT_MEMORY: /* memory entry */ 2313 if (!(item->i.m.flags & SRAT_ENABLED) || 2314 node_memory == NULL) 2315 break; 2316 2317 /* 2318 * Get domain (node) ID and fill in domain/node 2319 * to memory mapping table 2320 */ 2321 domain = item->i.m.domain; 2322 start = item->i.m.base_addr; 2323 length = item->i.m.len; 2324 end = start + length - 1; 2325 2326 if (lgrp_plat_node_memory_update(node_domain, node_cnt, 2327 node_memory, start, end, domain) < 0) 2328 return (-4); 2329 break; 2330 case SRAT_X2APIC: /* x2apic CPU entry */ 2331 if (!(item->i.xp.flags & SRAT_ENABLED) || 2332 cpu_node == NULL) 2333 break; 2334 2335 /* 2336 * Calculate domain (node) ID and fill in APIC ID to 2337 * domain/node mapping table 2338 */ 2339 domain = item->i.xp.domain; 2340 apic_id = item->i.xp.x2apic_id; 2341 2342 if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2343 cpu_node, cpu_count, apic_id, domain) < 0) 2344 return (-3); 2345 2346 proc_entry_count++; 2347 break; 2348 2349 default: 2350 break; 2351 } 2352 2353 item = (struct srat_item *)((uintptr_t)item + item->len); 2354 } 2355 2356 /* 2357 * Should have seen at least as many SRAT processor entries as CPUs 2358 */ 2359 if (proc_entry_count < cpu_count) 2360 return (-5); 2361 2362 /* 2363 * Need to sort nodes by starting physical address since VM system 2364 * assumes and expects memnodes to be sorted in ascending order by 2365 * physical address 2366 */ 2367 lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count, 2368 node_memory); 2369 2370 return (node_cnt); 2371 } 2372 2373 2374 /* 2375 * Return number of proximity domains given in ACPI SRAT 2376 */ 2377 static int 2378 lgrp_plat_srat_domains(struct srat *tp, uint32_t *prox_domain_min) 2379 { 2380 int domain_cnt; 2381 uint32_t domain_min; 2382 struct srat_item *end; 2383 int i; 2384 struct srat_item *item; 2385 node_domain_map_t node_domain[MAX_NODES]; 2386 2387 2388 if (tp == NULL || !lgrp_plat_srat_enable) 2389 return (1); 2390 2391 /* 2392 * Walk through SRAT to find minimum proximity domain ID 2393 */ 2394 domain_min = UINT32_MAX; 2395 item = tp->list; 2396 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2397 while (item < end) { 2398 uint32_t domain; 2399 2400 switch (item->type) { 2401 case SRAT_PROCESSOR: /* CPU entry */ 2402 if (!(item->i.p.flags & SRAT_ENABLED)) { 2403 item = (struct srat_item *)((uintptr_t)item + 2404 item->len); 2405 continue; 2406 } 2407 domain = item->i.p.domain1; 2408 for (i = 0; i < 3; i++) { 2409 domain += item->i.p.domain2[i] << 2410 ((i + 1) * 8); 2411 } 2412 break; 2413 2414 case SRAT_MEMORY: /* memory entry */ 2415 if (!(item->i.m.flags & SRAT_ENABLED)) { 2416 item = (struct srat_item *)((uintptr_t)item + 2417 item->len); 2418 continue; 2419 } 2420 domain = item->i.m.domain; 2421 break; 2422 2423 case SRAT_X2APIC: /* x2apic CPU entry */ 2424 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2425 item = (struct srat_item *)((uintptr_t)item + 2426 item->len); 2427 continue; 2428 } 2429 domain = item->i.xp.domain; 2430 break; 2431 2432 default: 2433 item = (struct srat_item *)((uintptr_t)item + 2434 item->len); 2435 continue; 2436 } 2437 2438 /* 2439 * Keep track of minimum proximity domain ID 2440 */ 2441 if (domain < domain_min) 2442 domain_min = domain; 2443 2444 item = (struct srat_item *)((uintptr_t)item + item->len); 2445 } 2446 if (lgrp_plat_domain_min_enable && prox_domain_min != NULL) 2447 *prox_domain_min = domain_min; 2448 2449 /* 2450 * Walk through SRAT, examining each CPU and memory entry to determine 2451 * proximity domain ID for each. 2452 */ 2453 domain_cnt = 0; 2454 item = tp->list; 2455 end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2456 bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); 2457 while (item < end) { 2458 uint32_t domain; 2459 boolean_t overflow; 2460 uint_t start; 2461 2462 switch (item->type) { 2463 case SRAT_PROCESSOR: /* CPU entry */ 2464 if (!(item->i.p.flags & SRAT_ENABLED)) { 2465 item = (struct srat_item *)((uintptr_t)item + 2466 item->len); 2467 continue; 2468 } 2469 domain = item->i.p.domain1; 2470 for (i = 0; i < 3; i++) { 2471 domain += item->i.p.domain2[i] << 2472 ((i + 1) * 8); 2473 } 2474 break; 2475 2476 case SRAT_MEMORY: /* memory entry */ 2477 if (!(item->i.m.flags & SRAT_ENABLED)) { 2478 item = (struct srat_item *)((uintptr_t)item + 2479 item->len); 2480 continue; 2481 } 2482 domain = item->i.m.domain; 2483 break; 2484 2485 case SRAT_X2APIC: /* x2apic CPU entry */ 2486 if (!(item->i.xp.flags & SRAT_ENABLED)) { 2487 item = (struct srat_item *)((uintptr_t)item + 2488 item->len); 2489 continue; 2490 } 2491 domain = item->i.xp.domain; 2492 break; 2493 2494 default: 2495 item = (struct srat_item *)((uintptr_t)item + 2496 item->len); 2497 continue; 2498 } 2499 2500 /* 2501 * Count and keep track of which proximity domain IDs seen 2502 */ 2503 start = i = domain % MAX_NODES; 2504 overflow = B_TRUE; 2505 do { 2506 /* 2507 * Create entry for proximity domain and increment 2508 * count when no entry exists where proximity domain 2509 * hashed 2510 */ 2511 if (!node_domain[i].exists) { 2512 node_domain[i].exists = 1; 2513 node_domain[i].prox_domain = domain; 2514 domain_cnt++; 2515 overflow = B_FALSE; 2516 break; 2517 } 2518 2519 /* 2520 * Nothing to do when proximity domain seen already 2521 * and its entry exists 2522 */ 2523 if (node_domain[i].prox_domain == domain) { 2524 overflow = B_FALSE; 2525 break; 2526 } 2527 2528 /* 2529 * Entry exists where proximity domain hashed, but for 2530 * different proximity domain so keep search for empty 2531 * slot to put it or matching entry whichever comes 2532 * first. 2533 */ 2534 i = (i + 1) % MAX_NODES; 2535 } while (i != start); 2536 2537 /* 2538 * Didn't find empty or matching entry which means have more 2539 * proximity domains than supported nodes (:-( 2540 */ 2541 ASSERT(overflow != B_TRUE); 2542 if (overflow == B_TRUE) 2543 return (-1); 2544 2545 item = (struct srat_item *)((uintptr_t)item + item->len); 2546 } 2547 return (domain_cnt); 2548 } 2549 2550 2551 /* 2552 * Set lgroup latencies for 2 level lgroup topology 2553 */ 2554 static void 2555 lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 2556 lgrp_plat_latency_stats_t *lat_stats) 2557 { 2558 int i; 2559 2560 ASSERT(node_memory != NULL && lat_stats != NULL); 2561 2562 if (lgrp_plat_node_cnt >= 4) 2563 cmn_err(CE_NOTE, 2564 "MPO only optimizing for local and remote\n"); 2565 for (i = 0; i < lgrp_plat_node_cnt; i++) { 2566 int j; 2567 2568 if (!node_memory[i].exists) 2569 continue; 2570 for (j = 0; j < lgrp_plat_node_cnt; j++) { 2571 if (!node_memory[j].exists) 2572 continue; 2573 if (i == j) 2574 lat_stats->latencies[i][j] = 2; 2575 else 2576 lat_stats->latencies[i][j] = 3; 2577 } 2578 } 2579 lat_stats->latency_min = 2; 2580 lat_stats->latency_max = 3; 2581 lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); 2582 } 2583 2584 2585 /* 2586 * The following Opteron specific constants, macros, types, and routines define 2587 * PCI configuration space registers and how to read them to determine the NUMA 2588 * configuration of *supported* Opteron processors. They provide the same 2589 * information that may be gotten from the ACPI System Resource Affinity Table 2590 * (SRAT) if it exists on the machine of interest. 2591 * 2592 * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family 2593 * of interest describes all of these registers and their contents. The main 2594 * registers used by this code to determine the NUMA configuration of the 2595 * machine are the node ID register for the number of NUMA nodes and the DRAM 2596 * address map registers for the physical address range of each node. 2597 * 2598 * NOTE: The format and how to determine the NUMA configuration using PCI 2599 * config space registers may change or may not be supported in future 2600 * Opteron processor families. 2601 */ 2602 2603 /* 2604 * How many bits to shift Opteron DRAM Address Map base and limit registers 2605 * to get actual value 2606 */ 2607 #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ 2608 #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ 2609 2610 #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ 2611 #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ 2612 2613 #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ 2614 2615 /* 2616 * Macros to derive addresses from Opteron DRAM Address Map registers 2617 */ 2618 #define OPT_DRAMADDR_HI(reg) \ 2619 (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ 2620 OPT_DRAMADDR_HI_LSHIFT_ADDR) 2621 2622 #define OPT_DRAMADDR_LO(reg) \ 2623 (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ 2624 OPT_DRAMADDR_LO_LSHIFT_ADDR) 2625 2626 #define OPT_DRAMADDR(high, low) \ 2627 (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) 2628 2629 /* 2630 * Bit masks defining what's in Opteron DRAM Address Map base register 2631 */ 2632 #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ 2633 #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ 2634 #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ 2635 2636 /* 2637 * Bit masks defining what's in Opteron DRAM Address Map limit register 2638 */ 2639 #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ 2640 #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ 2641 2642 2643 /* 2644 * Opteron Node ID register in PCI configuration space contains 2645 * number of nodes in system, etc. for Opteron K8. The following 2646 * constants and macros define its contents, structure, and access. 2647 */ 2648 2649 /* 2650 * Bit masks defining what's in Opteron Node ID register 2651 */ 2652 #define OPT_NODE_MASK_ID 0x7 /* node ID */ 2653 #define OPT_NODE_MASK_CNT 0x70 /* node count */ 2654 #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ 2655 #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ 2656 #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ 2657 2658 /* 2659 * How many bits in Opteron Node ID register to shift right to get actual value 2660 */ 2661 #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ 2662 2663 /* 2664 * Macros to get values from Opteron Node ID register 2665 */ 2666 #define OPT_NODE_CNT(reg) \ 2667 ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) 2668 2669 /* 2670 * Macro to setup PCI Extended Configuration Space (ECS) address to give to 2671 * "in/out" instructions 2672 * 2673 * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any 2674 * other uses should just do MMIO to access PCI ECS. 2675 * Must enable special bit in Northbridge Configuration Register on 2676 * Greyhound for extended CF8 space access to be able to access PCI ECS 2677 * using "in/out" instructions and restore special bit after done 2678 * accessing PCI ECS. 2679 */ 2680 #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ 2681 (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ 2682 (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ 2683 ((((reg) >> 8) & 0xf) << 24)) 2684 2685 /* 2686 * PCI configuration space registers accessed by specifying 2687 * a bus, device, function, and offset. The following constants 2688 * define the values needed to access Opteron K8 configuration 2689 * info to determine its node topology 2690 */ 2691 2692 #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ 2693 2694 /* 2695 * Opteron PCI configuration space register function values 2696 */ 2697 #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ 2698 #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ 2699 #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ 2700 #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ 2701 2702 /* 2703 * PCI Configuration Space register offsets 2704 */ 2705 #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ 2706 #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ 2707 #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ 2708 #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ 2709 2710 /* 2711 * Opteron PCI Configuration Space device IDs for nodes 2712 */ 2713 #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ 2714 2715 2716 /* 2717 * Opteron DRAM address map gives base and limit for physical memory in a node 2718 */ 2719 typedef struct opt_dram_addr_map { 2720 uint32_t base_hi; 2721 uint32_t base_lo; 2722 uint32_t limit_hi; 2723 uint32_t limit_lo; 2724 } opt_dram_addr_map_t; 2725 2726 2727 /* 2728 * Supported AMD processor families 2729 */ 2730 #define AMD_FAMILY_HAMMER 15 2731 #define AMD_FAMILY_GREYHOUND 16 2732 2733 /* 2734 * Whether to have is_opteron() return 1 even when processor isn't supported 2735 */ 2736 uint_t is_opteron_override = 0; 2737 2738 /* 2739 * AMD processor family for current CPU 2740 */ 2741 uint_t opt_family = 0; 2742 2743 2744 /* 2745 * Determine whether we're running on a supported AMD Opteron since reading 2746 * node count and DRAM address map registers may have different format or 2747 * may not be supported across processor families 2748 */ 2749 static int 2750 is_opteron(void) 2751 { 2752 2753 if (x86_vendor != X86_VENDOR_AMD) 2754 return (0); 2755 2756 opt_family = cpuid_getfamily(CPU); 2757 if (opt_family == AMD_FAMILY_HAMMER || 2758 opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) 2759 return (1); 2760 else 2761 return (0); 2762 } 2763 2764 2765 /* 2766 * Determine NUMA configuration for Opteron from registers that live in PCI 2767 * configuration space 2768 */ 2769 static void 2770 opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 2771 node_phys_addr_map_t *node_memory) 2772 { 2773 uint_t bus; 2774 uint_t dev; 2775 struct opt_dram_addr_map dram_map[MAX_NODES]; 2776 uint_t node; 2777 uint_t node_info[MAX_NODES]; 2778 uint_t off_hi; 2779 uint_t off_lo; 2780 uint64_t nb_cfg_reg; 2781 2782 /* 2783 * Read configuration registers from PCI configuration space to 2784 * determine node information, which memory is in each node, etc. 2785 * 2786 * Write to PCI configuration space address register to specify 2787 * which configuration register to read and read/write PCI 2788 * configuration space data register to get/set contents 2789 */ 2790 bus = OPT_PCS_BUS_CONFIG; 2791 dev = OPT_PCS_DEV_NODE0; 2792 off_hi = OPT_PCS_OFF_DRAMBASE_HI; 2793 off_lo = OPT_PCS_OFF_DRAMBASE_LO; 2794 2795 /* 2796 * Read node ID register for node 0 to get node count 2797 */ 2798 node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, 2799 OPT_PCS_OFF_NODEID); 2800 *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; 2801 2802 /* 2803 * If number of nodes is more than maximum supported, then set node 2804 * count to 1 and treat system as UMA instead of NUMA. 2805 */ 2806 if (*node_cnt > MAX_NODES) { 2807 *node_cnt = 1; 2808 return; 2809 } 2810 2811 /* 2812 * For Greyhound, PCI Extended Configuration Space must be enabled to 2813 * read high DRAM address map base and limit registers 2814 */ 2815 if (opt_family == AMD_FAMILY_GREYHOUND) { 2816 nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); 2817 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2818 wrmsr(MSR_AMD_NB_CFG, 2819 nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); 2820 } 2821 2822 for (node = 0; node < *node_cnt; node++) { 2823 uint32_t base_hi; 2824 uint32_t base_lo; 2825 uint32_t limit_hi; 2826 uint32_t limit_lo; 2827 2828 /* 2829 * Read node ID register (except for node 0 which we just read) 2830 */ 2831 if (node > 0) { 2832 node_info[node] = pci_getl_func(bus, dev, 2833 OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); 2834 } 2835 2836 /* 2837 * Read DRAM base and limit registers which specify 2838 * physical memory range of each node 2839 */ 2840 if (opt_family != AMD_FAMILY_GREYHOUND) 2841 base_hi = 0; 2842 else { 2843 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2844 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2845 base_hi = dram_map[node].base_hi = 2846 inl(PCI_CONFDATA); 2847 } 2848 base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, 2849 OPT_PCS_FUNC_ADDRMAP, off_lo); 2850 2851 if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && 2852 mem_intrlv) 2853 *mem_intrlv = *mem_intrlv + 1; 2854 2855 off_hi += 4; /* high limit register offset */ 2856 if (opt_family != AMD_FAMILY_GREYHOUND) 2857 limit_hi = 0; 2858 else { 2859 outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2860 OPT_PCS_FUNC_ADDRMAP, off_hi)); 2861 limit_hi = dram_map[node].limit_hi = 2862 inl(PCI_CONFDATA); 2863 } 2864 2865 off_lo += 4; /* low limit register offset */ 2866 limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, 2867 dev, OPT_PCS_FUNC_ADDRMAP, off_lo); 2868 2869 /* 2870 * Increment device number to next node and register offsets 2871 * for DRAM base register of next node 2872 */ 2873 off_hi += 4; 2874 off_lo += 4; 2875 dev++; 2876 2877 /* 2878 * Both read and write enable bits must be enabled in DRAM 2879 * address map base register for physical memory to exist in 2880 * node 2881 */ 2882 if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || 2883 (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { 2884 /* 2885 * Mark node memory as non-existent and set start and 2886 * end addresses to be same in node_memory[] 2887 */ 2888 node_memory[node].exists = 0; 2889 node_memory[node].start = node_memory[node].end = 2890 (pfn_t)-1; 2891 continue; 2892 } 2893 2894 /* 2895 * Mark node memory as existing and remember physical address 2896 * range of each node for use later 2897 */ 2898 node_memory[node].exists = 1; 2899 2900 node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); 2901 2902 node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | 2903 OPT_DRAMADDR_LO_MASK_OFF); 2904 } 2905 2906 /* 2907 * Restore PCI Extended Configuration Space enable bit 2908 */ 2909 if (opt_family == AMD_FAMILY_GREYHOUND) { 2910 if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2911 wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); 2912 } 2913 } 2914 2915 2916 /* 2917 * Return average amount of time to read vendor ID register on Northbridge 2918 * N times on specified destination node from current CPU 2919 */ 2920 static hrtime_t 2921 opt_probe_vendor(int dest_node, int nreads) 2922 { 2923 int cnt; 2924 uint_t dev; 2925 /* LINTED: set but not used in function */ 2926 volatile uint_t dev_vendor; 2927 hrtime_t elapsed; 2928 hrtime_t end; 2929 int ipl; 2930 hrtime_t start; 2931 2932 dev = OPT_PCS_DEV_NODE0 + dest_node; 2933 kpreempt_disable(); 2934 ipl = spl8(); 2935 outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, 2936 OPT_PCS_OFF_VENDOR)); 2937 start = gethrtime(); 2938 for (cnt = 0; cnt < nreads; cnt++) 2939 dev_vendor = inl(PCI_CONFDATA); 2940 end = gethrtime(); 2941 elapsed = (end - start) / nreads; 2942 splx(ipl); 2943 kpreempt_enable(); 2944 return (elapsed); 2945 } 2946