1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * CPU Module Interface - hardware abstraction. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/cpu_module.h> 35 #include <sys/kmem.h> 36 #include <sys/x86_archext.h> 37 #include <sys/cpuvar.h> 38 #include <sys/ksynch.h> 39 #include <sys/x_call.h> 40 #include <sys/pghw.h> 41 #include <sys/pci_cfgspace.h> 42 #include <sys/archsystm.h> 43 #include <sys/ontrap.h> 44 #include <sys/controlregs.h> 45 #include <sys/sunddi.h> 46 47 /* 48 * Outside of this file consumers use the opaque cmi_hdl_t. This 49 * definition is duplicated in the generic_cpu mdb module, so keep 50 * them in-sync when making changes. 51 */ 52 typedef struct cmi_hdl_impl { 53 enum cmi_hdl_class cmih_class; /* Handle nature */ 54 struct cmi_hdl_ops *cmih_ops; /* Operations vector */ 55 uint_t cmih_chipid; /* Chipid of cpu resource */ 56 uint_t cmih_coreid; /* Core within die */ 57 uint_t cmih_strandid; /* Thread within core */ 58 volatile uint32_t *cmih_refcntp; /* Reference count pointer */ 59 uint64_t cmih_msrsrc; /* MSR data source flags */ 60 void *cmih_hdlpriv; /* cmi_hw.c private data */ 61 void *cmih_spec; /* cmi_hdl_{set,get}_specific */ 62 void *cmih_cmi; /* cpu mod control structure */ 63 void *cmih_cmidata; /* cpu mod private data */ 64 const struct cmi_mc_ops *cmih_mcops; /* Memory-controller ops */ 65 void *cmih_mcdata; /* Memory-controller data */ 66 } cmi_hdl_impl_t; 67 68 #define IMPLHDL(ophdl) ((cmi_hdl_impl_t *)ophdl) 69 70 /* 71 * Handles are looked up from contexts such as polling, injection etc 72 * where the context is reasonably well defined (although a poller could 73 * interrupt any old thread holding any old lock). They are also looked 74 * up by machine check handlers, which may strike at inconvenient times 75 * such as during handle initialization or destruction or during handle 76 * lookup (which the #MC handler itself will also have to perform). 77 * 78 * So keeping handles in a linked list makes locking difficult when we 79 * consider #MC handlers. Our solution is to have an array indexed 80 * by that which uniquely identifies a handle - chip/core/strand id - 81 * with each array member a structure including a pointer to a handle 82 * structure for the resource, and a reference count for the handle. 83 * Reference counts are modified atomically. The public cmi_hdl_hold 84 * always succeeds because this can only be used after handle creation 85 * and before the call to destruct, so the hold count it already at least one. 86 * In other functions that lookup a handle (cmi_hdl_lookup, cmi_hdl_any) 87 * we must be certain that the count has not already decrmented to zero 88 * before applying our hold. 89 * 90 * This array is allocated when first we want to populate an entry. 91 * When allocated it is maximal - ideally we should scale to the 92 * actual number of chips, cores per chip and strand per core but 93 * that info is not readily available if we are virtualized so 94 * for now we stick with the dumb approach. 95 */ 96 #define CMI_MAX_CHIPS_NBITS 4 /* 16 chips packages max */ 97 #define CMI_MAX_CORES_PER_CHIP_NBITS 3 /* 8 cores per chip max */ 98 #define CMI_MAX_STRANDS_PER_CORE_NBITS 1 /* 2 strands per core max */ 99 100 #define CMI_MAX_CHIPS (1 << CMI_MAX_CHIPS_NBITS) 101 #define CMI_MAX_CORES_PER_CHIP (1 << CMI_MAX_CORES_PER_CHIP_NBITS) 102 #define CMI_MAX_STRANDS_PER_CORE (1 << CMI_MAX_STRANDS_PER_CORE_NBITS) 103 104 /* 105 * Handle array indexing. 106 * [7:4] = Chip package. 107 * [3:1] = Core in package, 108 * [0:0] = Strand in core, 109 */ 110 #define CMI_HDL_ARR_IDX_CHIP(chipid) \ 111 (((chipid) & (CMI_MAX_CHIPS - 1)) << \ 112 (CMI_MAX_STRANDS_PER_CORE_NBITS + CMI_MAX_CORES_PER_CHIP_NBITS)) 113 114 #define CMI_HDL_ARR_IDX_CORE(coreid) \ 115 (((coreid) & (CMI_MAX_CORES_PER_CHIP - 1)) << \ 116 CMI_MAX_STRANDS_PER_CORE_NBITS) 117 118 #define CMI_HDL_ARR_IDX_STRAND(strandid) \ 119 (((strandid) & (CMI_MAX_STRANDS_PER_CORE - 1))) 120 121 #define CMI_HDL_ARR_IDX(chipid, coreid, strandid) \ 122 (CMI_HDL_ARR_IDX_CHIP(chipid) | CMI_HDL_ARR_IDX_CORE(coreid) | \ 123 CMI_HDL_ARR_IDX_STRAND(strandid)) 124 125 #define CMI_HDL_ARR_SZ (CMI_MAX_CHIPS * CMI_MAX_CORES_PER_CHIP * \ 126 CMI_MAX_STRANDS_PER_CORE) 127 128 struct cmi_hdl_arr_ent { 129 volatile uint32_t cmae_refcnt; 130 cmi_hdl_impl_t *cmae_hdlp; 131 }; 132 133 static struct cmi_hdl_arr_ent *cmi_hdl_arr; 134 135 /* 136 * Controls where we will source PCI config space data. 137 */ 138 #define CMI_PCICFG_FLAG_RD_HWOK 0x0001 139 #define CMI_PCICFG_FLAG_RD_INTERPOSEOK 0X0002 140 #define CMI_PCICFG_FLAG_WR_HWOK 0x0004 141 #define CMI_PCICFG_FLAG_WR_INTERPOSEOK 0X0008 142 143 static uint64_t cmi_pcicfg_flags = 144 CMI_PCICFG_FLAG_RD_HWOK | CMI_PCICFG_FLAG_RD_INTERPOSEOK | 145 CMI_PCICFG_FLAG_WR_HWOK | CMI_PCICFG_FLAG_WR_INTERPOSEOK; 146 147 /* 148 * The flags for individual cpus are kept in their per-cpu handle cmih_msrsrc 149 */ 150 #define CMI_MSR_FLAG_RD_HWOK 0x0001 151 #define CMI_MSR_FLAG_RD_INTERPOSEOK 0x0002 152 #define CMI_MSR_FLAG_WR_HWOK 0x0004 153 #define CMI_MSR_FLAG_WR_INTERPOSEOK 0x0008 154 155 int cmi_call_func_ntv_tries = 3; 156 157 static cmi_errno_t 158 call_func_ntv(int cpuid, xc_func_t func, xc_arg_t arg1, xc_arg_t arg2) 159 { 160 cmi_errno_t rc = -1; 161 int i; 162 163 kpreempt_disable(); 164 165 if (CPU->cpu_id == cpuid) { 166 (*func)(arg1, arg2, (xc_arg_t)&rc); 167 } else { 168 /* 169 * This should not happen for a #MC trap or a poll, so 170 * this is likely an error injection or similar. 171 * We will try to cross call with xc_trycall - we 172 * can't guarantee success with xc_call because 173 * the interrupt code in the case of a #MC may 174 * already hold the xc mutex. 175 */ 176 for (i = 0; i < cmi_call_func_ntv_tries; i++) { 177 cpuset_t cpus; 178 179 CPUSET_ONLY(cpus, cpuid); 180 xc_trycall(arg1, arg2, (xc_arg_t)&rc, cpus, func); 181 if (rc != -1) 182 break; 183 184 DELAY(1); 185 } 186 } 187 188 kpreempt_enable(); 189 190 return (rc != -1 ? rc : CMIERR_DEADLOCK); 191 } 192 193 /* 194 * ======================================================= 195 * | MSR Interposition | 196 * | ----------------- | 197 * | | 198 * ------------------------------------------------------- 199 */ 200 201 #define CMI_MSRI_HASHSZ 16 202 #define CMI_MSRI_HASHIDX(hdl, msr) \ 203 (((uintptr_t)(hdl) >> 3 + (msr)) % (CMI_MSRI_HASHSZ - 1)) 204 205 struct cmi_msri_bkt { 206 kmutex_t msrib_lock; 207 struct cmi_msri_hashent *msrib_head; 208 }; 209 210 struct cmi_msri_hashent { 211 struct cmi_msri_hashent *msrie_next; 212 struct cmi_msri_hashent *msrie_prev; 213 cmi_hdl_impl_t *msrie_hdl; 214 uint_t msrie_msrnum; 215 uint64_t msrie_msrval; 216 }; 217 218 #define CMI_MSRI_MATCH(ent, hdl, req_msr) \ 219 ((ent)->msrie_hdl == (hdl) && (ent)->msrie_msrnum == (req_msr)) 220 221 static struct cmi_msri_bkt msrihash[CMI_MSRI_HASHSZ]; 222 223 static void 224 msri_addent(cmi_hdl_impl_t *hdl, cmi_mca_regs_t *regp) 225 { 226 int idx = CMI_MSRI_HASHIDX(hdl, regp->cmr_msrnum); 227 struct cmi_msri_bkt *hbp = &msrihash[idx]; 228 struct cmi_msri_hashent *hep; 229 230 mutex_enter(&hbp->msrib_lock); 231 232 for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { 233 if (CMI_MSRI_MATCH(hep, hdl, regp->cmr_msrnum)) 234 break; 235 } 236 237 if (hep != NULL) { 238 hep->msrie_msrval = regp->cmr_msrval; 239 } else { 240 hep = kmem_alloc(sizeof (*hep), KM_SLEEP); 241 hep->msrie_hdl = hdl; 242 hep->msrie_msrnum = regp->cmr_msrnum; 243 hep->msrie_msrval = regp->cmr_msrval; 244 245 if (hbp->msrib_head != NULL) 246 hbp->msrib_head->msrie_prev = hep; 247 hep->msrie_next = hbp->msrib_head; 248 hep->msrie_prev = NULL; 249 hbp->msrib_head = hep; 250 } 251 252 mutex_exit(&hbp->msrib_lock); 253 } 254 255 /* 256 * Look for a match for the given hanlde and msr. Return 1 with valp 257 * filled if a match is found, otherwise return 0 with valp untouched. 258 */ 259 static int 260 msri_lookup(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t *valp) 261 { 262 int idx = CMI_MSRI_HASHIDX(hdl, msr); 263 struct cmi_msri_bkt *hbp = &msrihash[idx]; 264 struct cmi_msri_hashent *hep; 265 266 /* 267 * This function is called during #MC trap handling, so we should 268 * consider the possibility that the hash mutex is held by the 269 * interrupted thread. This should not happen because interposition 270 * is an artificial injection mechanism and the #MC is requested 271 * after adding entries, but just in case of a real #MC at an 272 * unlucky moment we'll use mutex_tryenter here. 273 */ 274 if (!mutex_tryenter(&hbp->msrib_lock)) 275 return (0); 276 277 for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { 278 if (CMI_MSRI_MATCH(hep, hdl, msr)) { 279 *valp = hep->msrie_msrval; 280 break; 281 } 282 } 283 284 mutex_exit(&hbp->msrib_lock); 285 286 return (hep != NULL); 287 } 288 289 /* 290 * Remove any interposed value that matches. 291 */ 292 static void 293 msri_rment(cmi_hdl_impl_t *hdl, uint_t msr) 294 { 295 296 int idx = CMI_MSRI_HASHIDX(hdl, msr); 297 struct cmi_msri_bkt *hbp = &msrihash[idx]; 298 struct cmi_msri_hashent *hep; 299 300 if (!mutex_tryenter(&hbp->msrib_lock)) 301 return; 302 303 for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { 304 if (CMI_MSRI_MATCH(hep, hdl, msr)) { 305 if (hep->msrie_prev != NULL) 306 hep->msrie_prev->msrie_next = hep->msrie_next; 307 308 if (hep->msrie_next != NULL) 309 hep->msrie_next->msrie_prev = hep->msrie_prev; 310 311 if (hbp->msrib_head == hep) 312 hbp->msrib_head = hep->msrie_next; 313 314 kmem_free(hep, sizeof (*hep)); 315 break; 316 } 317 } 318 319 mutex_exit(&hbp->msrib_lock); 320 } 321 322 /* 323 * ======================================================= 324 * | PCI Config Space Interposition | 325 * | ------------------------------ | 326 * | | 327 * ------------------------------------------------------- 328 */ 329 330 /* 331 * Hash for interposed PCI config space values. We lookup on bus/dev/fun/offset 332 * and then record whether the value stashed was made with a byte, word or 333 * doubleword access; we will only return a hit for an access of the 334 * same size. If you access say a 32-bit register using byte accesses 335 * and then attempt to read the full 32-bit value back you will not obtain 336 * any sort of merged result - you get a lookup miss. 337 */ 338 339 #define CMI_PCII_HASHSZ 16 340 #define CMI_PCII_HASHIDX(b, d, f, o) \ 341 (((b) + (d) + (f) + (o)) % (CMI_PCII_HASHSZ - 1)) 342 343 struct cmi_pcii_bkt { 344 kmutex_t pciib_lock; 345 struct cmi_pcii_hashent *pciib_head; 346 }; 347 348 struct cmi_pcii_hashent { 349 struct cmi_pcii_hashent *pcii_next; 350 struct cmi_pcii_hashent *pcii_prev; 351 int pcii_bus; 352 int pcii_dev; 353 int pcii_func; 354 int pcii_reg; 355 int pcii_asize; 356 uint32_t pcii_val; 357 }; 358 359 #define CMI_PCII_MATCH(ent, b, d, f, r, asz) \ 360 ((ent)->pcii_bus == (b) && (ent)->pcii_dev == (d) && \ 361 (ent)->pcii_func == (f) && (ent)->pcii_reg == (r) && \ 362 (ent)->pcii_asize == (asz)) 363 364 static struct cmi_pcii_bkt pciihash[CMI_PCII_HASHSZ]; 365 366 367 /* 368 * Add a new entry to the PCI interpose hash, overwriting any existing 369 * entry that is found. 370 */ 371 static void 372 pcii_addent(int bus, int dev, int func, int reg, uint32_t val, int asz) 373 { 374 int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); 375 struct cmi_pcii_bkt *hbp = &pciihash[idx]; 376 struct cmi_pcii_hashent *hep; 377 378 mutex_enter(&hbp->pciib_lock); 379 380 for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { 381 if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) 382 break; 383 } 384 385 if (hep != NULL) { 386 hep->pcii_val = val; 387 } else { 388 hep = kmem_alloc(sizeof (*hep), KM_SLEEP); 389 hep->pcii_bus = bus; 390 hep->pcii_dev = dev; 391 hep->pcii_func = func; 392 hep->pcii_reg = reg; 393 hep->pcii_asize = asz; 394 hep->pcii_val = val; 395 396 if (hbp->pciib_head != NULL) 397 hbp->pciib_head->pcii_prev = hep; 398 hep->pcii_next = hbp->pciib_head; 399 hep->pcii_prev = NULL; 400 hbp->pciib_head = hep; 401 } 402 403 mutex_exit(&hbp->pciib_lock); 404 } 405 406 /* 407 * Look for a match for the given bus/dev/func/reg; return 1 with valp 408 * filled if a match is found, otherwise return 0 with valp untouched. 409 */ 410 static int 411 pcii_lookup(int bus, int dev, int func, int reg, int asz, uint32_t *valp) 412 { 413 int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); 414 struct cmi_pcii_bkt *hbp = &pciihash[idx]; 415 struct cmi_pcii_hashent *hep; 416 417 if (!mutex_tryenter(&hbp->pciib_lock)) 418 return (0); 419 420 for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { 421 if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) { 422 *valp = hep->pcii_val; 423 break; 424 } 425 } 426 427 mutex_exit(&hbp->pciib_lock); 428 429 return (hep != NULL); 430 } 431 432 static void 433 pcii_rment(int bus, int dev, int func, int reg, int asz) 434 { 435 int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); 436 struct cmi_pcii_bkt *hbp = &pciihash[idx]; 437 struct cmi_pcii_hashent *hep; 438 439 mutex_enter(&hbp->pciib_lock); 440 441 for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { 442 if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) { 443 if (hep->pcii_prev != NULL) 444 hep->pcii_prev->pcii_next = hep->pcii_next; 445 446 if (hep->pcii_next != NULL) 447 hep->pcii_next->pcii_prev = hep->pcii_prev; 448 449 if (hbp->pciib_head == hep) 450 hbp->pciib_head = hep->pcii_next; 451 452 kmem_free(hep, sizeof (*hep)); 453 break; 454 } 455 } 456 457 mutex_exit(&hbp->pciib_lock); 458 } 459 460 /* 461 * ======================================================= 462 * | Native methods | 463 * | -------------- | 464 * | | 465 * | These are used when we are running native on bare- | 466 * | metal, or simply don't know any better. | 467 * --------------------------------------------------------- 468 */ 469 470 static uint_t 471 ntv_vendor(cmi_hdl_impl_t *hdl) 472 { 473 return (cpuid_getvendor((cpu_t *)hdl->cmih_hdlpriv)); 474 } 475 476 static const char * 477 ntv_vendorstr(cmi_hdl_impl_t *hdl) 478 { 479 return (cpuid_getvendorstr((cpu_t *)hdl->cmih_hdlpriv)); 480 } 481 482 static uint_t 483 ntv_family(cmi_hdl_impl_t *hdl) 484 { 485 return (cpuid_getfamily((cpu_t *)hdl->cmih_hdlpriv)); 486 } 487 488 static uint_t 489 ntv_model(cmi_hdl_impl_t *hdl) 490 { 491 return (cpuid_getmodel((cpu_t *)hdl->cmih_hdlpriv)); 492 } 493 494 static uint_t 495 ntv_stepping(cmi_hdl_impl_t *hdl) 496 { 497 return (cpuid_getstep((cpu_t *)hdl->cmih_hdlpriv)); 498 } 499 500 static uint_t 501 ntv_chipid(cmi_hdl_impl_t *hdl) 502 { 503 return (hdl->cmih_chipid); 504 505 } 506 507 static uint_t 508 ntv_coreid(cmi_hdl_impl_t *hdl) 509 { 510 return (hdl->cmih_coreid); 511 } 512 513 static uint_t 514 ntv_strandid(cmi_hdl_impl_t *hdl) 515 { 516 return (hdl->cmih_strandid); 517 } 518 519 static uint32_t 520 ntv_chiprev(cmi_hdl_impl_t *hdl) 521 { 522 return (cpuid_getchiprev((cpu_t *)hdl->cmih_hdlpriv)); 523 } 524 525 static const char * 526 ntv_chiprevstr(cmi_hdl_impl_t *hdl) 527 { 528 return (cpuid_getchiprevstr((cpu_t *)hdl->cmih_hdlpriv)); 529 } 530 531 static uint32_t 532 ntv_getsockettype(cmi_hdl_impl_t *hdl) 533 { 534 return (cpuid_getsockettype((cpu_t *)hdl->cmih_hdlpriv)); 535 } 536 537 /*ARGSUSED*/ 538 static int 539 ntv_getcr4_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 540 { 541 ulong_t *dest = (ulong_t *)arg1; 542 cmi_errno_t *rcp = (cmi_errno_t *)arg3; 543 544 *dest = getcr4(); 545 *rcp = CMI_SUCCESS; 546 547 return (0); 548 } 549 550 static ulong_t 551 ntv_getcr4(cmi_hdl_impl_t *hdl) 552 { 553 cpu_t *cp = (cpu_t *)hdl->cmih_hdlpriv; 554 ulong_t val; 555 556 (void) call_func_ntv(cp->cpu_id, ntv_getcr4_xc, (xc_arg_t)&val, NULL); 557 558 return (val); 559 } 560 561 /*ARGSUSED*/ 562 static int 563 ntv_setcr4_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 564 { 565 ulong_t val = (ulong_t)arg1; 566 cmi_errno_t *rcp = (cmi_errno_t *)arg3; 567 568 setcr4(val); 569 *rcp = CMI_SUCCESS; 570 571 return (0); 572 } 573 574 static void 575 ntv_setcr4(cmi_hdl_impl_t *hdl, ulong_t val) 576 { 577 cpu_t *cp = (cpu_t *)hdl->cmih_hdlpriv; 578 579 (void) call_func_ntv(cp->cpu_id, ntv_setcr4_xc, (xc_arg_t)val, NULL); 580 } 581 582 volatile uint32_t cmi_trapped_rdmsr; 583 584 /*ARGSUSED*/ 585 static int 586 ntv_rdmsr_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 587 { 588 uint_t msr = (uint_t)arg1; 589 uint64_t *valp = (uint64_t *)arg2; 590 cmi_errno_t *rcp = (cmi_errno_t *)arg3; 591 592 on_trap_data_t otd; 593 594 if (on_trap(&otd, OT_DATA_ACCESS) == 0) { 595 if (checked_rdmsr(msr, valp) == 0) 596 *rcp = CMI_SUCCESS; 597 else 598 *rcp = CMIERR_NOTSUP; 599 } else { 600 *rcp = CMIERR_MSRGPF; 601 atomic_inc_32(&cmi_trapped_rdmsr); 602 } 603 no_trap(); 604 605 return (0); 606 } 607 608 static cmi_errno_t 609 ntv_rdmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t *valp) 610 { 611 cpu_t *cp = (cpu_t *)hdl->cmih_hdlpriv; 612 613 return (call_func_ntv(cp->cpu_id, ntv_rdmsr_xc, 614 (xc_arg_t)msr, (xc_arg_t)valp)); 615 } 616 617 volatile uint32_t cmi_trapped_wrmsr; 618 619 /*ARGSUSED*/ 620 static int 621 ntv_wrmsr_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 622 { 623 uint_t msr = (uint_t)arg1; 624 uint64_t val = *((uint64_t *)arg2); 625 cmi_errno_t *rcp = (cmi_errno_t *)arg3; 626 on_trap_data_t otd; 627 628 if (on_trap(&otd, OT_DATA_ACCESS) == 0) { 629 if (checked_wrmsr(msr, val) == 0) 630 *rcp = CMI_SUCCESS; 631 else 632 *rcp = CMIERR_NOTSUP; 633 } else { 634 *rcp = CMIERR_MSRGPF; 635 atomic_inc_32(&cmi_trapped_wrmsr); 636 } 637 no_trap(); 638 639 return (0); 640 641 } 642 643 static cmi_errno_t 644 ntv_wrmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) 645 { 646 cpu_t *cp = (cpu_t *)hdl->cmih_hdlpriv; 647 648 return (call_func_ntv(cp->cpu_id, ntv_wrmsr_xc, 649 (xc_arg_t)msr, (xc_arg_t)&val)); 650 } 651 652 /*ARGSUSED*/ 653 static int 654 ntv_mcheck_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 655 { 656 cmi_errno_t *rcp = (cmi_errno_t *)arg3; 657 658 int18(); 659 *rcp = CMI_SUCCESS; 660 661 return (0); 662 } 663 664 static void 665 ntv_mcheck(cmi_hdl_impl_t *hdl) 666 { 667 cpu_t *cp = (cpu_t *)hdl->cmih_hdlpriv; 668 669 (void) call_func_ntv(cp->cpu_id, ntv_mcheck_xc, NULL, NULL); 670 } 671 672 /* 673 * Ops structure for handle operations. 674 */ 675 struct cmi_hdl_ops { 676 uint_t (*cmio_vendor)(cmi_hdl_impl_t *); 677 const char *(*cmio_vendorstr)(cmi_hdl_impl_t *); 678 uint_t (*cmio_family)(cmi_hdl_impl_t *); 679 uint_t (*cmio_model)(cmi_hdl_impl_t *); 680 uint_t (*cmio_stepping)(cmi_hdl_impl_t *); 681 uint_t (*cmio_chipid)(cmi_hdl_impl_t *); 682 uint_t (*cmio_coreid)(cmi_hdl_impl_t *); 683 uint_t (*cmio_strandid)(cmi_hdl_impl_t *); 684 uint32_t (*cmio_chiprev)(cmi_hdl_impl_t *); 685 const char *(*cmio_chiprevstr)(cmi_hdl_impl_t *); 686 uint32_t (*cmio_getsockettype)(cmi_hdl_impl_t *); 687 ulong_t (*cmio_getcr4)(cmi_hdl_impl_t *); 688 void (*cmio_setcr4)(cmi_hdl_impl_t *, ulong_t); 689 cmi_errno_t (*cmio_rdmsr)(cmi_hdl_impl_t *, uint_t, uint64_t *); 690 cmi_errno_t (*cmio_wrmsr)(cmi_hdl_impl_t *, uint_t, uint64_t); 691 void (*cmio_mcheck)(cmi_hdl_impl_t *); 692 } cmi_hdl_ops[] = { 693 /* 694 * CMI_HDL_NATIVE - ops when apparently running on bare-metal 695 */ 696 { 697 ntv_vendor, 698 ntv_vendorstr, 699 ntv_family, 700 ntv_model, 701 ntv_stepping, 702 ntv_chipid, 703 ntv_coreid, 704 ntv_strandid, 705 ntv_chiprev, 706 ntv_chiprevstr, 707 ntv_getsockettype, 708 ntv_getcr4, 709 ntv_setcr4, 710 ntv_rdmsr, 711 ntv_wrmsr, 712 ntv_mcheck 713 }, 714 }; 715 716 #ifndef __xpv 717 static void * 718 cpu_search(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, 719 uint_t strandid) 720 { 721 switch (class) { 722 case CMI_HDL_NATIVE: { 723 cpu_t *cp, *startcp; 724 725 kpreempt_disable(); 726 cp = startcp = CPU; 727 do { 728 if (cmi_ntv_hwchipid(cp) == chipid && 729 cmi_ntv_hwcoreid(cp) == coreid && 730 cmi_ntv_hwstrandid(cp) == strandid) { 731 kpreempt_enable(); 732 return ((void *)cp); 733 } 734 735 cp = cp->cpu_next; 736 } while (cp != startcp); 737 kpreempt_enable(); 738 return (NULL); 739 } 740 741 default: 742 return (NULL); 743 } 744 } 745 #endif 746 747 cmi_hdl_t 748 cmi_hdl_create(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, 749 uint_t strandid) 750 { 751 cmi_hdl_impl_t *hdl; 752 void *priv = NULL; 753 int idx; 754 755 if (chipid > CMI_MAX_CHIPS - 1 || coreid > CMI_MAX_CORES_PER_CHIP - 1 || 756 strandid > CMI_MAX_STRANDS_PER_CORE - 1) 757 return (NULL); 758 759 #ifndef __xpv 760 if ((priv = cpu_search(class, chipid, coreid, strandid)) == NULL) 761 return (NULL); 762 #endif 763 764 hdl = kmem_zalloc(sizeof (*hdl), KM_SLEEP); 765 766 hdl->cmih_class = class; 767 hdl->cmih_ops = &cmi_hdl_ops[class]; 768 hdl->cmih_chipid = chipid; 769 hdl->cmih_coreid = coreid; 770 hdl->cmih_strandid = strandid; 771 hdl->cmih_hdlpriv = priv; 772 hdl->cmih_msrsrc = CMI_MSR_FLAG_RD_HWOK | CMI_MSR_FLAG_RD_INTERPOSEOK | 773 CMI_MSR_FLAG_WR_HWOK | CMI_MSR_FLAG_WR_INTERPOSEOK; 774 775 if (cmi_hdl_arr == NULL) { 776 size_t sz = CMI_HDL_ARR_SZ * sizeof (struct cmi_hdl_arr_ent); 777 void *arr = kmem_zalloc(sz, KM_SLEEP); 778 779 if (atomic_cas_ptr(&cmi_hdl_arr, NULL, arr) != NULL) 780 kmem_free(arr, sz); /* someone beat us */ 781 } 782 783 idx = CMI_HDL_ARR_IDX(chipid, coreid, strandid); 784 if (cmi_hdl_arr[idx].cmae_refcnt != 0 || 785 cmi_hdl_arr[idx].cmae_hdlp != NULL) { 786 /* 787 * Somehow this (chipid, coreid, strandid) id tuple has 788 * already been assigned! This indicates that the 789 * callers logic in determining these values is busted, 790 * or perhaps undermined by bad BIOS setup. Complain, 791 * and refuse to initialize this tuple again as bad things 792 * will happen. 793 */ 794 cmn_err(CE_NOTE, "cmi_hdl_create: chipid %d coreid %d " 795 "strandid %d handle already allocated!", 796 chipid, coreid, strandid); 797 kmem_free(hdl, sizeof (*hdl)); 798 return (NULL); 799 } 800 801 /* 802 * Once we store a nonzero reference count others can find this 803 * handle via cmi_hdl_lookup etc. This initial hold on the handle 804 * is to be dropped only if some other part of cmi initialization 805 * fails or, if it succeeds, at later cpu deconfigure. Note the 806 * the module private data we hold in cmih_cmi and cmih_cmidata 807 * is still NULL at this point (the caller will fill it with 808 * cmi_hdl_setcmi if it initializes) so consumers of handles 809 * should always be ready for that possibility. 810 */ 811 cmi_hdl_arr[idx].cmae_hdlp = hdl; 812 hdl->cmih_refcntp = &cmi_hdl_arr[idx].cmae_refcnt; 813 cmi_hdl_arr[idx].cmae_refcnt = 1; 814 815 return ((cmi_hdl_t)hdl); 816 } 817 818 void 819 cmi_hdl_hold(cmi_hdl_t ophdl) 820 { 821 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 822 823 ASSERT(*hdl->cmih_refcntp != 0); /* must not be the initial hold */ 824 825 atomic_inc_32(hdl->cmih_refcntp); 826 } 827 828 static int 829 cmi_hdl_canref(int arridx) 830 { 831 volatile uint32_t *refcntp; 832 uint32_t refcnt; 833 834 if (cmi_hdl_arr == NULL) 835 return (0); 836 837 refcntp = &cmi_hdl_arr[arridx].cmae_refcnt; 838 refcnt = *refcntp; 839 840 if (refcnt == 0) { 841 /* 842 * Associated object never existed, is being destroyed, 843 * or has been destroyed. 844 */ 845 return (0); 846 } 847 848 /* 849 * We cannot use atomic increment here because once the reference 850 * count reaches zero it must never be bumped up again. 851 */ 852 while (refcnt != 0) { 853 if (atomic_cas_32(refcntp, refcnt, refcnt + 1) == refcnt) 854 return (1); 855 refcnt = *refcntp; 856 } 857 858 /* 859 * Somebody dropped the reference count to 0 after our initial 860 * check. 861 */ 862 return (0); 863 } 864 865 866 void 867 cmi_hdl_rele(cmi_hdl_t ophdl) 868 { 869 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 870 int idx; 871 872 ASSERT(*hdl->cmih_refcntp > 0); 873 874 if (atomic_dec_32_nv(hdl->cmih_refcntp) > 0) 875 return; 876 877 idx = CMI_HDL_ARR_IDX(hdl->cmih_chipid, hdl->cmih_coreid, 878 hdl->cmih_strandid); 879 cmi_hdl_arr[idx].cmae_hdlp = NULL; 880 881 kmem_free(hdl, sizeof (*hdl)); 882 } 883 884 void 885 cmi_hdl_setspecific(cmi_hdl_t ophdl, void *arg) 886 { 887 IMPLHDL(ophdl)->cmih_spec = arg; 888 } 889 890 void * 891 cmi_hdl_getspecific(cmi_hdl_t ophdl) 892 { 893 return (IMPLHDL(ophdl)->cmih_spec); 894 } 895 896 void 897 cmi_hdl_setmc(cmi_hdl_t ophdl, const struct cmi_mc_ops *mcops, void *mcdata) 898 { 899 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 900 901 ASSERT(hdl->cmih_mcops == NULL && hdl->cmih_mcdata == NULL); 902 hdl->cmih_mcops = mcops; 903 hdl->cmih_mcdata = mcdata; 904 } 905 906 const struct cmi_mc_ops * 907 cmi_hdl_getmcops(cmi_hdl_t ophdl) 908 { 909 return (IMPLHDL(ophdl)->cmih_mcops); 910 } 911 912 void * 913 cmi_hdl_getmcdata(cmi_hdl_t ophdl) 914 { 915 return (IMPLHDL(ophdl)->cmih_mcdata); 916 } 917 918 cmi_hdl_t 919 cmi_hdl_lookup(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, 920 uint_t strandid) 921 { 922 int idx; 923 924 if (chipid > CMI_MAX_CHIPS - 1 || coreid > CMI_MAX_CORES_PER_CHIP - 1 || 925 strandid > CMI_MAX_STRANDS_PER_CORE - 1) 926 return (NULL); 927 928 idx = CMI_HDL_ARR_IDX(chipid, coreid, strandid); 929 930 if (!cmi_hdl_canref(idx)) 931 return (NULL); 932 933 if (cmi_hdl_arr[idx].cmae_hdlp->cmih_class != class) { 934 cmi_hdl_rele((cmi_hdl_t)cmi_hdl_arr[idx].cmae_hdlp); 935 return (NULL); 936 } 937 938 return ((cmi_hdl_t)cmi_hdl_arr[idx].cmae_hdlp); 939 } 940 941 cmi_hdl_t 942 cmi_hdl_any(void) 943 { 944 int i; 945 946 for (i = 0; i < CMI_HDL_ARR_SZ; i++) { 947 if (cmi_hdl_canref(i)) 948 return ((cmi_hdl_t)cmi_hdl_arr[i].cmae_hdlp); 949 } 950 951 return (NULL); 952 } 953 954 void 955 cmi_hdl_walk(int (*cbfunc)(cmi_hdl_t, void *, void *, void *), 956 void *arg1, void *arg2, void *arg3) 957 { 958 int i; 959 960 for (i = 0; i < CMI_HDL_ARR_SZ; i++) { 961 if (cmi_hdl_canref(i)) { 962 cmi_hdl_impl_t *hdl = cmi_hdl_arr[i].cmae_hdlp; 963 964 if ((*cbfunc)((cmi_hdl_t)hdl, arg1, arg2, arg3) == 965 CMI_HDL_WALK_DONE) { 966 cmi_hdl_rele((cmi_hdl_t)hdl); 967 break; 968 } 969 cmi_hdl_rele((cmi_hdl_t)hdl); 970 } 971 } 972 } 973 974 void 975 cmi_hdl_setcmi(cmi_hdl_t ophdl, void *cmi, void *cmidata) 976 { 977 IMPLHDL(ophdl)->cmih_cmidata = cmidata; 978 IMPLHDL(ophdl)->cmih_cmi = cmi; 979 } 980 981 void * 982 cmi_hdl_getcmi(cmi_hdl_t ophdl) 983 { 984 return (IMPLHDL(ophdl)->cmih_cmi); 985 } 986 987 void * 988 cmi_hdl_getcmidata(cmi_hdl_t ophdl) 989 { 990 return (IMPLHDL(ophdl)->cmih_cmidata); 991 } 992 993 enum cmi_hdl_class 994 cmi_hdl_class(cmi_hdl_t ophdl) 995 { 996 return (IMPLHDL(ophdl)->cmih_class); 997 } 998 999 #define CMI_HDL_OPFUNC(what, type) \ 1000 type \ 1001 cmi_hdl_##what(cmi_hdl_t ophdl) \ 1002 { \ 1003 return (IMPLHDL(ophdl)->cmih_ops-> \ 1004 cmio_##what(IMPLHDL(ophdl))); \ 1005 } 1006 1007 CMI_HDL_OPFUNC(vendor, uint_t) 1008 CMI_HDL_OPFUNC(vendorstr, const char *) 1009 CMI_HDL_OPFUNC(family, uint_t) 1010 CMI_HDL_OPFUNC(model, uint_t) 1011 CMI_HDL_OPFUNC(stepping, uint_t) 1012 CMI_HDL_OPFUNC(chipid, uint_t) 1013 CMI_HDL_OPFUNC(coreid, uint_t) 1014 CMI_HDL_OPFUNC(strandid, uint_t) 1015 CMI_HDL_OPFUNC(chiprev, uint32_t) 1016 CMI_HDL_OPFUNC(chiprevstr, const char *) 1017 CMI_HDL_OPFUNC(getsockettype, uint32_t) 1018 1019 void 1020 cmi_hdl_mcheck(cmi_hdl_t ophdl) 1021 { 1022 IMPLHDL(ophdl)->cmih_ops->cmio_mcheck(IMPLHDL(ophdl)); 1023 } 1024 1025 #ifndef __xpv 1026 /* 1027 * Return hardware chip instance; cpuid_get_chipid provides this directly. 1028 */ 1029 uint_t 1030 cmi_ntv_hwchipid(cpu_t *cp) 1031 { 1032 return (cpuid_get_chipid(cp)); 1033 } 1034 1035 /* 1036 * Return core instance within a single chip. 1037 */ 1038 uint_t 1039 cmi_ntv_hwcoreid(cpu_t *cp) 1040 { 1041 return (cpuid_get_pkgcoreid(cp)); 1042 } 1043 1044 /* 1045 * Return strand number within a single core. cpuid_get_clogid numbers 1046 * all execution units (strands, or cores in unstranded models) sequentially 1047 * within a single chip. 1048 */ 1049 uint_t 1050 cmi_ntv_hwstrandid(cpu_t *cp) 1051 { 1052 int strands_per_core = cpuid_get_ncpu_per_chip(cp) / 1053 cpuid_get_ncore_per_chip(cp); 1054 1055 return (cpuid_get_clogid(cp) % strands_per_core); 1056 } 1057 #endif /* __xpv */ 1058 1059 void 1060 cmi_hdlconf_rdmsr_nohw(cmi_hdl_t ophdl) 1061 { 1062 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1063 1064 hdl->cmih_msrsrc &= ~CMI_MSR_FLAG_RD_HWOK; 1065 } 1066 1067 void 1068 cmi_hdlconf_wrmsr_nohw(cmi_hdl_t ophdl) 1069 { 1070 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1071 1072 hdl->cmih_msrsrc &= ~CMI_MSR_FLAG_WR_HWOK; 1073 } 1074 1075 cmi_errno_t 1076 cmi_hdl_rdmsr(cmi_hdl_t ophdl, uint_t msr, uint64_t *valp) 1077 { 1078 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1079 1080 /* 1081 * Regardless of the handle class, we first check for am 1082 * interposed value. In the xVM case you probably want to 1083 * place interposed values within the hypervisor itself, but 1084 * we still allow interposing them in dom0 for test and bringup 1085 * purposes. 1086 */ 1087 if ((hdl->cmih_msrsrc & CMI_MSR_FLAG_RD_INTERPOSEOK) && 1088 msri_lookup(hdl, msr, valp)) 1089 return (CMI_SUCCESS); 1090 1091 if (!(hdl->cmih_msrsrc & CMI_MSR_FLAG_RD_HWOK)) 1092 return (CMIERR_INTERPOSE); 1093 1094 return (hdl->cmih_ops->cmio_rdmsr(hdl, msr, valp)); 1095 } 1096 1097 cmi_errno_t 1098 cmi_hdl_wrmsr(cmi_hdl_t ophdl, uint_t msr, uint64_t val) 1099 { 1100 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1101 1102 /* Invalidate any interposed value */ 1103 msri_rment(hdl, msr); 1104 1105 if (!(hdl->cmih_msrsrc & CMI_MSR_FLAG_WR_HWOK)) 1106 return (CMI_SUCCESS); 1107 1108 return (hdl->cmih_ops->cmio_wrmsr(hdl, msr, val)); 1109 } 1110 1111 void 1112 cmi_hdl_enable_mce(cmi_hdl_t ophdl) 1113 { 1114 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1115 ulong_t cr4 = hdl->cmih_ops->cmio_getcr4(hdl); 1116 1117 hdl->cmih_ops->cmio_setcr4(hdl, cr4 | CR4_MCE); 1118 } 1119 1120 void 1121 cmi_hdl_msrinterpose(cmi_hdl_t ophdl, cmi_mca_regs_t *regs, uint_t nregs) 1122 { 1123 cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); 1124 int i; 1125 1126 for (i = 0; i < nregs; i++) 1127 msri_addent(hdl, regs++); 1128 } 1129 1130 void 1131 cmi_pcird_nohw(void) 1132 { 1133 cmi_pcicfg_flags &= ~CMI_PCICFG_FLAG_RD_HWOK; 1134 } 1135 1136 void 1137 cmi_pciwr_nohw(void) 1138 { 1139 cmi_pcicfg_flags &= ~CMI_PCICFG_FLAG_WR_HWOK; 1140 } 1141 1142 static uint32_t 1143 cmi_pci_get_cmn(int bus, int dev, int func, int reg, int asz, 1144 int *interpose, ddi_acc_handle_t hdl) 1145 { 1146 uint32_t val; 1147 1148 if (cmi_pcicfg_flags & CMI_PCICFG_FLAG_RD_INTERPOSEOK && 1149 pcii_lookup(bus, dev, func, reg, asz, &val)) { 1150 if (interpose) 1151 *interpose = 1; 1152 return (val); 1153 } 1154 if (interpose) 1155 *interpose = 0; 1156 1157 if (!(cmi_pcicfg_flags & CMI_PCICFG_FLAG_RD_HWOK)) 1158 return (0); 1159 1160 switch (asz) { 1161 case 1: 1162 if (hdl) 1163 val = pci_config_get8(hdl, (off_t)reg); 1164 else 1165 val = (*pci_getb_func)(bus, dev, func, reg); 1166 break; 1167 case 2: 1168 if (hdl) 1169 val = pci_config_get16(hdl, (off_t)reg); 1170 else 1171 val = (*pci_getw_func)(bus, dev, func, reg); 1172 break; 1173 case 4: 1174 if (hdl) 1175 val = pci_config_get32(hdl, (off_t)reg); 1176 else 1177 val = (*pci_getl_func)(bus, dev, func, reg); 1178 break; 1179 default: 1180 val = 0; 1181 } 1182 return (val); 1183 } 1184 1185 uint8_t 1186 cmi_pci_getb(int bus, int dev, int func, int reg, int *interpose, 1187 ddi_acc_handle_t hdl) 1188 { 1189 return ((uint8_t)cmi_pci_get_cmn(bus, dev, func, reg, 1, interpose, 1190 hdl)); 1191 } 1192 1193 uint16_t 1194 cmi_pci_getw(int bus, int dev, int func, int reg, int *interpose, 1195 ddi_acc_handle_t hdl) 1196 { 1197 return ((uint16_t)cmi_pci_get_cmn(bus, dev, func, reg, 2, interpose, 1198 hdl)); 1199 } 1200 1201 uint32_t 1202 cmi_pci_getl(int bus, int dev, int func, int reg, int *interpose, 1203 ddi_acc_handle_t hdl) 1204 { 1205 return (cmi_pci_get_cmn(bus, dev, func, reg, 4, interpose, hdl)); 1206 } 1207 1208 void 1209 cmi_pci_interposeb(int bus, int dev, int func, int reg, uint8_t val) 1210 { 1211 pcii_addent(bus, dev, func, reg, val, 1); 1212 } 1213 1214 void 1215 cmi_pci_interposew(int bus, int dev, int func, int reg, uint16_t val) 1216 { 1217 pcii_addent(bus, dev, func, reg, val, 2); 1218 } 1219 1220 void 1221 cmi_pci_interposel(int bus, int dev, int func, int reg, uint32_t val) 1222 { 1223 pcii_addent(bus, dev, func, reg, val, 4); 1224 } 1225 1226 static void 1227 cmi_pci_put_cmn(int bus, int dev, int func, int reg, int asz, 1228 ddi_acc_handle_t hdl, uint32_t val) 1229 { 1230 /* 1231 * If there is an interposed value for this register invalidate it. 1232 */ 1233 pcii_rment(bus, dev, func, reg, asz); 1234 1235 if (!(cmi_pcicfg_flags & CMI_PCICFG_FLAG_WR_HWOK)) 1236 return; 1237 1238 switch (asz) { 1239 case 1: 1240 if (hdl) 1241 pci_config_put8(hdl, (off_t)reg, (uint8_t)val); 1242 else 1243 (*pci_putb_func)(bus, dev, func, reg, (uint8_t)val); 1244 break; 1245 1246 case 2: 1247 if (hdl) 1248 pci_config_put16(hdl, (off_t)reg, (uint16_t)val); 1249 else 1250 (*pci_putw_func)(bus, dev, func, reg, (uint16_t)val); 1251 break; 1252 1253 case 4: 1254 if (hdl) 1255 pci_config_put32(hdl, (off_t)reg, val); 1256 else 1257 (*pci_putl_func)(bus, dev, func, reg, val); 1258 break; 1259 1260 default: 1261 break; 1262 } 1263 } 1264 1265 extern void 1266 cmi_pci_putb(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, 1267 uint8_t val) 1268 { 1269 cmi_pci_put_cmn(bus, dev, func, reg, 1, hdl, val); 1270 } 1271 1272 extern void 1273 cmi_pci_putw(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, 1274 uint16_t val) 1275 { 1276 cmi_pci_put_cmn(bus, dev, func, reg, 2, hdl, val); 1277 } 1278 1279 extern void 1280 cmi_pci_putl(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, 1281 uint32_t val) 1282 { 1283 cmi_pci_put_cmn(bus, dev, func, reg, 4, hdl, val); 1284 } 1285