1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019, Joyent, Inc. 14 * Copyright 2022 Oxide Computer Company 15 */ 16 17 /* 18 * Nexus Driver for AMD Zen family systems. The purpose of this driver is to 19 * provide access to the following resources in a single, centralized fashion: 20 * 21 * - The per-chip Data Fabric 22 * - The North Bridge 23 * - The System Management Network (SMN) 24 * 25 * This is a nexus driver as once we have attached to all the requisite 26 * components, we will enumerate child devices which consume this functionality. 27 * 28 * ------------------------ 29 * Mapping Devices Together 30 * ------------------------ 31 * 32 * The operating system needs to expose things like temperature sensors and DRAM 33 * configuration registers in terms that are meaningful to the system such as 34 * logical CPUs, cores, etc. This driver attaches to the PCI IDs that represent 35 * the northbridge and data fabric; however, there are multiple PCI devices (one 36 * per die) that exist. This driver does manage to map all of these three things 37 * together; however, it requires some acrobatics. Unfortunately, there's no 38 * direct way to map a northbridge to its corresponding die. However, we can map 39 * a CPU die to a data fabric PCI device and a data fabric PCI device to a 40 * corresponding northbridge PCI device. 41 * 42 * In current Zen based products, there is a direct mapping between processor 43 * nodes and a data fabric PCI device. All of the devices are on PCI Bus 0 and 44 * start from Device 0x18. Device 0x18 maps to processor node 0, 0x19 to 45 * processor node 1, etc. This means that to map a logical CPU to a data fabric 46 * device, we take its processor node id, add it to 0x18 and find the PCI device 47 * that is on bus 0, device 0x18. As each data fabric device is attached based 48 * on its PCI ID, we add it to the global list, amd_nbdf_dfs that is in the 49 * amd_f17nbdf_t structure. 50 * 51 * The northbridge PCI device has a defined device and function, but the PCI bus 52 * that it's on can vary. Each die has its own series of PCI buses that are 53 * assigned to it and the northbridge PCI device is on the first of die-specific 54 * PCI bus for each die. This also means that the northbridge will not show up 55 * on PCI bus 0, which is the PCI bus that all of the data fabric devices are 56 * on. While conventionally the northbridge with the lowest PCI bus value 57 * would correspond to processor node zero, hardware does not guarantee that at 58 * all. Because we don't want to be at the mercy of firmware, we don't rely on 59 * this ordering, even though we have yet to find a system that deviates from 60 * this scheme. 61 * 62 * One of the registers in the data fabric device's function 0 63 * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to have the first PCI bus that is 64 * associated with the processor node. This means that we can map a data fabric 65 * device to a northbridge by finding the northbridge whose PCI bus matches the 66 * value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. 67 * 68 * We can map a northbridge to a data fabric device and a data fabric device to 69 * a die. Because these are generally 1:1 mappings, there is a transitive 70 * relationship and therefore we know which northbridge is associated with which 71 * processor die. This is summarized in the following image: 72 * 73 * +-------+ +-----------------------------------+ +--------------+ 74 * | Die 0 |--->| Data Fabric PCI BDF 0/18/0 |------->| Northbridge | 75 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | 76 * ... +-----------------------------------+ +--------------+ 77 * +-------+ +------------------------------------+ +--------------+ 78 * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |------->| Northbridge | 79 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | 80 * +------------------------------------+ +--------------+ 81 * 82 * Note, the PCI buses used by the northbridges here are arbitrary. They do not 83 * reflect the actual values by hardware; however, the bus/device/function (BDF) 84 * of the data fabric accurately models hardware. All of the BDF values are in 85 * hex. 86 * 87 * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), 88 * AMD has multiple northbridges that exist on a given die. All of these 89 * northbridges share the same data fabric and system management network port. 90 * From our perspective this means that some of the northbridge devices will be 91 * redundant and that we will no longer have a 1:1 mapping between the 92 * northbridge and the data fabric devices. Every data fabric will have a 93 * northbridge, but not every northbridge will have a data fabric device mapped. 94 * Because we're always trying to map from a die to a northbridge and not the 95 * reverse, the fact that there are extra northbridge devices hanging around 96 * that we don't know about shouldn't be a problem. 97 * 98 * ------------------------------- 99 * Attach and Detach Complications 100 * ------------------------------- 101 * 102 * Because we need to map different PCI devices together, this means that we 103 * have multiple dev_info_t structures that we need to manage. Each of these is 104 * independently attached and detached. While this is easily managed for attach, 105 * it is not for detach. Each of these devices is a 'stub'. 106 * 107 * Once a device has been detached it will only come back if we have an active 108 * minor node that will be accessed. This means that if they are detached, 109 * nothing would ever cause them to be reattached. The system also doesn't 110 * provide us a way or any guarantees around making sure that we're attached to 111 * all such devices before we detach. As a result, unfortunately, it's easier to 112 * basically have detach always fail. 113 * 114 * --------------- 115 * Exposed Devices 116 * --------------- 117 * 118 * Rather than try and have all of the different functions that could be 119 * provided by one driver, we instead have created a nexus driver that will 120 * itself try and load children. Children are all pseudo-device drivers that 121 * provide different pieces of functionality that use this. 122 * 123 * ------- 124 * Locking 125 * ------- 126 * 127 * The amdzen_data structure contains a single lock, azn_mutex. The various 128 * client functions are intended for direct children of our nexus, but have been 129 * designed in case someone else depends on this driver despite not being a 130 * child. Once a DF has been discovered, the set of entities inside of it 131 * (adf_nents, adf_ents[]) is considered static, constant data. This means that 132 * iterating over it in and of itself does not require locking; however, the 133 * discovery of the amd_df_t does. In addition, whenever performing register 134 * accesses to the DF or SMN, those require locking. This means that one must 135 * hold the lock in the following circumstances: 136 * 137 * o Looking up DF structures 138 * o Reading or writing to DF registers 139 * o Reading or writing to SMN registers 140 * 141 * In general, it is preferred that the lock be held across an entire client 142 * operation if possible. The only time this becomes an issue are when we have 143 * callbacks into our callers (ala amdzen_c_df_iter()) as they will likely 144 * recursively call into us. 145 */ 146 147 #include <sys/modctl.h> 148 #include <sys/conf.h> 149 #include <sys/devops.h> 150 #include <sys/ddi.h> 151 #include <sys/sunddi.h> 152 #include <sys/pci.h> 153 #include <sys/sysmacros.h> 154 #include <sys/sunndi.h> 155 #include <sys/x86_archext.h> 156 #include <sys/cpuvar.h> 157 158 #include <sys/amdzen/df.h> 159 #include "amdzen_client.h" 160 #include "amdzen.h" 161 162 amdzen_t *amdzen_data; 163 164 /* 165 * Array of northbridge IDs that we care about. 166 */ 167 static const uint16_t amdzen_nb_ids[] = { 168 /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 169 0x1450, 170 /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 171 0x15d0, 172 /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 173 0x1480, 174 /* Family 17h/19h Renoir, Cezanne, Van Gogh Zen 2/3 uarch */ 175 0x1630, 176 /* Family 19h Genoa */ 177 0x14a4, 178 /* Family 17h Mendocino, Family 19h Rembrandt */ 179 0x14b5, 180 /* Family 19h Raphael */ 181 0x14d8 182 }; 183 184 typedef struct { 185 char *acd_name; 186 amdzen_child_t acd_addr; 187 } amdzen_child_data_t; 188 189 static const amdzen_child_data_t amdzen_children[] = { 190 { "smntemp", AMDZEN_C_SMNTEMP }, 191 { "usmn", AMDZEN_C_USMN }, 192 { "zen_udf", AMDZEN_C_ZEN_UDF }, 193 { "zen_umc", AMDZEN_C_ZEN_UMC } 194 }; 195 196 static uint8_t 197 amdzen_stub_get8(amdzen_stub_t *stub, off_t reg) 198 { 199 return (pci_config_get8(stub->azns_cfgspace, reg)); 200 } 201 202 static uint16_t 203 amdzen_stub_get16(amdzen_stub_t *stub, off_t reg) 204 { 205 return (pci_config_get16(stub->azns_cfgspace, reg)); 206 } 207 208 static uint32_t 209 amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) 210 { 211 return (pci_config_get32(stub->azns_cfgspace, reg)); 212 } 213 214 static uint64_t 215 amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) 216 { 217 return (pci_config_get64(stub->azns_cfgspace, reg)); 218 } 219 220 static void 221 amdzen_stub_put8(amdzen_stub_t *stub, off_t reg, uint8_t val) 222 { 223 pci_config_put8(stub->azns_cfgspace, reg, val); 224 } 225 226 static void 227 amdzen_stub_put16(amdzen_stub_t *stub, off_t reg, uint16_t val) 228 { 229 pci_config_put16(stub->azns_cfgspace, reg, val); 230 } 231 232 static void 233 amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) 234 { 235 pci_config_put32(stub->azns_cfgspace, reg, val); 236 } 237 238 static uint64_t 239 amdzen_df_read_regdef(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def, 240 uint8_t inst, boolean_t do_64) 241 { 242 df_reg_def_t ficaa; 243 df_reg_def_t ficad; 244 uint32_t val = 0; 245 df_rev_t df_rev = azn->azn_dfs[0].adf_rev; 246 247 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 248 ASSERT3U(def.drd_gens & df_rev, ==, df_rev); 249 val = DF_FICAA_V2_SET_TARG_INST(val, 1); 250 val = DF_FICAA_V2_SET_FUNC(val, def.drd_func); 251 val = DF_FICAA_V2_SET_INST(val, inst); 252 val = DF_FICAA_V2_SET_64B(val, do_64 ? 1 : 0); 253 254 switch (df_rev) { 255 case DF_REV_2: 256 case DF_REV_3: 257 case DF_REV_3P5: 258 ficaa = DF_FICAA_V2; 259 ficad = DF_FICAD_LO_V2; 260 /* 261 * Both here and in the DFv4 case, the register ignores the 262 * lower 2 bits. That is we can only address and encode things 263 * in units of 4 bytes. 264 */ 265 val = DF_FICAA_V2_SET_REG(val, def.drd_reg >> 2); 266 break; 267 case DF_REV_4: 268 ficaa = DF_FICAA_V4; 269 ficad = DF_FICAD_LO_V4; 270 val = DF_FICAA_V4_SET_REG(val, def.drd_reg >> 2); 271 break; 272 default: 273 panic("encountered unexpected DF rev: %u", df_rev); 274 } 275 276 amdzen_stub_put32(df->adf_funcs[ficaa.drd_func], ficaa.drd_reg, val); 277 if (do_64) { 278 return (amdzen_stub_get64(df->adf_funcs[ficad.drd_func], 279 ficad.drd_reg)); 280 } else { 281 return (amdzen_stub_get32(df->adf_funcs[ficad.drd_func], 282 ficad.drd_reg)); 283 } 284 } 285 286 /* 287 * Perform a targeted 32-bit indirect read to a specific instance and function. 288 */ 289 static uint32_t 290 amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, 291 const df_reg_def_t def) 292 { 293 return (amdzen_df_read_regdef(azn, df, def, inst, B_FALSE)); 294 } 295 296 /* 297 * For a broadcast read, just go to the underlying PCI function and perform a 298 * read. At this point in time, we don't believe we need to use the FICAA/FICAD 299 * to access it (though it does have a broadcast mode). 300 */ 301 static uint32_t 302 amdzen_df_read32_bcast(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def) 303 { 304 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 305 return (amdzen_stub_get32(df->adf_funcs[def.drd_func], def.drd_reg)); 306 } 307 308 static uint32_t 309 amdzen_smn_read(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg) 310 { 311 const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); 312 const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); 313 314 VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); 315 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 316 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); 317 318 switch (SMN_REG_SIZE(reg)) { 319 case 1: 320 return ((uint32_t)amdzen_stub_get8(df->adf_nb, 321 AMDZEN_NB_SMN_DATA + addr_off)); 322 case 2: 323 return ((uint32_t)amdzen_stub_get16(df->adf_nb, 324 AMDZEN_NB_SMN_DATA + addr_off)); 325 case 4: 326 return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); 327 default: 328 panic("unreachable invalid SMN register size %u", 329 SMN_REG_SIZE(reg)); 330 } 331 } 332 333 static void 334 amdzen_smn_write(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg, 335 const uint32_t val) 336 { 337 const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); 338 const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); 339 340 VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); 341 VERIFY(SMN_REG_VALUE_FITS(reg, val)); 342 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 343 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); 344 345 switch (SMN_REG_SIZE(reg)) { 346 case 1: 347 amdzen_stub_put8(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, 348 (uint8_t)val); 349 break; 350 case 2: 351 amdzen_stub_put16(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, 352 (uint16_t)val); 353 break; 354 case 4: 355 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val); 356 break; 357 default: 358 panic("unreachable invalid SMN register size %u", 359 SMN_REG_SIZE(reg)); 360 } 361 } 362 363 static amdzen_df_t * 364 amdzen_df_find(amdzen_t *azn, uint_t dfno) 365 { 366 uint_t i; 367 368 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 369 if (dfno >= azn->azn_ndfs) { 370 return (NULL); 371 } 372 373 for (i = 0; i < azn->azn_ndfs; i++) { 374 amdzen_df_t *df = &azn->azn_dfs[i]; 375 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { 376 continue; 377 } 378 379 if (dfno == 0) { 380 return (df); 381 } 382 dfno--; 383 } 384 385 return (NULL); 386 } 387 388 /* 389 * Client functions that are used by nexus children. 390 */ 391 int 392 amdzen_c_smn_read(uint_t dfno, const smn_reg_t reg, uint32_t *valp) 393 { 394 amdzen_df_t *df; 395 amdzen_t *azn = amdzen_data; 396 397 if (!SMN_REG_SIZE_IS_VALID(reg)) 398 return (EINVAL); 399 if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) 400 return (EINVAL); 401 402 mutex_enter(&azn->azn_mutex); 403 df = amdzen_df_find(azn, dfno); 404 if (df == NULL) { 405 mutex_exit(&azn->azn_mutex); 406 return (ENOENT); 407 } 408 409 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 410 mutex_exit(&azn->azn_mutex); 411 return (ENXIO); 412 } 413 414 *valp = amdzen_smn_read(azn, df, reg); 415 mutex_exit(&azn->azn_mutex); 416 return (0); 417 } 418 419 int 420 amdzen_c_smn_write(uint_t dfno, const smn_reg_t reg, const uint32_t val) 421 { 422 amdzen_df_t *df; 423 amdzen_t *azn = amdzen_data; 424 425 if (!SMN_REG_SIZE_IS_VALID(reg)) 426 return (EINVAL); 427 if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) 428 return (EINVAL); 429 if (!SMN_REG_VALUE_FITS(reg, val)) 430 return (EOVERFLOW); 431 432 mutex_enter(&azn->azn_mutex); 433 df = amdzen_df_find(azn, dfno); 434 if (df == NULL) { 435 mutex_exit(&azn->azn_mutex); 436 return (ENOENT); 437 } 438 439 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 440 mutex_exit(&azn->azn_mutex); 441 return (ENXIO); 442 } 443 444 amdzen_smn_write(azn, df, reg, val); 445 mutex_exit(&azn->azn_mutex); 446 return (0); 447 } 448 449 uint_t 450 amdzen_c_df_count(void) 451 { 452 uint_t ret; 453 amdzen_t *azn = amdzen_data; 454 455 mutex_enter(&azn->azn_mutex); 456 ret = azn->azn_ndfs; 457 mutex_exit(&azn->azn_mutex); 458 return (ret); 459 } 460 461 df_rev_t 462 amdzen_c_df_rev(void) 463 { 464 amdzen_df_t *df; 465 amdzen_t *azn = amdzen_data; 466 df_rev_t rev; 467 468 /* 469 * Always use the first DF instance to determine what we're using. Our 470 * current assumption, which seems to generally be true, is that the 471 * given DF revisions are the same in a given system when the DFs are 472 * directly connected. 473 */ 474 mutex_enter(&azn->azn_mutex); 475 df = amdzen_df_find(azn, 0); 476 if (df == NULL) { 477 rev = DF_REV_UNKNOWN; 478 } else { 479 rev = df->adf_rev; 480 } 481 mutex_exit(&azn->azn_mutex); 482 483 return (rev); 484 } 485 486 int 487 amdzen_c_df_read32(uint_t dfno, uint8_t inst, const df_reg_def_t def, 488 uint32_t *valp) 489 { 490 amdzen_df_t *df; 491 amdzen_t *azn = amdzen_data; 492 493 mutex_enter(&azn->azn_mutex); 494 df = amdzen_df_find(azn, dfno); 495 if (df == NULL) { 496 mutex_exit(&azn->azn_mutex); 497 return (ENOENT); 498 } 499 500 *valp = amdzen_df_read_regdef(azn, df, def, inst, B_FALSE); 501 mutex_exit(&azn->azn_mutex); 502 503 return (0); 504 } 505 506 int 507 amdzen_c_df_read64(uint_t dfno, uint8_t inst, const df_reg_def_t def, 508 uint64_t *valp) 509 { 510 amdzen_df_t *df; 511 amdzen_t *azn = amdzen_data; 512 513 mutex_enter(&azn->azn_mutex); 514 df = amdzen_df_find(azn, dfno); 515 if (df == NULL) { 516 mutex_exit(&azn->azn_mutex); 517 return (ENOENT); 518 } 519 520 *valp = amdzen_df_read_regdef(azn, df, def, inst, B_TRUE); 521 mutex_exit(&azn->azn_mutex); 522 523 return (0); 524 } 525 526 int 527 amdzen_c_df_iter(uint_t dfno, zen_df_type_t type, amdzen_c_iter_f func, 528 void *arg) 529 { 530 amdzen_df_t *df; 531 amdzen_t *azn = amdzen_data; 532 df_type_t df_type; 533 uint8_t df_subtype; 534 535 /* 536 * Unlike other calls here, we hold our lock only to find the DF here. 537 * The main reason for this is the nature of the callback function. 538 * Folks are iterating over instances so they can call back into us. If 539 * you look at the locking statement, the thing that is most volatile 540 * right here and what we need to protect is the DF itself and 541 * subsequent register accesses to it. The actual data about which 542 * entities exist is static and so once we have found a DF we should 543 * hopefully be in good shape as they only come, but don't go. 544 */ 545 mutex_enter(&azn->azn_mutex); 546 df = amdzen_df_find(azn, dfno); 547 if (df == NULL) { 548 mutex_exit(&azn->azn_mutex); 549 return (ENOENT); 550 } 551 mutex_exit(&azn->azn_mutex); 552 553 switch (type) { 554 case ZEN_DF_TYPE_CS_UMC: 555 df_type = DF_TYPE_CS; 556 /* 557 * In the original Zeppelin DFv2 die there was no subtype field 558 * used for the CS. The UMC is the only type and has a subtype 559 * of zero. 560 */ 561 if (df->adf_rev != DF_REV_2) { 562 df_subtype = DF_CS_SUBTYPE_UMC; 563 } else { 564 df_subtype = 0; 565 } 566 break; 567 case ZEN_DF_TYPE_CCM_CPU: 568 /* 569 * While the wording of the PPR is a little weird, the CCM still 570 * has subtype 0 in DFv4 systems; however, what's said to be for 571 * the CPU appears to apply to the ACM. 572 */ 573 df_type = DF_TYPE_CCM; 574 df_subtype = 0; 575 break; 576 default: 577 return (EINVAL); 578 } 579 580 for (uint_t i = 0; i < df->adf_nents; i++) { 581 amdzen_df_ent_t *ent = &df->adf_ents[i]; 582 583 /* 584 * Some DF components are not considered enabled and therefore 585 * will end up having bogus values in their ID fields. If we do 586 * not have an enable flag set, we must skip this node. 587 */ 588 if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 589 continue; 590 591 if (ent->adfe_type == df_type && 592 ent->adfe_subtype == df_subtype) { 593 int ret = func(dfno, ent->adfe_fabric_id, 594 ent->adfe_inst_id, arg); 595 if (ret != 0) { 596 return (ret); 597 } 598 } 599 } 600 601 return (0); 602 } 603 604 int 605 amdzen_c_df_fabric_decomp(df_fabric_decomp_t *decomp) 606 { 607 const amdzen_df_t *df; 608 amdzen_t *azn = amdzen_data; 609 610 mutex_enter(&azn->azn_mutex); 611 df = amdzen_df_find(azn, 0); 612 if (df == NULL) { 613 mutex_exit(&azn->azn_mutex); 614 return (ENOENT); 615 } 616 617 *decomp = df->adf_decomp; 618 mutex_exit(&azn->azn_mutex); 619 return (0); 620 } 621 622 static boolean_t 623 amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) 624 { 625 int ret; 626 dev_info_t *child; 627 628 if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, 629 (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { 630 dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " 631 "dip for %s", acd->acd_name); 632 return (B_FALSE); 633 } 634 635 ddi_set_parent_data(child, (void *)acd); 636 if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { 637 dev_err(azn->azn_dip, CE_WARN, "!failed to online child " 638 "dip %s: %d", acd->acd_name, ret); 639 return (B_FALSE); 640 } 641 642 return (B_TRUE); 643 } 644 645 static boolean_t 646 amdzen_map_dfs(amdzen_t *azn) 647 { 648 amdzen_stub_t *stub; 649 650 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 651 652 for (stub = list_head(&azn->azn_df_stubs); stub != NULL; 653 stub = list_next(&azn->azn_df_stubs, stub)) { 654 amdzen_df_t *df; 655 uint_t dfno; 656 657 dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; 658 if (dfno > AMDZEN_MAX_DFS) { 659 dev_err(stub->azns_dip, CE_WARN, "encountered df " 660 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 661 stub->azns_bus, stub->azns_dev, stub->azns_func); 662 goto err; 663 } 664 665 df = &azn->azn_dfs[dfno]; 666 667 if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { 668 dev_err(stub->azns_dip, CE_WARN, "encountered df " 669 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 670 stub->azns_bus, stub->azns_dev, stub->azns_func); 671 goto err; 672 } 673 674 if (df->adf_funcs[stub->azns_func] != NULL) { 675 dev_err(stub->azns_dip, CE_WARN, "encountered " 676 "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", 677 stub->azns_bus, stub->azns_dev, stub->azns_func); 678 goto err; 679 } 680 df->adf_funcs[stub->azns_func] = stub; 681 } 682 683 return (B_TRUE); 684 685 err: 686 azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; 687 return (B_FALSE); 688 } 689 690 static boolean_t 691 amdzen_check_dfs(amdzen_t *azn) 692 { 693 uint_t i; 694 boolean_t ret = B_TRUE; 695 696 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 697 amdzen_df_t *df = &azn->azn_dfs[i]; 698 uint_t count = 0; 699 700 /* 701 * We require all platforms to have DFs functions 0-6. Not all 702 * platforms have DF function 7. 703 */ 704 for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { 705 if (df->adf_funcs[func] != NULL) { 706 count++; 707 } 708 } 709 710 if (count == 0) 711 continue; 712 713 if (count != 7) { 714 ret = B_FALSE; 715 dev_err(azn->azn_dip, CE_WARN, "df %u devices " 716 "incomplete", i); 717 } else { 718 df->adf_flags |= AMDZEN_DF_F_VALID; 719 azn->azn_ndfs++; 720 } 721 } 722 723 return (ret); 724 } 725 726 static const uint8_t amdzen_df_rome_ids[0x2b] = { 727 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 728 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 729 44, 45, 46, 47, 48 730 }; 731 732 /* 733 * Check the first df entry to see if it belongs to Rome or Milan. If so, then 734 * it uses the disjoint ID space. 735 */ 736 static boolean_t 737 amdzen_is_rome_style(uint_t id) 738 { 739 return (id == 0x1490 || id == 0x1650); 740 } 741 742 /* 743 * To be able to do most other things we want to do, we must first determine 744 * what revision of the DF (data fabric) that we're using. 745 * 746 * Snapshot the df version. This was added explicitly in DFv4.0, around the Zen 747 * 4 timeframe and allows us to tell apart different version of the DF register 748 * set, most usefully when various subtypes were added. 749 * 750 * Older versions can theoretically be told apart based on usage of reserved 751 * registers. We walk these in the following order, starting with the newest rev 752 * and walking backwards to tell things apart: 753 * 754 * o v3.5 -> Check function 1, register 0x150. This was reserved prior 755 * to this point. This is actually DF_FIDMASK0_V3P5. We are supposed 756 * to check bits [7:0]. 757 * 758 * o v3.0 -> Check function 1, register 0x208. The low byte (7:0) was 759 * changed to indicate a component mask. This is non-zero 760 * in the 3.0 generation. This is actually DF_FIDMASK_V2. 761 * 762 * o v2.0 -> This is just the not that case. Presumably v1 wasn't part 763 * of the Zen generation. 764 * 765 * Because we don't know what version we are yet, we do not use the normal 766 * versioned register accesses which would check what DF version we are and 767 * would want to use the normal indirect register accesses (which also require 768 * us to know the version). We instead do direct broadcast reads. 769 */ 770 static void 771 amdzen_determine_df_vers(amdzen_t *azn, amdzen_df_t *df) 772 { 773 uint32_t val; 774 df_reg_def_t rd = DF_FBICNT; 775 776 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 777 df->adf_major = DF_FBICNT_V4_GET_MAJOR(val); 778 df->adf_minor = DF_FBICNT_V4_GET_MINOR(val); 779 if (df->adf_major == 0 && df->adf_minor == 0) { 780 rd = DF_FIDMASK0_V3P5; 781 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 782 if (bitx32(val, 7, 0) != 0) { 783 df->adf_major = 3; 784 df->adf_minor = 5; 785 df->adf_rev = DF_REV_3P5; 786 } else { 787 rd = DF_FIDMASK_V2; 788 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], 789 rd.drd_reg); 790 if (bitx32(val, 7, 0) != 0) { 791 df->adf_major = 3; 792 df->adf_minor = 0; 793 df->adf_rev = DF_REV_3; 794 } else { 795 df->adf_major = 2; 796 df->adf_minor = 0; 797 df->adf_rev = DF_REV_2; 798 } 799 } 800 } else if (df->adf_major == 4 && df->adf_minor == 0) { 801 df->adf_rev = DF_REV_4; 802 } else { 803 df->adf_rev = DF_REV_UNKNOWN; 804 } 805 } 806 807 /* 808 * All of the different versions of the DF have different ways of getting at and 809 * answering the question of how do I break a fabric ID into a corresponding 810 * socket, die, and component. Importantly the goal here is to obtain, cache, 811 * and normalize: 812 * 813 * o The DF System Configuration 814 * o The various Mask registers 815 * o The Node ID 816 */ 817 static void 818 amdzen_determine_fabric_decomp(amdzen_t *azn, amdzen_df_t *df) 819 { 820 uint32_t mask; 821 df_fabric_decomp_t *decomp = &df->adf_decomp; 822 823 switch (df->adf_rev) { 824 case DF_REV_2: 825 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V2); 826 switch (DF_SYSCFG_V2_GET_MY_TYPE(df->adf_syscfg)) { 827 case DF_DIE_TYPE_CPU: 828 mask = amdzen_df_read32_bcast(azn, df, 829 DF_DIEMASK_CPU_V2); 830 break; 831 case DF_DIE_TYPE_APU: 832 mask = amdzen_df_read32_bcast(azn, df, 833 DF_DIEMASK_APU_V2); 834 break; 835 default: 836 panic("DF thinks we're not on a CPU!"); 837 } 838 df->adf_mask0 = mask; 839 840 /* 841 * DFv2 is a bit different in how the fabric mask register is 842 * phrased. Logically a fabric ID is broken into something that 843 * uniquely identifies a "node" (a particular die on a socket) 844 * and something that identifies a "component", e.g. a memory 845 * controller. 846 * 847 * Starting with DFv3, these registers logically called out how 848 * to separate the fabric ID first into a node and a component. 849 * Then the node was then broken down into a socket and die. In 850 * DFv2, there is no separate mask and shift of a node. Instead 851 * the socket and die are absolute offsets into the fabric ID 852 * rather than relative offsets into the node ID. As such, when 853 * we encounter DFv2, we fake up a node mask and shift and make 854 * it look like DFv3+. 855 */ 856 decomp->dfd_node_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) | 857 DF_DIEMASK_V2_GET_DIE_MASK(mask); 858 decomp->dfd_node_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask); 859 decomp->dfd_comp_mask = DF_DIEMASK_V2_GET_COMP_MASK(mask); 860 decomp->dfd_comp_shift = 0; 861 862 decomp->dfd_sock_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) >> 863 decomp->dfd_node_shift; 864 decomp->dfd_die_mask = DF_DIEMASK_V2_GET_DIE_MASK(mask) >> 865 decomp->dfd_node_shift; 866 decomp->dfd_sock_shift = DF_DIEMASK_V2_GET_SOCK_SHIFT(mask) - 867 decomp->dfd_node_shift; 868 decomp->dfd_die_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask) - 869 decomp->dfd_node_shift; 870 ASSERT3U(decomp->dfd_die_shift, ==, 0); 871 break; 872 case DF_REV_3: 873 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3); 874 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 875 DF_FIDMASK0_V3); 876 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 877 DF_FIDMASK1_V3); 878 879 decomp->dfd_sock_mask = 880 DF_FIDMASK1_V3_GET_SOCK_MASK(df->adf_mask1); 881 decomp->dfd_sock_shift = 882 DF_FIDMASK1_V3_GET_SOCK_SHIFT(df->adf_mask1); 883 decomp->dfd_die_mask = 884 DF_FIDMASK1_V3_GET_DIE_MASK(df->adf_mask1); 885 decomp->dfd_die_shift = 0; 886 decomp->dfd_node_mask = 887 DF_FIDMASK0_V3_GET_NODE_MASK(df->adf_mask0); 888 decomp->dfd_node_shift = 889 DF_FIDMASK1_V3_GET_NODE_SHIFT(df->adf_mask1); 890 decomp->dfd_comp_mask = 891 DF_FIDMASK0_V3_GET_COMP_MASK(df->adf_mask0); 892 decomp->dfd_comp_shift = 0; 893 break; 894 case DF_REV_3P5: 895 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, 896 DF_SYSCFG_V3P5); 897 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 898 DF_FIDMASK0_V3P5); 899 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 900 DF_FIDMASK1_V3P5); 901 df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 902 DF_FIDMASK2_V3P5); 903 904 decomp->dfd_sock_mask = 905 DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 906 decomp->dfd_sock_shift = 907 DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 908 decomp->dfd_die_mask = 909 DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 910 decomp->dfd_die_shift = 0; 911 decomp->dfd_node_mask = 912 DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 913 decomp->dfd_node_shift = 914 DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 915 decomp->dfd_comp_mask = 916 DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 917 decomp->dfd_comp_shift = 0; 918 break; 919 case DF_REV_4: 920 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V4); 921 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 922 DF_FIDMASK0_V4); 923 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 924 DF_FIDMASK1_V4); 925 df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 926 DF_FIDMASK2_V4); 927 928 /* 929 * The DFv4 registers are at a different location in the DF; 930 * however, the actual layout of fields is the same as DFv3.5. 931 * This is why you see V3P5 below. 932 */ 933 decomp->dfd_sock_mask = 934 DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 935 decomp->dfd_sock_shift = 936 DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 937 decomp->dfd_die_mask = 938 DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 939 decomp->dfd_die_shift = 0; 940 decomp->dfd_node_mask = 941 DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 942 decomp->dfd_node_shift = 943 DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 944 decomp->dfd_comp_mask = 945 DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 946 decomp->dfd_comp_shift = 0; 947 break; 948 default: 949 panic("encountered suspicious, previously rejected DF " 950 "rev: 0x%x", df->adf_rev); 951 } 952 } 953 954 /* 955 * Initialize our knowledge about a given series of nodes on the data fabric. 956 */ 957 static void 958 amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) 959 { 960 uint_t i; 961 uint32_t val; 962 963 amdzen_determine_df_vers(azn, df); 964 965 switch (df->adf_rev) { 966 case DF_REV_2: 967 case DF_REV_3: 968 case DF_REV_3P5: 969 val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V2); 970 break; 971 case DF_REV_4: 972 val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V4); 973 break; 974 default: 975 dev_err(azn->azn_dip, CE_WARN, "encountered unsupported DF " 976 "revision: 0x%x", df->adf_rev); 977 return; 978 } 979 df->adf_nb_busno = DF_CFG_ADDR_CTL_GET_BUS_NUM(val); 980 val = amdzen_df_read32_bcast(azn, df, DF_FBICNT); 981 df->adf_nents = DF_FBICNT_GET_COUNT(val); 982 if (df->adf_nents == 0) 983 return; 984 df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, 985 KM_SLEEP); 986 987 for (i = 0; i < df->adf_nents; i++) { 988 amdzen_df_ent_t *dfe = &df->adf_ents[i]; 989 uint8_t inst = i; 990 991 /* 992 * Unfortunately, Rome uses a discontinuous instance ID pattern 993 * while everything else we can find uses a contiguous instance 994 * ID pattern. This means that for Rome, we need to adjust the 995 * indexes that we iterate over, though the total number of 996 * entries is right. This was carried over into Milan, but not 997 * Genoa. 998 */ 999 if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { 1000 if (inst > ARRAY_SIZE(amdzen_df_rome_ids)) { 1001 dev_err(azn->azn_dip, CE_WARN, "Rome family " 1002 "processor reported more ids than the PPR, " 1003 "resetting %u to instance zero", inst); 1004 inst = 0; 1005 } else { 1006 inst = amdzen_df_rome_ids[inst]; 1007 } 1008 } 1009 1010 dfe->adfe_drvid = inst; 1011 dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, DF_FBIINFO0); 1012 dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, DF_FBIINFO1); 1013 dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, DF_FBIINFO2); 1014 dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, DF_FBIINFO3); 1015 1016 dfe->adfe_type = DF_FBIINFO0_GET_TYPE(dfe->adfe_info0); 1017 dfe->adfe_subtype = DF_FBIINFO0_GET_SUBTYPE(dfe->adfe_info0); 1018 1019 /* 1020 * The enabled flag was not present in Zen 1. Simulate it by 1021 * checking for a non-zero register instead. 1022 */ 1023 if (DF_FBIINFO0_V3_GET_ENABLED(dfe->adfe_info0) || 1024 (df->adf_rev == DF_REV_2 && dfe->adfe_info0 != 0)) { 1025 dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; 1026 } 1027 if (DF_FBIINFO0_GET_HAS_MCA(dfe->adfe_info0)) { 1028 dfe->adfe_flags |= AMDZEN_DFE_F_MCA; 1029 } 1030 dfe->adfe_inst_id = DF_FBIINFO3_GET_INSTID(dfe->adfe_info3); 1031 switch (df->adf_rev) { 1032 case DF_REV_2: 1033 dfe->adfe_fabric_id = 1034 DF_FBIINFO3_V2_GET_BLOCKID(dfe->adfe_info3); 1035 break; 1036 case DF_REV_3: 1037 dfe->adfe_fabric_id = 1038 DF_FBIINFO3_V3_GET_BLOCKID(dfe->adfe_info3); 1039 break; 1040 case DF_REV_3P5: 1041 dfe->adfe_fabric_id = 1042 DF_FBIINFO3_V3P5_GET_BLOCKID(dfe->adfe_info3); 1043 break; 1044 case DF_REV_4: 1045 dfe->adfe_fabric_id = 1046 DF_FBIINFO3_V4_GET_BLOCKID(dfe->adfe_info3); 1047 break; 1048 default: 1049 panic("encountered suspicious, previously rejected DF " 1050 "rev: 0x%x", df->adf_rev); 1051 } 1052 } 1053 1054 amdzen_determine_fabric_decomp(azn, df); 1055 } 1056 1057 static void 1058 amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) 1059 { 1060 amdzen_stub_t *stub; 1061 1062 for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; 1063 stub = list_next(&azn->azn_nb_stubs, stub)) { 1064 if (stub->azns_bus == df->adf_nb_busno) { 1065 df->adf_flags |= AMDZEN_DF_F_FOUND_NB; 1066 df->adf_nb = stub; 1067 return; 1068 } 1069 } 1070 } 1071 1072 static void 1073 amdzen_nexus_init(void *arg) 1074 { 1075 uint_t i; 1076 amdzen_t *azn = arg; 1077 1078 /* 1079 * First go through all of the stubs and assign the DF entries. 1080 */ 1081 mutex_enter(&azn->azn_mutex); 1082 if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { 1083 azn->azn_flags |= AMDZEN_F_MAP_ERROR; 1084 goto done; 1085 } 1086 1087 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 1088 amdzen_df_t *df = &azn->azn_dfs[i]; 1089 1090 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) 1091 continue; 1092 amdzen_setup_df(azn, df); 1093 amdzen_find_nb(azn, df); 1094 } 1095 1096 /* 1097 * Not all children may be installed. As such, we do not treat the 1098 * failure of a child as fatal to the driver. 1099 */ 1100 mutex_exit(&azn->azn_mutex); 1101 for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { 1102 (void) amdzen_create_child(azn, &amdzen_children[i]); 1103 } 1104 mutex_enter(&azn->azn_mutex); 1105 1106 done: 1107 azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; 1108 azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; 1109 azn->azn_taskqid = TASKQID_INVALID; 1110 cv_broadcast(&azn->azn_cv); 1111 mutex_exit(&azn->azn_mutex); 1112 } 1113 1114 static int 1115 amdzen_stub_scan_cb(dev_info_t *dip, void *arg) 1116 { 1117 amdzen_t *azn = arg; 1118 uint16_t vid, did; 1119 int *regs; 1120 uint_t nregs, i; 1121 boolean_t match = B_FALSE; 1122 1123 if (dip == ddi_root_node()) { 1124 return (DDI_WALK_CONTINUE); 1125 } 1126 1127 /* 1128 * If a node in question is not a pci node, then we have no interest in 1129 * it as all the stubs that we care about are related to pci devices. 1130 */ 1131 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 1132 return (DDI_WALK_PRUNECHILD); 1133 } 1134 1135 /* 1136 * If we can't get a device or vendor ID and prove that this is an AMD 1137 * part, then we don't care about it. 1138 */ 1139 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1140 "vendor-id", PCI_EINVAL16); 1141 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1142 "device-id", PCI_EINVAL16); 1143 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 1144 return (DDI_WALK_CONTINUE); 1145 } 1146 1147 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 1148 return (DDI_WALK_CONTINUE); 1149 } 1150 1151 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 1152 if (amdzen_nb_ids[i] == did) { 1153 match = B_TRUE; 1154 } 1155 } 1156 1157 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1158 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 1159 return (DDI_WALK_CONTINUE); 1160 } 1161 1162 if (nregs == 0) { 1163 ddi_prop_free(regs); 1164 return (DDI_WALK_CONTINUE); 1165 } 1166 1167 if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && 1168 PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { 1169 match = B_TRUE; 1170 } 1171 1172 ddi_prop_free(regs); 1173 if (match) { 1174 mutex_enter(&azn->azn_mutex); 1175 azn->azn_nscanned++; 1176 mutex_exit(&azn->azn_mutex); 1177 } 1178 1179 return (DDI_WALK_CONTINUE); 1180 } 1181 1182 static void 1183 amdzen_stub_scan(void *arg) 1184 { 1185 amdzen_t *azn = arg; 1186 1187 mutex_enter(&azn->azn_mutex); 1188 azn->azn_nscanned = 0; 1189 mutex_exit(&azn->azn_mutex); 1190 1191 ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); 1192 1193 mutex_enter(&azn->azn_mutex); 1194 azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; 1195 azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; 1196 1197 if (azn->azn_nscanned == 0) { 1198 azn->azn_flags |= AMDZEN_F_UNSUPPORTED; 1199 azn->azn_taskqid = TASKQID_INVALID; 1200 cv_broadcast(&azn->azn_cv); 1201 } else if (azn->azn_npresent == azn->azn_nscanned) { 1202 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 1203 azn->azn_taskqid = taskq_dispatch(system_taskq, 1204 amdzen_nexus_init, azn, TQ_SLEEP); 1205 } 1206 mutex_exit(&azn->azn_mutex); 1207 } 1208 1209 /* 1210 * Unfortunately we can't really let the stubs detach as we may need them to be 1211 * available for client operations. We may be able to improve this if we know 1212 * that the actual nexus is going away. However, as long as it's active, we need 1213 * all the stubs. 1214 */ 1215 int 1216 amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) 1217 { 1218 if (cmd == DDI_SUSPEND) { 1219 return (DDI_SUCCESS); 1220 } 1221 1222 return (DDI_FAILURE); 1223 } 1224 1225 int 1226 amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) 1227 { 1228 int *regs, reg; 1229 uint_t nregs, i; 1230 uint16_t vid, did; 1231 amdzen_stub_t *stub; 1232 amdzen_t *azn = amdzen_data; 1233 boolean_t valid = B_FALSE; 1234 boolean_t nb = B_FALSE; 1235 1236 if (cmd == DDI_RESUME) { 1237 return (DDI_SUCCESS); 1238 } else if (cmd != DDI_ATTACH) { 1239 return (DDI_FAILURE); 1240 } 1241 1242 /* 1243 * Make sure that the stub that we've been asked to attach is a pci type 1244 * device. If not, then there is no reason for us to proceed. 1245 */ 1246 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 1247 dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " 1248 "stub: %s", ddi_get_name(dip)); 1249 return (DDI_FAILURE); 1250 } 1251 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1252 "vendor-id", PCI_EINVAL16); 1253 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1254 "device-id", PCI_EINVAL16); 1255 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 1256 dev_err(dip, CE_WARN, "failed to get PCI ID properties"); 1257 return (DDI_FAILURE); 1258 } 1259 1260 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 1261 dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", 1262 cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? 1263 AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); 1264 return (DDI_FAILURE); 1265 } 1266 1267 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1268 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 1269 dev_err(dip, CE_WARN, "failed to get 'reg' property"); 1270 return (DDI_FAILURE); 1271 } 1272 1273 if (nregs == 0) { 1274 ddi_prop_free(regs); 1275 dev_err(dip, CE_WARN, "missing 'reg' property values"); 1276 return (DDI_FAILURE); 1277 } 1278 reg = *regs; 1279 ddi_prop_free(regs); 1280 1281 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 1282 if (amdzen_nb_ids[i] == did) { 1283 valid = B_TRUE; 1284 nb = B_TRUE; 1285 } 1286 } 1287 1288 if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && 1289 PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { 1290 valid = B_TRUE; 1291 nb = B_FALSE; 1292 } 1293 1294 if (!valid) { 1295 dev_err(dip, CE_WARN, "device %s didn't match the nexus list", 1296 ddi_get_name(dip)); 1297 return (DDI_FAILURE); 1298 } 1299 1300 stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); 1301 if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { 1302 dev_err(dip, CE_WARN, "failed to set up config space"); 1303 kmem_free(stub, sizeof (amdzen_stub_t)); 1304 return (DDI_FAILURE); 1305 } 1306 1307 stub->azns_dip = dip; 1308 stub->azns_vid = vid; 1309 stub->azns_did = did; 1310 stub->azns_bus = PCI_REG_BUS_G(reg); 1311 stub->azns_dev = PCI_REG_DEV_G(reg); 1312 stub->azns_func = PCI_REG_FUNC_G(reg); 1313 ddi_set_driver_private(dip, stub); 1314 1315 mutex_enter(&azn->azn_mutex); 1316 azn->azn_npresent++; 1317 if (nb) { 1318 list_insert_tail(&azn->azn_nb_stubs, stub); 1319 } else { 1320 list_insert_tail(&azn->azn_df_stubs, stub); 1321 } 1322 1323 if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && 1324 azn->azn_nscanned == azn->azn_npresent) { 1325 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 1326 azn->azn_taskqid = taskq_dispatch(system_taskq, 1327 amdzen_nexus_init, azn, TQ_SLEEP); 1328 } 1329 mutex_exit(&azn->azn_mutex); 1330 1331 return (DDI_SUCCESS); 1332 } 1333 1334 static int 1335 amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 1336 void *arg, void *result) 1337 { 1338 char buf[32]; 1339 dev_info_t *child; 1340 const amdzen_child_data_t *acd; 1341 1342 switch (ctlop) { 1343 case DDI_CTLOPS_REPORTDEV: 1344 if (rdip == NULL) { 1345 return (DDI_FAILURE); 1346 } 1347 cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", 1348 ddi_node_name(rdip), ddi_get_name_addr(rdip), 1349 ddi_driver_name(rdip), ddi_get_instance(rdip)); 1350 break; 1351 case DDI_CTLOPS_INITCHILD: 1352 child = arg; 1353 if (child == NULL) { 1354 dev_err(dip, CE_WARN, "!no child passed for " 1355 "DDI_CTLOPS_INITCHILD"); 1356 } 1357 1358 acd = ddi_get_parent_data(child); 1359 if (acd == NULL) { 1360 dev_err(dip, CE_WARN, "!missing child parent data"); 1361 return (DDI_FAILURE); 1362 } 1363 1364 if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= 1365 sizeof (buf)) { 1366 dev_err(dip, CE_WARN, "!failed to construct device " 1367 "addr due to overflow"); 1368 return (DDI_FAILURE); 1369 } 1370 1371 ddi_set_name_addr(child, buf); 1372 break; 1373 case DDI_CTLOPS_UNINITCHILD: 1374 child = arg; 1375 if (child == NULL) { 1376 dev_err(dip, CE_WARN, "!no child passed for " 1377 "DDI_CTLOPS_UNINITCHILD"); 1378 } 1379 1380 ddi_set_name_addr(child, NULL); 1381 break; 1382 default: 1383 return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 1384 } 1385 return (DDI_SUCCESS); 1386 } 1387 1388 static int 1389 amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 1390 { 1391 amdzen_t *azn = amdzen_data; 1392 1393 if (cmd == DDI_RESUME) { 1394 return (DDI_SUCCESS); 1395 } else if (cmd != DDI_ATTACH) { 1396 return (DDI_FAILURE); 1397 } 1398 1399 mutex_enter(&azn->azn_mutex); 1400 if (azn->azn_dip != NULL) { 1401 dev_err(dip, CE_WARN, "driver is already attached!"); 1402 mutex_exit(&azn->azn_mutex); 1403 return (DDI_FAILURE); 1404 } 1405 1406 azn->azn_dip = dip; 1407 azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, 1408 azn, TQ_SLEEP); 1409 azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; 1410 mutex_exit(&azn->azn_mutex); 1411 1412 return (DDI_SUCCESS); 1413 } 1414 1415 static int 1416 amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 1417 { 1418 amdzen_t *azn = amdzen_data; 1419 1420 if (cmd == DDI_SUSPEND) { 1421 return (DDI_SUCCESS); 1422 } else if (cmd != DDI_DETACH) { 1423 return (DDI_FAILURE); 1424 } 1425 1426 mutex_enter(&azn->azn_mutex); 1427 while (azn->azn_taskqid != TASKQID_INVALID) { 1428 cv_wait(&azn->azn_cv, &azn->azn_mutex); 1429 } 1430 1431 /* 1432 * If we've attached any stub drivers, e.g. this platform is important 1433 * for us, then we fail detach. 1434 */ 1435 if (!list_is_empty(&azn->azn_df_stubs) || 1436 !list_is_empty(&azn->azn_nb_stubs)) { 1437 mutex_exit(&azn->azn_mutex); 1438 return (DDI_FAILURE); 1439 } 1440 1441 azn->azn_dip = NULL; 1442 mutex_exit(&azn->azn_mutex); 1443 1444 return (DDI_SUCCESS); 1445 } 1446 1447 static void 1448 amdzen_free(void) 1449 { 1450 if (amdzen_data == NULL) { 1451 return; 1452 } 1453 1454 VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); 1455 list_destroy(&amdzen_data->azn_df_stubs); 1456 VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); 1457 list_destroy(&amdzen_data->azn_nb_stubs); 1458 cv_destroy(&amdzen_data->azn_cv); 1459 mutex_destroy(&amdzen_data->azn_mutex); 1460 kmem_free(amdzen_data, sizeof (amdzen_t)); 1461 amdzen_data = NULL; 1462 } 1463 1464 static void 1465 amdzen_alloc(void) 1466 { 1467 amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); 1468 mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); 1469 list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), 1470 offsetof(amdzen_stub_t, azns_link)); 1471 list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), 1472 offsetof(amdzen_stub_t, azns_link)); 1473 cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); 1474 } 1475 1476 struct bus_ops amdzen_bus_ops = { 1477 .busops_rev = BUSO_REV, 1478 .bus_map = nullbusmap, 1479 .bus_dma_map = ddi_no_dma_map, 1480 .bus_dma_allochdl = ddi_no_dma_allochdl, 1481 .bus_dma_freehdl = ddi_no_dma_freehdl, 1482 .bus_dma_bindhdl = ddi_no_dma_bindhdl, 1483 .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, 1484 .bus_dma_flush = ddi_no_dma_flush, 1485 .bus_dma_win = ddi_no_dma_win, 1486 .bus_dma_ctl = ddi_no_dma_mctl, 1487 .bus_prop_op = ddi_bus_prop_op, 1488 .bus_ctl = amdzen_bus_ctl 1489 }; 1490 1491 static struct dev_ops amdzen_dev_ops = { 1492 .devo_rev = DEVO_REV, 1493 .devo_refcnt = 0, 1494 .devo_getinfo = nodev, 1495 .devo_identify = nulldev, 1496 .devo_probe = nulldev, 1497 .devo_attach = amdzen_attach, 1498 .devo_detach = amdzen_detach, 1499 .devo_reset = nodev, 1500 .devo_quiesce = ddi_quiesce_not_needed, 1501 .devo_bus_ops = &amdzen_bus_ops 1502 }; 1503 1504 static struct modldrv amdzen_modldrv = { 1505 .drv_modops = &mod_driverops, 1506 .drv_linkinfo = "AMD Zen Nexus Driver", 1507 .drv_dev_ops = &amdzen_dev_ops 1508 }; 1509 1510 static struct modlinkage amdzen_modlinkage = { 1511 .ml_rev = MODREV_1, 1512 .ml_linkage = { &amdzen_modldrv, NULL } 1513 }; 1514 1515 int 1516 _init(void) 1517 { 1518 int ret; 1519 1520 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && 1521 cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { 1522 return (ENOTSUP); 1523 } 1524 1525 if ((ret = mod_install(&amdzen_modlinkage)) == 0) { 1526 amdzen_alloc(); 1527 } 1528 1529 return (ret); 1530 } 1531 1532 int 1533 _info(struct modinfo *modinfop) 1534 { 1535 return (mod_info(&amdzen_modlinkage, modinfop)); 1536 } 1537 1538 int 1539 _fini(void) 1540 { 1541 int ret; 1542 1543 if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { 1544 amdzen_free(); 1545 } 1546 1547 return (ret); 1548 } 1549