1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019, Joyent, Inc. 14 * Copyright 2022 Oxide Computer Company 15 */ 16 17 /* 18 * Nexus Driver for AMD Zen family systems. The purpose of this driver is to 19 * provide access to the following resources in a single, centralized fashion: 20 * 21 * - The per-chip Data Fabric 22 * - The North Bridge 23 * - The System Management Network (SMN) 24 * 25 * This is a nexus driver as once we have attached to all the requisite 26 * components, we will enumerate child devices which consume this functionality. 27 * 28 * ------------------------ 29 * Mapping Devices Together 30 * ------------------------ 31 * 32 * The operating system needs to expose things like temperature sensors and DRAM 33 * configuration registers in terms that are meaningful to the system such as 34 * logical CPUs, cores, etc. This driver attaches to the PCI IDs that represent 35 * the northbridge and data fabric; however, there are multiple PCI devices (one 36 * per die) that exist. This driver does manage to map all of these three things 37 * together; however, it requires some acrobatics. Unfortunately, there's no 38 * direct way to map a northbridge to its corresponding die. However, we can map 39 * a CPU die to a data fabric PCI device and a data fabric PCI device to a 40 * corresponding northbridge PCI device. 41 * 42 * In current Zen based products, there is a direct mapping between processor 43 * nodes and a data fabric PCI device. All of the devices are on PCI Bus 0 and 44 * start from Device 0x18. Device 0x18 maps to processor node 0, 0x19 to 45 * processor node 1, etc. This means that to map a logical CPU to a data fabric 46 * device, we take its processor node id, add it to 0x18 and find the PCI device 47 * that is on bus 0, device 0x18. As each data fabric device is attached based 48 * on its PCI ID, we add it to the global list, amd_nbdf_dfs that is in the 49 * amd_f17nbdf_t structure. 50 * 51 * The northbridge PCI device has a defined device and function, but the PCI bus 52 * that it's on can vary. Each die has its own series of PCI buses that are 53 * assigned to it and the northbridge PCI device is on the first of die-specific 54 * PCI bus for each die. This also means that the northbridge will not show up 55 * on PCI bus 0, which is the PCI bus that all of the data fabric devices are 56 * on. While conventionally the northbridge with the lowest PCI bus value 57 * would correspond to processor node zero, hardware does not guarantee that at 58 * all. Because we don't want to be at the mercy of firmware, we don't rely on 59 * this ordering, even though we have yet to find a system that deviates from 60 * this scheme. 61 * 62 * One of the registers in the data fabric device's function 0 63 * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to have the first PCI bus that is 64 * associated with the processor node. This means that we can map a data fabric 65 * device to a northbridge by finding the northbridge whose PCI bus matches the 66 * value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. 67 * 68 * We can map a northbridge to a data fabric device and a data fabric device to 69 * a die. Because these are generally 1:1 mappings, there is a transitive 70 * relationship and therefore we know which northbridge is associated with which 71 * processor die. This is summarized in the following image: 72 * 73 * +-------+ +-----------------------------------+ +--------------+ 74 * | Die 0 |--->| Data Fabric PCI BDF 0/18/0 |------->| Northbridge | 75 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | 76 * ... +-----------------------------------+ +--------------+ 77 * +-------+ +------------------------------------+ +--------------+ 78 * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |------->| Northbridge | 79 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | 80 * +------------------------------------+ +--------------+ 81 * 82 * Note, the PCI buses used by the northbridges here are arbitrary. They do not 83 * reflect the actual values by hardware; however, the bus/device/function (BDF) 84 * of the data fabric accurately models hardware. All of the BDF values are in 85 * hex. 86 * 87 * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), 88 * AMD has multiple northbridges that exist on a given die. All of these 89 * northbridges share the same data fabric and system management network port. 90 * From our perspective this means that some of the northbridge devices will be 91 * redundant and that we will no longer have a 1:1 mapping between the 92 * northbridge and the data fabric devices. Every data fabric will have a 93 * northbridge, but not every northbridge will have a data fabric device mapped. 94 * Because we're always trying to map from a die to a northbridge and not the 95 * reverse, the fact that there are extra northbridge devices hanging around 96 * that we don't know about shouldn't be a problem. 97 * 98 * ------------------------------- 99 * Attach and Detach Complications 100 * ------------------------------- 101 * 102 * Because we need to map different PCI devices together, this means that we 103 * have multiple dev_info_t structures that we need to manage. Each of these is 104 * independently attached and detached. While this is easily managed for attach, 105 * it is not for detach. Each of these devices is a 'stub'. 106 * 107 * Once a device has been detached it will only come back if we have an active 108 * minor node that will be accessed. This means that if they are detached, 109 * nothing would ever cause them to be reattached. The system also doesn't 110 * provide us a way or any guarantees around making sure that we're attached to 111 * all such devices before we detach. As a result, unfortunately, it's easier to 112 * basically have detach always fail. 113 * 114 * --------------- 115 * Exposed Devices 116 * --------------- 117 * 118 * Rather than try and have all of the different functions that could be 119 * provided by one driver, we instead have created a nexus driver that will 120 * itself try and load children. Children are all pseudo-device drivers that 121 * provide different pieces of functionality that use this. 122 * 123 * ------- 124 * Locking 125 * ------- 126 * 127 * The amdzen_data structure contains a single lock, azn_mutex. The various 128 * client functions are intended for direct children of our nexus, but have been 129 * designed in case someone else depends on this driver despite not being a 130 * child. Once a DF has been discovered, the set of entities inside of it 131 * (adf_nents, adf_ents[]) is considered static, constant data. This means that 132 * iterating over it in and of itself does not require locking; however, the 133 * discovery of the amd_df_t does. In addition, whenever performing register 134 * accesses to the DF or SMN, those require locking. This means that one must 135 * hold the lock in the following circumstances: 136 * 137 * o Looking up DF structures 138 * o Reading or writing to DF registers 139 * o Reading or writing to SMN registers 140 * 141 * In general, it is preferred that the lock be held across an entire client 142 * operation if possible. The only time this becomes an issue are when we have 143 * callbacks into our callers (ala amdzen_c_df_iter()) as they will likely 144 * recursively call into us. 145 */ 146 147 #include <sys/modctl.h> 148 #include <sys/conf.h> 149 #include <sys/devops.h> 150 #include <sys/ddi.h> 151 #include <sys/sunddi.h> 152 #include <sys/pci.h> 153 #include <sys/sysmacros.h> 154 #include <sys/sunndi.h> 155 #include <sys/x86_archext.h> 156 #include <sys/cpuvar.h> 157 158 #include <sys/amdzen/df.h> 159 #include "amdzen_client.h" 160 #include "amdzen.h" 161 162 amdzen_t *amdzen_data; 163 164 /* 165 * Array of northbridge IDs that we care about. 166 */ 167 static const uint16_t amdzen_nb_ids[] = { 168 /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 169 0x1450, 170 /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 171 0x15d0, 172 /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 173 0x1480, 174 /* Family 17h/19h Renoir, Cezanne Zen 2/3 uarch) */ 175 0x1630 176 }; 177 178 typedef struct { 179 char *acd_name; 180 amdzen_child_t acd_addr; 181 } amdzen_child_data_t; 182 183 static const amdzen_child_data_t amdzen_children[] = { 184 { "smntemp", AMDZEN_C_SMNTEMP }, 185 { "usmn", AMDZEN_C_USMN }, 186 { "zen_udf", AMDZEN_C_ZEN_UDF }, 187 { "zen_umc", AMDZEN_C_ZEN_UMC } 188 }; 189 190 static uint32_t 191 amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) 192 { 193 return (pci_config_get32(stub->azns_cfgspace, reg)); 194 } 195 196 static uint64_t 197 amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) 198 { 199 return (pci_config_get64(stub->azns_cfgspace, reg)); 200 } 201 202 static void 203 amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) 204 { 205 pci_config_put32(stub->azns_cfgspace, reg, val); 206 } 207 208 static uint64_t 209 amdzen_df_read_regdef(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def, 210 uint8_t inst, boolean_t do_64) 211 { 212 df_reg_def_t ficaa; 213 df_reg_def_t ficad; 214 uint32_t val = 0; 215 df_rev_t df_rev = azn->azn_dfs[0].adf_rev; 216 217 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 218 ASSERT3U(def.drd_gens & df_rev, ==, df_rev); 219 val = DF_FICAA_V2_SET_TARG_INST(val, 1); 220 val = DF_FICAA_V2_SET_FUNC(val, def.drd_func); 221 val = DF_FICAA_V2_SET_INST(val, inst); 222 val = DF_FICAA_V2_SET_64B(val, do_64 ? 1 : 0); 223 224 switch (df_rev) { 225 case DF_REV_2: 226 case DF_REV_3: 227 case DF_REV_3P5: 228 ficaa = DF_FICAA_V2; 229 ficad = DF_FICAD_LO_V2; 230 /* 231 * Both here and in the DFv4 case, the register ignores the 232 * lower 2 bits. That is we can only address and encode things 233 * in units of 4 bytes. 234 */ 235 val = DF_FICAA_V2_SET_REG(val, def.drd_reg >> 2); 236 break; 237 case DF_REV_4: 238 ficaa = DF_FICAA_V4; 239 ficad = DF_FICAD_LO_V4; 240 val = DF_FICAA_V4_SET_REG(val, def.drd_reg >> 2); 241 break; 242 default: 243 panic("encountered unexpected DF rev: %u", df_rev); 244 } 245 246 amdzen_stub_put32(df->adf_funcs[ficaa.drd_func], ficaa.drd_reg, val); 247 if (do_64) { 248 return (amdzen_stub_get64(df->adf_funcs[ficad.drd_func], 249 ficad.drd_reg)); 250 } else { 251 return (amdzen_stub_get32(df->adf_funcs[ficad.drd_func], 252 ficad.drd_reg)); 253 } 254 } 255 256 /* 257 * Perform a targeted 32-bit indirect read to a specific instance and function. 258 */ 259 static uint32_t 260 amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, 261 const df_reg_def_t def) 262 { 263 return (amdzen_df_read_regdef(azn, df, def, inst, B_FALSE)); 264 } 265 266 /* 267 * For a broadcast read, just go to the underlying PCI function and perform a 268 * read. At this point in time, we don't believe we need to use the FICAA/FICAD 269 * to access it (though it does have a broadcast mode). 270 */ 271 static uint32_t 272 amdzen_df_read32_bcast(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def) 273 { 274 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 275 return (amdzen_stub_get32(df->adf_funcs[def.drd_func], def.drd_reg)); 276 } 277 278 279 static uint32_t 280 amdzen_smn_read32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg) 281 { 282 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 283 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, reg); 284 return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); 285 } 286 287 static void 288 amdzen_smn_write32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg, uint32_t val) 289 { 290 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 291 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, reg); 292 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val); 293 } 294 295 static amdzen_df_t * 296 amdzen_df_find(amdzen_t *azn, uint_t dfno) 297 { 298 uint_t i; 299 300 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 301 if (dfno >= azn->azn_ndfs) { 302 return (NULL); 303 } 304 305 for (i = 0; i < azn->azn_ndfs; i++) { 306 amdzen_df_t *df = &azn->azn_dfs[i]; 307 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { 308 continue; 309 } 310 311 if (dfno == 0) { 312 return (df); 313 } 314 dfno--; 315 } 316 317 return (NULL); 318 } 319 320 /* 321 * Client functions that are used by nexus children. 322 */ 323 int 324 amdzen_c_smn_read32(uint_t dfno, uint32_t reg, uint32_t *valp) 325 { 326 amdzen_df_t *df; 327 amdzen_t *azn = amdzen_data; 328 329 mutex_enter(&azn->azn_mutex); 330 df = amdzen_df_find(azn, dfno); 331 if (df == NULL) { 332 mutex_exit(&azn->azn_mutex); 333 return (ENOENT); 334 } 335 336 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 337 mutex_exit(&azn->azn_mutex); 338 return (ENXIO); 339 } 340 341 *valp = amdzen_smn_read32(azn, df, reg); 342 mutex_exit(&azn->azn_mutex); 343 return (0); 344 } 345 346 int 347 amdzen_c_smn_write32(uint_t dfno, uint32_t reg, uint32_t val) 348 { 349 amdzen_df_t *df; 350 amdzen_t *azn = amdzen_data; 351 352 mutex_enter(&azn->azn_mutex); 353 df = amdzen_df_find(azn, dfno); 354 if (df == NULL) { 355 mutex_exit(&azn->azn_mutex); 356 return (ENOENT); 357 } 358 359 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 360 mutex_exit(&azn->azn_mutex); 361 return (ENXIO); 362 } 363 364 amdzen_smn_write32(azn, df, reg, val); 365 mutex_exit(&azn->azn_mutex); 366 return (0); 367 } 368 369 370 uint_t 371 amdzen_c_df_count(void) 372 { 373 uint_t ret; 374 amdzen_t *azn = amdzen_data; 375 376 mutex_enter(&azn->azn_mutex); 377 ret = azn->azn_ndfs; 378 mutex_exit(&azn->azn_mutex); 379 return (ret); 380 } 381 382 df_rev_t 383 amdzen_c_df_rev(void) 384 { 385 amdzen_df_t *df; 386 amdzen_t *azn = amdzen_data; 387 df_rev_t rev; 388 389 /* 390 * Always use the first DF instance to determine what we're using. Our 391 * current assumption, which seems to generally be true, is that the 392 * given DF revisions are the same in a given system when the DFs are 393 * directly connected. 394 */ 395 mutex_enter(&azn->azn_mutex); 396 df = amdzen_df_find(azn, 0); 397 if (df == NULL) { 398 rev = DF_REV_UNKNOWN; 399 } else { 400 rev = df->adf_rev; 401 } 402 mutex_exit(&azn->azn_mutex); 403 404 return (rev); 405 } 406 407 int 408 amdzen_c_df_read32(uint_t dfno, uint8_t inst, const df_reg_def_t def, 409 uint32_t *valp) 410 { 411 amdzen_df_t *df; 412 amdzen_t *azn = amdzen_data; 413 414 mutex_enter(&azn->azn_mutex); 415 df = amdzen_df_find(azn, dfno); 416 if (df == NULL) { 417 mutex_exit(&azn->azn_mutex); 418 return (ENOENT); 419 } 420 421 *valp = amdzen_df_read_regdef(azn, df, def, inst, B_FALSE); 422 mutex_exit(&azn->azn_mutex); 423 424 return (0); 425 } 426 427 int 428 amdzen_c_df_read64(uint_t dfno, uint8_t inst, const df_reg_def_t def, 429 uint64_t *valp) 430 { 431 amdzen_df_t *df; 432 amdzen_t *azn = amdzen_data; 433 434 mutex_enter(&azn->azn_mutex); 435 df = amdzen_df_find(azn, dfno); 436 if (df == NULL) { 437 mutex_exit(&azn->azn_mutex); 438 return (ENOENT); 439 } 440 441 *valp = amdzen_df_read_regdef(azn, df, def, inst, B_TRUE); 442 mutex_exit(&azn->azn_mutex); 443 444 return (0); 445 } 446 447 int 448 amdzen_c_df_iter(uint_t dfno, zen_df_type_t type, amdzen_c_iter_f func, 449 void *arg) 450 { 451 amdzen_df_t *df; 452 amdzen_t *azn = amdzen_data; 453 df_type_t df_type; 454 uint8_t df_subtype; 455 456 /* 457 * Unlike other calls here, we hold our lock only to find the DF here. 458 * The main reason for this is the nature of the callback function. 459 * Folks are iterating over instances so they can call back into us. If 460 * you look at the locking statement, the thing that is most volatile 461 * right here and what we need to protect is the DF itself and 462 * subsequent register accesses to it. The actual data about which 463 * entities exist is static and so once we have found a DF we should 464 * hopefully be in good shape as they only come, but don't go. 465 */ 466 mutex_enter(&azn->azn_mutex); 467 df = amdzen_df_find(azn, dfno); 468 if (df == NULL) { 469 mutex_exit(&azn->azn_mutex); 470 return (ENOENT); 471 } 472 mutex_exit(&azn->azn_mutex); 473 474 switch (type) { 475 case ZEN_DF_TYPE_CS_UMC: 476 df_type = DF_TYPE_CS; 477 /* 478 * In the original Zeppelin DFv2 die there was no subtype field 479 * used for the CS. The UMC is the only type and has a subtype 480 * of zero. 481 */ 482 if (df->adf_rev != DF_REV_2) { 483 df_subtype = DF_CS_SUBTYPE_UMC; 484 } else { 485 df_subtype = 0; 486 } 487 break; 488 case ZEN_DF_TYPE_CCM_CPU: 489 df_type = DF_TYPE_CCM; 490 /* 491 * In the Genoa/DFv4 timeframe, with the introduction of CXL and 492 * related, a subtype was added here where as previously it was 493 * always zero. 494 */ 495 if (df->adf_major >= 4) { 496 df_subtype = DF_CCM_SUBTYPE_CPU; 497 } else { 498 df_subtype = 0; 499 } 500 break; 501 default: 502 return (EINVAL); 503 } 504 505 for (uint_t i = 0; i < df->adf_nents; i++) { 506 amdzen_df_ent_t *ent = &df->adf_ents[i]; 507 508 /* 509 * Some DF components are not considered enabled and therefore 510 * will end up having bogus values in their ID fields. If we do 511 * not have an enable flag set, we must skip this node. 512 */ 513 if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 514 continue; 515 516 if (ent->adfe_type == df_type && 517 ent->adfe_subtype == df_subtype) { 518 int ret = func(dfno, ent->adfe_fabric_id, 519 ent->adfe_inst_id, arg); 520 if (ret != 0) { 521 return (ret); 522 } 523 } 524 } 525 526 return (0); 527 } 528 529 int 530 amdzen_c_df_fabric_decomp(df_fabric_decomp_t *decomp) 531 { 532 const amdzen_df_t *df; 533 amdzen_t *azn = amdzen_data; 534 535 mutex_enter(&azn->azn_mutex); 536 df = amdzen_df_find(azn, 0); 537 if (df == NULL) { 538 mutex_exit(&azn->azn_mutex); 539 return (ENOENT); 540 } 541 542 *decomp = df->adf_decomp; 543 mutex_exit(&azn->azn_mutex); 544 return (0); 545 } 546 547 static boolean_t 548 amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) 549 { 550 int ret; 551 dev_info_t *child; 552 553 if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, 554 (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { 555 dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " 556 "dip for %s", acd->acd_name); 557 return (B_FALSE); 558 } 559 560 ddi_set_parent_data(child, (void *)acd); 561 if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { 562 dev_err(azn->azn_dip, CE_WARN, "!failed to online child " 563 "dip %s: %d", acd->acd_name, ret); 564 return (B_FALSE); 565 } 566 567 return (B_TRUE); 568 } 569 570 static boolean_t 571 amdzen_map_dfs(amdzen_t *azn) 572 { 573 amdzen_stub_t *stub; 574 575 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 576 577 for (stub = list_head(&azn->azn_df_stubs); stub != NULL; 578 stub = list_next(&azn->azn_df_stubs, stub)) { 579 amdzen_df_t *df; 580 uint_t dfno; 581 582 dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; 583 if (dfno > AMDZEN_MAX_DFS) { 584 dev_err(stub->azns_dip, CE_WARN, "encountered df " 585 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 586 stub->azns_bus, stub->azns_dev, stub->azns_func); 587 goto err; 588 } 589 590 df = &azn->azn_dfs[dfno]; 591 592 if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { 593 dev_err(stub->azns_dip, CE_WARN, "encountered df " 594 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 595 stub->azns_bus, stub->azns_dev, stub->azns_func); 596 goto err; 597 } 598 599 if (df->adf_funcs[stub->azns_func] != NULL) { 600 dev_err(stub->azns_dip, CE_WARN, "encountered " 601 "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", 602 stub->azns_bus, stub->azns_dev, stub->azns_func); 603 goto err; 604 } 605 df->adf_funcs[stub->azns_func] = stub; 606 } 607 608 return (B_TRUE); 609 610 err: 611 azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; 612 return (B_FALSE); 613 } 614 615 static boolean_t 616 amdzen_check_dfs(amdzen_t *azn) 617 { 618 uint_t i; 619 boolean_t ret = B_TRUE; 620 621 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 622 amdzen_df_t *df = &azn->azn_dfs[i]; 623 uint_t count = 0; 624 625 /* 626 * We require all platforms to have DFs functions 0-6. Not all 627 * platforms have DF function 7. 628 */ 629 for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { 630 if (df->adf_funcs[func] != NULL) { 631 count++; 632 } 633 } 634 635 if (count == 0) 636 continue; 637 638 if (count != 7) { 639 ret = B_FALSE; 640 dev_err(azn->azn_dip, CE_WARN, "df %u devices " 641 "incomplete", i); 642 } else { 643 df->adf_flags |= AMDZEN_DF_F_VALID; 644 azn->azn_ndfs++; 645 } 646 } 647 648 return (ret); 649 } 650 651 static const uint8_t amdzen_df_rome_ids[0x2b] = { 652 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 653 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 654 44, 45, 46, 47, 48 655 }; 656 657 /* 658 * Check the first df entry to see if it belongs to Rome or Milan. If so, then 659 * it uses the disjoint ID space. 660 */ 661 static boolean_t 662 amdzen_is_rome_style(uint_t id) 663 { 664 return (id == 0x1490 || id == 0x1650); 665 } 666 667 /* 668 * To be able to do most other things we want to do, we must first determine 669 * what revision of the DF (data fabric) that we're using. 670 * 671 * Snapshot the df version. This was added explicitly in DFv4.0, around the Zen 672 * 4 timeframe and allows us to tell apart different version of the DF register 673 * set, most usefully when various subtypes were added. 674 * 675 * Older versions can theoretically be told apart based on usage of reserved 676 * registers. We walk these in the following order, starting with the newest rev 677 * and walking backwards to tell things apart: 678 * 679 * o v3.5 -> Check function 1, register 0x150. This was reserved prior 680 * to this point. This is actually DF_FIDMASK0_V3P5. We are supposed 681 * to check bits [7:0]. 682 * 683 * o v3.0 -> Check function 1, register 0x208. The low byte (7:0) was 684 * changed to indicate a component mask. This is non-zero 685 * in the 3.0 generation. This is actually DF_FIDMASK_V2. 686 * 687 * o v2.0 -> This is just the not that case. Presumably v1 wasn't part 688 * of the Zen generation. 689 * 690 * Because we don't know what version we are yet, we do not use the normal 691 * versioned register accesses which would check what DF version we are and 692 * would want to use the normal indirect register accesses (which also require 693 * us to know the version). We instead do direct broadcast reads. 694 */ 695 static void 696 amdzen_determine_df_vers(amdzen_t *azn, amdzen_df_t *df) 697 { 698 uint32_t val; 699 df_reg_def_t rd = DF_FBICNT; 700 701 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 702 df->adf_major = DF_FBICNT_V4_GET_MAJOR(val); 703 df->adf_minor = DF_FBICNT_V4_GET_MINOR(val); 704 if (df->adf_major == 0 && df->adf_minor == 0) { 705 rd = DF_FIDMASK0_V3P5; 706 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 707 if (bitx32(val, 7, 0) != 0) { 708 df->adf_major = 3; 709 df->adf_minor = 5; 710 df->adf_rev = DF_REV_3P5; 711 } else { 712 rd = DF_FIDMASK_V2; 713 val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], 714 rd.drd_reg); 715 if (bitx32(val, 7, 0) != 0) { 716 df->adf_major = 3; 717 df->adf_minor = 0; 718 df->adf_rev = DF_REV_3; 719 } else { 720 df->adf_major = 2; 721 df->adf_minor = 0; 722 df->adf_rev = DF_REV_2; 723 } 724 } 725 } else if (df->adf_major == 4 && df->adf_minor == 0) { 726 df->adf_rev = DF_REV_4; 727 } else { 728 df->adf_rev = DF_REV_UNKNOWN; 729 } 730 } 731 732 /* 733 * All of the different versions of the DF have different ways of getting at and 734 * answering the question of how do I break a fabric ID into a corresponding 735 * socket, die, and component. Importantly the goal here is to obtain, cache, 736 * and normalize: 737 * 738 * o The DF System Configuration 739 * o The various Mask registers 740 * o The Node ID 741 */ 742 static void 743 amdzen_determine_fabric_decomp(amdzen_t *azn, amdzen_df_t *df) 744 { 745 uint32_t mask; 746 df_fabric_decomp_t *decomp = &df->adf_decomp; 747 748 switch (df->adf_rev) { 749 case DF_REV_2: 750 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V2); 751 switch (DF_SYSCFG_V2_GET_MY_TYPE(df->adf_syscfg)) { 752 case DF_DIE_TYPE_CPU: 753 mask = amdzen_df_read32_bcast(azn, df, 754 DF_DIEMASK_CPU_V2); 755 break; 756 case DF_DIE_TYPE_APU: 757 mask = amdzen_df_read32_bcast(azn, df, 758 DF_DIEMASK_APU_V2); 759 break; 760 default: 761 panic("DF thinks we're not on a CPU!"); 762 } 763 df->adf_mask0 = mask; 764 765 /* 766 * DFv2 is a bit different in how the fabric mask register is 767 * phrased. Logically a fabric ID is broken into something that 768 * uniquely identifies a "node" (a particular die on a socket) 769 * and something that identifies a "component", e.g. a memory 770 * controller. 771 * 772 * Starting with DFv3, these registers logically called out how 773 * to separate the fabric ID first into a node and a component. 774 * Then the node was then broken down into a socket and die. In 775 * DFv2, there is no separate mask and shift of a node. Instead 776 * the socket and die are absolute offsets into the fabric ID 777 * rather than relative offsets into the node ID. As such, when 778 * we encounter DFv2, we fake up a node mask and shift and make 779 * it look like DFv3+. 780 */ 781 decomp->dfd_node_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) | 782 DF_DIEMASK_V2_GET_DIE_MASK(mask); 783 decomp->dfd_node_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask); 784 decomp->dfd_comp_mask = DF_DIEMASK_V2_GET_COMP_MASK(mask); 785 decomp->dfd_comp_shift = 0; 786 787 decomp->dfd_sock_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) >> 788 decomp->dfd_node_shift; 789 decomp->dfd_die_mask = DF_DIEMASK_V2_GET_DIE_MASK(mask) >> 790 decomp->dfd_node_shift; 791 decomp->dfd_sock_shift = DF_DIEMASK_V2_GET_SOCK_SHIFT(mask) - 792 decomp->dfd_node_shift; 793 decomp->dfd_die_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask) - 794 decomp->dfd_node_shift; 795 ASSERT3U(decomp->dfd_die_shift, ==, 0); 796 break; 797 case DF_REV_3: 798 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3); 799 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 800 DF_FIDMASK0_V3); 801 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 802 DF_FIDMASK1_V3); 803 804 decomp->dfd_sock_mask = 805 DF_FIDMASK1_V3_GET_SOCK_MASK(df->adf_mask1); 806 decomp->dfd_sock_shift = 807 DF_FIDMASK1_V3_GET_SOCK_SHIFT(df->adf_mask1); 808 decomp->dfd_die_mask = 809 DF_FIDMASK1_V3_GET_DIE_MASK(df->adf_mask1); 810 decomp->dfd_die_shift = 0; 811 decomp->dfd_node_mask = 812 DF_FIDMASK0_V3_GET_NODE_MASK(df->adf_mask0); 813 decomp->dfd_node_shift = 814 DF_FIDMASK1_V3_GET_NODE_SHIFT(df->adf_mask1); 815 decomp->dfd_comp_mask = 816 DF_FIDMASK0_V3_GET_COMP_MASK(df->adf_mask0); 817 decomp->dfd_comp_shift = 0; 818 break; 819 case DF_REV_3P5: 820 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, 821 DF_SYSCFG_V3P5); 822 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 823 DF_FIDMASK0_V3P5); 824 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 825 DF_FIDMASK1_V3P5); 826 df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 827 DF_FIDMASK2_V3P5); 828 829 decomp->dfd_sock_mask = 830 DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 831 decomp->dfd_sock_shift = 832 DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 833 decomp->dfd_die_mask = 834 DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 835 decomp->dfd_die_shift = 0; 836 decomp->dfd_node_mask = 837 DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 838 decomp->dfd_node_shift = 839 DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 840 decomp->dfd_comp_mask = 841 DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 842 decomp->dfd_comp_shift = 0; 843 break; 844 case DF_REV_4: 845 df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V4); 846 df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 847 DF_FIDMASK0_V4); 848 df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 849 DF_FIDMASK1_V4); 850 df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 851 DF_FIDMASK2_V4); 852 853 /* 854 * The DFv4 registers are at a different location in the DF; 855 * however, the actual layout of fields is the same as DFv3.5. 856 * This is why you see V3P5 below. 857 */ 858 decomp->dfd_sock_mask = 859 DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 860 decomp->dfd_sock_shift = 861 DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 862 decomp->dfd_die_mask = 863 DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 864 decomp->dfd_die_shift = 0; 865 decomp->dfd_node_mask = 866 DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 867 decomp->dfd_node_shift = 868 DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 869 decomp->dfd_comp_mask = 870 DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 871 decomp->dfd_comp_shift = 0; 872 break; 873 default: 874 panic("encountered suspicious, previously rejected DF " 875 "rev: 0x%x", df->adf_rev); 876 } 877 } 878 879 /* 880 * Initialize our knowledge about a given series of nodes on the data fabric. 881 */ 882 static void 883 amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) 884 { 885 uint_t i; 886 uint32_t val; 887 888 amdzen_determine_df_vers(azn, df); 889 890 switch (df->adf_rev) { 891 case DF_REV_2: 892 case DF_REV_3: 893 case DF_REV_3P5: 894 val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V2); 895 break; 896 case DF_REV_4: 897 val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V4); 898 break; 899 default: 900 dev_err(azn->azn_dip, CE_WARN, "encountered unsupported DF " 901 "revision: 0x%x", df->adf_rev); 902 return; 903 } 904 df->adf_nb_busno = DF_CFG_ADDR_CTL_GET_BUS_NUM(val); 905 val = amdzen_df_read32_bcast(azn, df, DF_FBICNT); 906 df->adf_nents = DF_FBICNT_GET_COUNT(val); 907 if (df->adf_nents == 0) 908 return; 909 df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, 910 KM_SLEEP); 911 912 for (i = 0; i < df->adf_nents; i++) { 913 amdzen_df_ent_t *dfe = &df->adf_ents[i]; 914 uint8_t inst = i; 915 916 /* 917 * Unfortunately, Rome uses a discontinuous instance ID pattern 918 * while everything else we can find uses a contiguous instance 919 * ID pattern. This means that for Rome, we need to adjust the 920 * indexes that we iterate over, though the total number of 921 * entries is right. This was carried over into Milan, but not 922 * Genoa. 923 */ 924 if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { 925 if (inst > ARRAY_SIZE(amdzen_df_rome_ids)) { 926 dev_err(azn->azn_dip, CE_WARN, "Rome family " 927 "processor reported more ids than the PPR, " 928 "resetting %u to instance zero", inst); 929 inst = 0; 930 } else { 931 inst = amdzen_df_rome_ids[inst]; 932 } 933 } 934 935 dfe->adfe_drvid = inst; 936 dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, DF_FBIINFO0); 937 dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, DF_FBIINFO1); 938 dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, DF_FBIINFO2); 939 dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, DF_FBIINFO3); 940 941 dfe->adfe_type = DF_FBIINFO0_GET_TYPE(dfe->adfe_info0); 942 dfe->adfe_subtype = DF_FBIINFO0_GET_SUBTYPE(dfe->adfe_info0); 943 944 /* 945 * The enabled flag was not present in Zen 1. Simulate it by 946 * checking for a non-zero register instead. 947 */ 948 if (DF_FBIINFO0_V3_GET_ENABLED(dfe->adfe_info0) || 949 (df->adf_rev == DF_REV_2 && dfe->adfe_info0 != 0)) { 950 dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; 951 } 952 if (DF_FBIINFO0_GET_HAS_MCA(dfe->adfe_info0)) { 953 dfe->adfe_flags |= AMDZEN_DFE_F_MCA; 954 } 955 dfe->adfe_inst_id = DF_FBIINFO3_GET_INSTID(dfe->adfe_info3); 956 switch (df->adf_rev) { 957 case DF_REV_2: 958 dfe->adfe_fabric_id = 959 DF_FBIINFO3_V2_GET_BLOCKID(dfe->adfe_info3); 960 break; 961 case DF_REV_3: 962 dfe->adfe_fabric_id = 963 DF_FBIINFO3_V3_GET_BLOCKID(dfe->adfe_info3); 964 break; 965 case DF_REV_3P5: 966 dfe->adfe_fabric_id = 967 DF_FBIINFO3_V3P5_GET_BLOCKID(dfe->adfe_info3); 968 break; 969 case DF_REV_4: 970 dfe->adfe_fabric_id = 971 DF_FBIINFO3_V4_GET_BLOCKID(dfe->adfe_info3); 972 break; 973 default: 974 panic("encountered suspicious, previously rejected DF " 975 "rev: 0x%x", df->adf_rev); 976 } 977 } 978 979 amdzen_determine_fabric_decomp(azn, df); 980 } 981 982 static void 983 amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) 984 { 985 amdzen_stub_t *stub; 986 987 for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; 988 stub = list_next(&azn->azn_nb_stubs, stub)) { 989 if (stub->azns_bus == df->adf_nb_busno) { 990 df->adf_flags |= AMDZEN_DF_F_FOUND_NB; 991 df->adf_nb = stub; 992 return; 993 } 994 } 995 } 996 997 static void 998 amdzen_nexus_init(void *arg) 999 { 1000 uint_t i; 1001 amdzen_t *azn = arg; 1002 1003 /* 1004 * First go through all of the stubs and assign the DF entries. 1005 */ 1006 mutex_enter(&azn->azn_mutex); 1007 if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { 1008 azn->azn_flags |= AMDZEN_F_MAP_ERROR; 1009 goto done; 1010 } 1011 1012 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 1013 amdzen_df_t *df = &azn->azn_dfs[i]; 1014 1015 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) 1016 continue; 1017 amdzen_setup_df(azn, df); 1018 amdzen_find_nb(azn, df); 1019 } 1020 1021 /* 1022 * Not all children may be installed. As such, we do not treat the 1023 * failure of a child as fatal to the driver. 1024 */ 1025 mutex_exit(&azn->azn_mutex); 1026 for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { 1027 (void) amdzen_create_child(azn, &amdzen_children[i]); 1028 } 1029 mutex_enter(&azn->azn_mutex); 1030 1031 done: 1032 azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; 1033 azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; 1034 azn->azn_taskqid = TASKQID_INVALID; 1035 cv_broadcast(&azn->azn_cv); 1036 mutex_exit(&azn->azn_mutex); 1037 } 1038 1039 static int 1040 amdzen_stub_scan_cb(dev_info_t *dip, void *arg) 1041 { 1042 amdzen_t *azn = arg; 1043 uint16_t vid, did; 1044 int *regs; 1045 uint_t nregs, i; 1046 boolean_t match = B_FALSE; 1047 1048 if (dip == ddi_root_node()) { 1049 return (DDI_WALK_CONTINUE); 1050 } 1051 1052 /* 1053 * If a node in question is not a pci node, then we have no interest in 1054 * it as all the stubs that we care about are related to pci devices. 1055 */ 1056 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 1057 return (DDI_WALK_PRUNECHILD); 1058 } 1059 1060 /* 1061 * If we can't get a device or vendor ID and prove that this is an AMD 1062 * part, then we don't care about it. 1063 */ 1064 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1065 "vendor-id", PCI_EINVAL16); 1066 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1067 "device-id", PCI_EINVAL16); 1068 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 1069 return (DDI_WALK_CONTINUE); 1070 } 1071 1072 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 1073 return (DDI_WALK_CONTINUE); 1074 } 1075 1076 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 1077 if (amdzen_nb_ids[i] == did) { 1078 match = B_TRUE; 1079 } 1080 } 1081 1082 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1083 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 1084 return (DDI_WALK_CONTINUE); 1085 } 1086 1087 if (nregs == 0) { 1088 ddi_prop_free(regs); 1089 return (DDI_WALK_CONTINUE); 1090 } 1091 1092 if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && 1093 PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { 1094 match = B_TRUE; 1095 } 1096 1097 ddi_prop_free(regs); 1098 if (match) { 1099 mutex_enter(&azn->azn_mutex); 1100 azn->azn_nscanned++; 1101 mutex_exit(&azn->azn_mutex); 1102 } 1103 1104 return (DDI_WALK_CONTINUE); 1105 } 1106 1107 static void 1108 amdzen_stub_scan(void *arg) 1109 { 1110 amdzen_t *azn = arg; 1111 1112 mutex_enter(&azn->azn_mutex); 1113 azn->azn_nscanned = 0; 1114 mutex_exit(&azn->azn_mutex); 1115 1116 ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); 1117 1118 mutex_enter(&azn->azn_mutex); 1119 azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; 1120 azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; 1121 1122 if (azn->azn_nscanned == 0) { 1123 azn->azn_flags |= AMDZEN_F_UNSUPPORTED; 1124 azn->azn_taskqid = TASKQID_INVALID; 1125 cv_broadcast(&azn->azn_cv); 1126 } else if (azn->azn_npresent == azn->azn_nscanned) { 1127 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 1128 azn->azn_taskqid = taskq_dispatch(system_taskq, 1129 amdzen_nexus_init, azn, TQ_SLEEP); 1130 } 1131 mutex_exit(&azn->azn_mutex); 1132 } 1133 1134 /* 1135 * Unfortunately we can't really let the stubs detach as we may need them to be 1136 * available for client operations. We may be able to improve this if we know 1137 * that the actual nexus is going away. However, as long as it's active, we need 1138 * all the stubs. 1139 */ 1140 int 1141 amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) 1142 { 1143 if (cmd == DDI_SUSPEND) { 1144 return (DDI_SUCCESS); 1145 } 1146 1147 return (DDI_FAILURE); 1148 } 1149 1150 int 1151 amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) 1152 { 1153 int *regs, reg; 1154 uint_t nregs, i; 1155 uint16_t vid, did; 1156 amdzen_stub_t *stub; 1157 amdzen_t *azn = amdzen_data; 1158 boolean_t valid = B_FALSE; 1159 boolean_t nb = B_FALSE; 1160 1161 if (cmd == DDI_RESUME) { 1162 return (DDI_SUCCESS); 1163 } else if (cmd != DDI_ATTACH) { 1164 return (DDI_FAILURE); 1165 } 1166 1167 /* 1168 * Make sure that the stub that we've been asked to attach is a pci type 1169 * device. If not, then there is no reason for us to proceed. 1170 */ 1171 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 1172 dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " 1173 "stub: %s", ddi_get_name(dip)); 1174 return (DDI_FAILURE); 1175 } 1176 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1177 "vendor-id", PCI_EINVAL16); 1178 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1179 "device-id", PCI_EINVAL16); 1180 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 1181 dev_err(dip, CE_WARN, "failed to get PCI ID properties"); 1182 return (DDI_FAILURE); 1183 } 1184 1185 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 1186 dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", 1187 cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? 1188 AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); 1189 return (DDI_FAILURE); 1190 } 1191 1192 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1193 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 1194 dev_err(dip, CE_WARN, "failed to get 'reg' property"); 1195 return (DDI_FAILURE); 1196 } 1197 1198 if (nregs == 0) { 1199 ddi_prop_free(regs); 1200 dev_err(dip, CE_WARN, "missing 'reg' property values"); 1201 return (DDI_FAILURE); 1202 } 1203 reg = *regs; 1204 ddi_prop_free(regs); 1205 1206 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 1207 if (amdzen_nb_ids[i] == did) { 1208 valid = B_TRUE; 1209 nb = B_TRUE; 1210 } 1211 } 1212 1213 if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && 1214 PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { 1215 valid = B_TRUE; 1216 nb = B_FALSE; 1217 } 1218 1219 if (!valid) { 1220 dev_err(dip, CE_WARN, "device %s didn't match the nexus list", 1221 ddi_get_name(dip)); 1222 return (DDI_FAILURE); 1223 } 1224 1225 stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); 1226 if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { 1227 dev_err(dip, CE_WARN, "failed to set up config space"); 1228 kmem_free(stub, sizeof (amdzen_stub_t)); 1229 return (DDI_FAILURE); 1230 } 1231 1232 stub->azns_dip = dip; 1233 stub->azns_vid = vid; 1234 stub->azns_did = did; 1235 stub->azns_bus = PCI_REG_BUS_G(reg); 1236 stub->azns_dev = PCI_REG_DEV_G(reg); 1237 stub->azns_func = PCI_REG_FUNC_G(reg); 1238 ddi_set_driver_private(dip, stub); 1239 1240 mutex_enter(&azn->azn_mutex); 1241 azn->azn_npresent++; 1242 if (nb) { 1243 list_insert_tail(&azn->azn_nb_stubs, stub); 1244 } else { 1245 list_insert_tail(&azn->azn_df_stubs, stub); 1246 } 1247 1248 if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && 1249 azn->azn_nscanned == azn->azn_npresent) { 1250 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 1251 azn->azn_taskqid = taskq_dispatch(system_taskq, 1252 amdzen_nexus_init, azn, TQ_SLEEP); 1253 } 1254 mutex_exit(&azn->azn_mutex); 1255 1256 return (DDI_SUCCESS); 1257 } 1258 1259 static int 1260 amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 1261 void *arg, void *result) 1262 { 1263 char buf[32]; 1264 dev_info_t *child; 1265 const amdzen_child_data_t *acd; 1266 1267 switch (ctlop) { 1268 case DDI_CTLOPS_REPORTDEV: 1269 if (rdip == NULL) { 1270 return (DDI_FAILURE); 1271 } 1272 cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", 1273 ddi_node_name(rdip), ddi_get_name_addr(rdip), 1274 ddi_driver_name(rdip), ddi_get_instance(rdip)); 1275 break; 1276 case DDI_CTLOPS_INITCHILD: 1277 child = arg; 1278 if (child == NULL) { 1279 dev_err(dip, CE_WARN, "!no child passed for " 1280 "DDI_CTLOPS_INITCHILD"); 1281 } 1282 1283 acd = ddi_get_parent_data(child); 1284 if (acd == NULL) { 1285 dev_err(dip, CE_WARN, "!missing child parent data"); 1286 return (DDI_FAILURE); 1287 } 1288 1289 if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= 1290 sizeof (buf)) { 1291 dev_err(dip, CE_WARN, "!failed to construct device " 1292 "addr due to overflow"); 1293 return (DDI_FAILURE); 1294 } 1295 1296 ddi_set_name_addr(child, buf); 1297 break; 1298 case DDI_CTLOPS_UNINITCHILD: 1299 child = arg; 1300 if (child == NULL) { 1301 dev_err(dip, CE_WARN, "!no child passed for " 1302 "DDI_CTLOPS_UNINITCHILD"); 1303 } 1304 1305 ddi_set_name_addr(child, NULL); 1306 break; 1307 default: 1308 return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 1309 } 1310 return (DDI_SUCCESS); 1311 } 1312 1313 static int 1314 amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 1315 { 1316 amdzen_t *azn = amdzen_data; 1317 1318 if (cmd == DDI_RESUME) { 1319 return (DDI_SUCCESS); 1320 } else if (cmd != DDI_ATTACH) { 1321 return (DDI_FAILURE); 1322 } 1323 1324 mutex_enter(&azn->azn_mutex); 1325 if (azn->azn_dip != NULL) { 1326 dev_err(dip, CE_WARN, "driver is already attached!"); 1327 mutex_exit(&azn->azn_mutex); 1328 return (DDI_FAILURE); 1329 } 1330 1331 azn->azn_dip = dip; 1332 azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, 1333 azn, TQ_SLEEP); 1334 azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; 1335 mutex_exit(&azn->azn_mutex); 1336 1337 return (DDI_SUCCESS); 1338 } 1339 1340 static int 1341 amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 1342 { 1343 amdzen_t *azn = amdzen_data; 1344 1345 if (cmd == DDI_SUSPEND) { 1346 return (DDI_SUCCESS); 1347 } else if (cmd != DDI_DETACH) { 1348 return (DDI_FAILURE); 1349 } 1350 1351 mutex_enter(&azn->azn_mutex); 1352 while (azn->azn_taskqid != TASKQID_INVALID) { 1353 cv_wait(&azn->azn_cv, &azn->azn_mutex); 1354 } 1355 1356 /* 1357 * If we've attached any stub drivers, e.g. this platform is important 1358 * for us, then we fail detach. 1359 */ 1360 if (!list_is_empty(&azn->azn_df_stubs) || 1361 !list_is_empty(&azn->azn_nb_stubs)) { 1362 mutex_exit(&azn->azn_mutex); 1363 return (DDI_FAILURE); 1364 } 1365 1366 azn->azn_dip = NULL; 1367 mutex_exit(&azn->azn_mutex); 1368 1369 return (DDI_SUCCESS); 1370 } 1371 1372 static void 1373 amdzen_free(void) 1374 { 1375 if (amdzen_data == NULL) { 1376 return; 1377 } 1378 1379 VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); 1380 list_destroy(&amdzen_data->azn_df_stubs); 1381 VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); 1382 list_destroy(&amdzen_data->azn_nb_stubs); 1383 cv_destroy(&amdzen_data->azn_cv); 1384 mutex_destroy(&amdzen_data->azn_mutex); 1385 kmem_free(amdzen_data, sizeof (amdzen_t)); 1386 amdzen_data = NULL; 1387 } 1388 1389 static void 1390 amdzen_alloc(void) 1391 { 1392 amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); 1393 mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); 1394 list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), 1395 offsetof(amdzen_stub_t, azns_link)); 1396 list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), 1397 offsetof(amdzen_stub_t, azns_link)); 1398 cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); 1399 } 1400 1401 struct bus_ops amdzen_bus_ops = { 1402 .busops_rev = BUSO_REV, 1403 .bus_map = nullbusmap, 1404 .bus_dma_map = ddi_no_dma_map, 1405 .bus_dma_allochdl = ddi_no_dma_allochdl, 1406 .bus_dma_freehdl = ddi_no_dma_freehdl, 1407 .bus_dma_bindhdl = ddi_no_dma_bindhdl, 1408 .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, 1409 .bus_dma_flush = ddi_no_dma_flush, 1410 .bus_dma_win = ddi_no_dma_win, 1411 .bus_dma_ctl = ddi_no_dma_mctl, 1412 .bus_prop_op = ddi_bus_prop_op, 1413 .bus_ctl = amdzen_bus_ctl 1414 }; 1415 1416 static struct dev_ops amdzen_dev_ops = { 1417 .devo_rev = DEVO_REV, 1418 .devo_refcnt = 0, 1419 .devo_getinfo = nodev, 1420 .devo_identify = nulldev, 1421 .devo_probe = nulldev, 1422 .devo_attach = amdzen_attach, 1423 .devo_detach = amdzen_detach, 1424 .devo_reset = nodev, 1425 .devo_quiesce = ddi_quiesce_not_needed, 1426 .devo_bus_ops = &amdzen_bus_ops 1427 }; 1428 1429 static struct modldrv amdzen_modldrv = { 1430 .drv_modops = &mod_driverops, 1431 .drv_linkinfo = "AMD Zen Nexus Driver", 1432 .drv_dev_ops = &amdzen_dev_ops 1433 }; 1434 1435 static struct modlinkage amdzen_modlinkage = { 1436 .ml_rev = MODREV_1, 1437 .ml_linkage = { &amdzen_modldrv, NULL } 1438 }; 1439 1440 int 1441 _init(void) 1442 { 1443 int ret; 1444 1445 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && 1446 cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { 1447 return (ENOTSUP); 1448 } 1449 1450 if ((ret = mod_install(&amdzen_modlinkage)) == 0) { 1451 amdzen_alloc(); 1452 } 1453 1454 return (ret); 1455 } 1456 1457 int 1458 _info(struct modinfo *modinfop) 1459 { 1460 return (mod_info(&amdzen_modlinkage, modinfop)); 1461 } 1462 1463 int 1464 _fini(void) 1465 { 1466 int ret; 1467 1468 if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { 1469 amdzen_free(); 1470 } 1471 1472 return (ret); 1473 } 1474