1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019, Joyent, Inc. 14 * Copyright 2020 Oxide Computer Company 15 */ 16 17 /* 18 * Nexus Driver for AMD Zen family systems. The purpose of this driver is to 19 * provide access to the following resources in a single, centralized fashion: 20 * 21 * - The per-chip Data Fabric 22 * - The North Bridge 23 * - The System Management Network (SMN) 24 * 25 * This is a nexus driver as once we have attached to all the requisite 26 * components, we will enumerate child devices which consume this functionality. 27 * 28 * ------------------------ 29 * Mapping Devices Together 30 * ------------------------ 31 * 32 * The operating system needs to expose things like temperature sensors and DRAM 33 * configuration registers in terms that are meaningful to the system such as 34 * logical CPUs, cores, etc. This driver attaches to the PCI IDs that represent 35 * the northbridge and data fabric; however, there are multiple PCI devices (one 36 * per die) that exist. This driver does manage to map all of these three things 37 * together; however, it requires some acrobatics. Unfortunately, there's no 38 * direct way to map a northbridge to its corresponding die. However, we can map 39 * a CPU die to a data fabric PCI device and a data fabric PCI device to a 40 * corresponding northbridge PCI device. 41 * 42 * In current Zen based products, there is a direct mapping between processor 43 * nodes and a data fabric PCI device. All of the devices are on PCI Bus 0 and 44 * start from Device 0x18. Device 0x18 maps to processor node 0, 0x19 to 45 * processor node 1, etc. This means that to map a logical CPU to a data fabric 46 * device, we take its processor node id, add it to 0x18 and find the PCI device 47 * that is on bus 0, device 0x18. As each data fabric device is attached based 48 * on its PCI ID, we add it to the global list, amd_nbdf_dfs that is in the 49 * amd_f17nbdf_t structure. 50 * 51 * The northbridge PCI device has a defined device and function, but the PCI bus 52 * that it's on can vary. Each die has its own series of PCI buses that are 53 * assigned to it and the northbridge PCI device is on the first of die-specific 54 * PCI bus for each die. This also means that the northbridge will not show up 55 * on PCI bus 0, which is the PCI bus that all of the data fabric devices are 56 * on. While conventionally the northbridge with the lowest PCI bus value 57 * would correspond to processor node zero, hardware does not guarantee that at 58 * all. Because we don't want to be at the mercy of firmware, we don't rely on 59 * this ordering, even though we have yet to find a system that deviates from 60 * this scheme. 61 * 62 * One of the registers in the data fabric device's function 0 63 * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to have the first PCI bus that is 64 * associated with the processor node. This means that we can map a data fabric 65 * device to a northbridge by finding the northbridge whose PCI bus matches the 66 * value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. 67 * 68 * We can map a northbridge to a data fabric device and a data fabric device to 69 * a die. Because these are generally 1:1 mappings, there is a transitive 70 * relationship and therefore we know which northbridge is associated with which 71 * processor die. This is summarized in the following image: 72 * 73 * +-------+ +-----------------------------------+ +--------------+ 74 * | Die 0 |--->| Data Fabric PCI BDF 0/18/0 |------->| Northbridge | 75 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | 76 * ... +-----------------------------------+ +--------------+ 77 * +-------+ +------------------------------------+ +--------------+ 78 * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |------->| Northbridge | 79 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | 80 * +------------------------------------+ +--------------+ 81 * 82 * Note, the PCI buses used by the northbridges here are arbitrary. They do not 83 * reflect the actual values by hardware; however, the bus/device/function (BDF) 84 * of the data fabric accurately models hardware. All of the BDF values are in 85 * hex. 86 * 87 * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), 88 * AMD has multiple northbridges that exist on a given die. All of these 89 * northbridges share the same data fabric and system management network port. 90 * From our perspective this means that some of the northbridge devices will be 91 * redundant and that we will no longer have a 1:1 mapping between the 92 * northbridge and the data fabric devices. Every data fabric will have a 93 * northbridge, but not every northbridge will have a data fabric device mapped. 94 * Because we're always trying to map from a die to a northbridge and not the 95 * reverse, the fact that there are extra northbridge devices hanging around 96 * that we don't know about shouldn't be a problem. 97 * 98 * ------------------------------- 99 * Attach and Detach Complications 100 * ------------------------------- 101 * 102 * Because we need to map different PCI devices together, this means that we 103 * have multiple dev_info_t structures that we need to manage. Each of these is 104 * independently attached and detached. While this is easily managed for attach, 105 * it is not for detach. Each of these devices is a 'stub'. 106 * 107 * Once a device has been detached it will only come back if we have an active 108 * minor node that will be accessed. This means that if they are detached, 109 * nothing would ever cause them to be reattached. The system also doesn't 110 * provide us a way or any guarantees around making sure that we're attached to 111 * all such devices before we detach. As a result, unfortunately, it's easier to 112 * basically have detach always fail. 113 * 114 * --------------- 115 * Exposed Devices 116 * --------------- 117 * 118 * Rather than try and have all of the different functions that could be 119 * provided by one driver, we instead have created a nexus driver that will 120 * itself try and load children. Children are all pseudo-device drivers that 121 * provide different pieces of functionality that use this. 122 */ 123 124 #include <sys/modctl.h> 125 #include <sys/conf.h> 126 #include <sys/devops.h> 127 #include <sys/ddi.h> 128 #include <sys/sunddi.h> 129 #include <sys/pci.h> 130 #include <sys/sysmacros.h> 131 #include <sys/sunndi.h> 132 #include <sys/x86_archext.h> 133 #include <sys/cpuvar.h> 134 135 #include "amdzen.h" 136 137 amdzen_t *amdzen_data; 138 139 /* 140 * Array of northbridge IDs that we care about. 141 */ 142 static const uint16_t amdzen_nb_ids[] = { 143 /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 144 0x1450, 145 /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 146 0x15d0, 147 /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 148 0x1480, 149 /* Family 17h Renoir Models 60-6fh (Zen 2 uarch) */ 150 0x1630 151 }; 152 153 typedef struct { 154 char *acd_name; 155 amdzen_child_t acd_addr; 156 } amdzen_child_data_t; 157 158 static const amdzen_child_data_t amdzen_children[] = { 159 { "smntemp", AMDZEN_C_SMNTEMP }, 160 { "usmn", AMDZEN_C_USMN }, 161 { "zen_udf", AMDZEN_C_ZEN_UDF } 162 }; 163 164 static uint32_t 165 amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) 166 { 167 return (pci_config_get32(stub->azns_cfgspace, reg)); 168 } 169 170 static uint64_t 171 amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) 172 { 173 return (pci_config_get64(stub->azns_cfgspace, reg)); 174 } 175 176 static void 177 amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) 178 { 179 pci_config_put32(stub->azns_cfgspace, reg, val); 180 } 181 182 /* 183 * Perform a targeted 32-bit indirect read to a specific instance and function. 184 */ 185 static uint32_t 186 amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func, 187 uint16_t reg) 188 { 189 uint32_t val; 190 191 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 192 val = AMDZEN_DF_F4_FICAA_TARG_INST | AMDZEN_DF_F4_FICAA_SET_REG(reg) | 193 AMDZEN_DF_F4_FICAA_SET_FUNC(func) | 194 AMDZEN_DF_F4_FICAA_SET_INST(inst); 195 amdzen_stub_put32(df->adf_funcs[4], AMDZEN_DF_F4_FICAA, val); 196 return (amdzen_stub_get32(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO)); 197 } 198 199 /* 200 * Perform a targeted 64-bit indirect read to a specific instance and function. 201 */ 202 static uint64_t 203 amdzen_df_read64(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func, 204 uint16_t reg) 205 { 206 uint32_t val; 207 208 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 209 val = AMDZEN_DF_F4_FICAA_TARG_INST | AMDZEN_DF_F4_FICAA_SET_REG(reg) | 210 AMDZEN_DF_F4_FICAA_SET_FUNC(func) | 211 AMDZEN_DF_F4_FICAA_SET_INST(inst) | AMDZEN_DF_F4_FICAA_SET_64B; 212 amdzen_stub_put32(df->adf_funcs[4], AMDZEN_DF_F4_FICAA, val); 213 return (amdzen_stub_get64(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO)); 214 } 215 216 217 static uint32_t 218 amdzen_smn_read32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg) 219 { 220 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 221 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, reg); 222 return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); 223 } 224 225 static amdzen_df_t * 226 amdzen_df_find(amdzen_t *azn, uint_t dfno) 227 { 228 uint_t i; 229 230 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 231 if (dfno >= azn->azn_ndfs) { 232 return (NULL); 233 } 234 235 for (i = 0; i < azn->azn_ndfs; i++) { 236 amdzen_df_t *df = &azn->azn_dfs[i]; 237 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { 238 continue; 239 } 240 241 if (dfno == 0) { 242 return (df); 243 } 244 dfno--; 245 } 246 247 return (NULL); 248 } 249 250 /* 251 * Client functions that are used by nexus children. 252 */ 253 int 254 amdzen_c_smn_read32(uint_t dfno, uint32_t reg, uint32_t *valp) 255 { 256 amdzen_df_t *df; 257 amdzen_t *azn = amdzen_data; 258 259 mutex_enter(&azn->azn_mutex); 260 df = amdzen_df_find(azn, dfno); 261 if (df == NULL) { 262 mutex_exit(&azn->azn_mutex); 263 return (ENOENT); 264 } 265 266 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 267 mutex_exit(&azn->azn_mutex); 268 return (ENXIO); 269 } 270 271 *valp = amdzen_smn_read32(azn, df, reg); 272 mutex_exit(&azn->azn_mutex); 273 return (0); 274 } 275 276 uint_t 277 amdzen_c_df_count(void) 278 { 279 uint_t ret; 280 amdzen_t *azn = amdzen_data; 281 282 mutex_enter(&azn->azn_mutex); 283 ret = azn->azn_ndfs; 284 mutex_exit(&azn->azn_mutex); 285 return (ret); 286 } 287 288 int 289 amdzen_c_df_read32(uint_t dfno, uint8_t inst, uint8_t func, 290 uint16_t reg, uint32_t *valp) 291 { 292 amdzen_df_t *df; 293 amdzen_t *azn = amdzen_data; 294 295 mutex_enter(&azn->azn_mutex); 296 df = amdzen_df_find(azn, dfno); 297 if (df == NULL) { 298 mutex_exit(&azn->azn_mutex); 299 return (ENOENT); 300 } 301 302 *valp = amdzen_df_read32(azn, df, inst, func, reg); 303 mutex_exit(&azn->azn_mutex); 304 305 return (0); 306 } 307 308 int 309 amdzen_c_df_read64(uint_t dfno, uint8_t inst, uint8_t func, 310 uint16_t reg, uint64_t *valp) 311 { 312 amdzen_df_t *df; 313 amdzen_t *azn = amdzen_data; 314 315 mutex_enter(&azn->azn_mutex); 316 df = amdzen_df_find(azn, dfno); 317 if (df == NULL) { 318 mutex_exit(&azn->azn_mutex); 319 return (ENOENT); 320 } 321 322 *valp = amdzen_df_read64(azn, df, inst, func, reg); 323 mutex_exit(&azn->azn_mutex); 324 325 return (0); 326 } 327 328 static boolean_t 329 amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) 330 { 331 int ret; 332 dev_info_t *child; 333 334 if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, 335 (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { 336 dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " 337 "dip for %s", acd->acd_name); 338 return (B_FALSE); 339 } 340 341 ddi_set_parent_data(child, (void *)acd); 342 if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { 343 dev_err(azn->azn_dip, CE_WARN, "!failed to online child " 344 "dip %s: %d", acd->acd_name, ret); 345 return (B_FALSE); 346 } 347 348 return (B_TRUE); 349 } 350 351 static boolean_t 352 amdzen_map_dfs(amdzen_t *azn) 353 { 354 amdzen_stub_t *stub; 355 356 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 357 358 for (stub = list_head(&azn->azn_df_stubs); stub != NULL; 359 stub = list_next(&azn->azn_df_stubs, stub)) { 360 amdzen_df_t *df; 361 uint_t dfno; 362 363 dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; 364 if (dfno > AMDZEN_MAX_DFS) { 365 dev_err(stub->azns_dip, CE_WARN, "encountered df " 366 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 367 stub->azns_bus, stub->azns_dev, stub->azns_func); 368 goto err; 369 } 370 371 df = &azn->azn_dfs[dfno]; 372 373 if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { 374 dev_err(stub->azns_dip, CE_WARN, "encountered df " 375 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 376 stub->azns_bus, stub->azns_dev, stub->azns_func); 377 goto err; 378 } 379 380 if (df->adf_funcs[stub->azns_func] != NULL) { 381 dev_err(stub->azns_dip, CE_WARN, "encountered " 382 "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", 383 stub->azns_bus, stub->azns_dev, stub->azns_func); 384 goto err; 385 } 386 df->adf_funcs[stub->azns_func] = stub; 387 } 388 389 return (B_TRUE); 390 391 err: 392 azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; 393 return (B_FALSE); 394 } 395 396 static boolean_t 397 amdzen_check_dfs(amdzen_t *azn) 398 { 399 uint_t i; 400 boolean_t ret = B_TRUE; 401 402 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 403 amdzen_df_t *df = &azn->azn_dfs[i]; 404 uint_t count = 0; 405 406 /* 407 * We require all platforms to have DFs functions 0-6. Not all 408 * platforms have DF function 7. 409 */ 410 for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { 411 if (df->adf_funcs[func] != NULL) { 412 count++; 413 } 414 } 415 416 if (count == 0) 417 continue; 418 419 if (count != 7) { 420 ret = B_FALSE; 421 dev_err(azn->azn_dip, CE_WARN, "df %u devices " 422 "incomplete", i); 423 } else { 424 df->adf_flags |= AMDZEN_DF_F_VALID; 425 azn->azn_ndfs++; 426 } 427 } 428 429 return (ret); 430 } 431 432 static const uint8_t amdzen_df_rome_ids[0x2b] = { 433 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 434 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 435 44, 45, 46, 47, 48 436 }; 437 438 /* 439 * Check the first df entry to see if it belongs to Rome or Milan. If so, then 440 * it uses the disjoint ID space. 441 */ 442 static boolean_t 443 amdzen_is_rome_style(uint_t id) 444 { 445 return (id == 0x1490 || id == 0x1650); 446 } 447 448 /* 449 * Initialize our knowledge about a given series of nodes on the data fabric. 450 */ 451 static void 452 amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) 453 { 454 uint_t i; 455 uint32_t val; 456 457 val = amdzen_stub_get32(df->adf_funcs[0], AMDZEN_DF_F0_CFG_ADDR_CTL); 458 df->adf_nb_busno = AMDZEN_DF_F0_CFG_ADDR_CTL_BUS_NUM(val); 459 val = amdzen_stub_get32(df->adf_funcs[0], AMDZEN_DF_F0_FBICNT); 460 df->adf_nents = AMDZEN_DF_F0_FBICNT_COUNT(val); 461 if (df->adf_nents == 0) 462 return; 463 df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, 464 KM_SLEEP); 465 466 for (i = 0; i < df->adf_nents; i++) { 467 amdzen_df_ent_t *dfe = &df->adf_ents[i]; 468 uint8_t inst = i; 469 470 /* 471 * Unfortunately, Rome uses a discontinuous instance ID pattern 472 * while everything else we can find uses a contiguous instance 473 * ID pattern. This means that for Rome, we need to adjust the 474 * indexes that we iterate over, though the total number of 475 * entries is right. 476 */ 477 if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { 478 if (inst > ARRAY_SIZE(amdzen_df_rome_ids)) { 479 dev_err(azn->azn_dip, CE_WARN, "Rome family " 480 "processor reported more ids than the PPR, " 481 "resetting %u to instance zero", inst); 482 inst = 0; 483 } else { 484 inst = amdzen_df_rome_ids[inst]; 485 } 486 } 487 488 dfe->adfe_drvid = inst; 489 dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, 0, 490 AMDZEN_DF_F0_FBIINFO0); 491 dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, 0, 492 AMDZEN_DF_F0_FBIINFO1); 493 dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, 0, 494 AMDZEN_DF_F0_FBIINFO2); 495 dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, 0, 496 AMDZEN_DF_F0_FBIINFO3); 497 dfe->adfe_syscfg = amdzen_df_read32(azn, df, inst, 1, 498 AMDZEN_DF_F1_SYSCFG); 499 dfe->adfe_mask0 = amdzen_df_read32(azn, df, inst, 1, 500 AMDZEN_DF_F1_FIDMASK0); 501 dfe->adfe_mask1 = amdzen_df_read32(azn, df, inst, 1, 502 AMDZEN_DF_F1_FIDMASK1); 503 504 dfe->adfe_type = AMDZEN_DF_F0_FBIINFO0_TYPE(dfe->adfe_info0); 505 dfe->adfe_sdp_width = 506 AMDZEN_DF_F0_FBIINFO0_SDP_WIDTH(dfe->adfe_info0); 507 if (AMDZEN_DF_F0_FBIINFO0_ENABLED(dfe->adfe_info0)) { 508 dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; 509 } 510 dfe->adfe_fti_width = 511 AMDZEN_DF_F0_FBIINFO0_FTI_WIDTH(dfe->adfe_info0); 512 dfe->adfe_sdp_count = 513 AMDZEN_DF_F0_FBIINFO0_SDP_PCOUNT(dfe->adfe_info0); 514 dfe->adfe_fti_count = 515 AMDZEN_DF_F0_FBIINFO0_FTI_PCOUNT(dfe->adfe_info0); 516 if (AMDZEN_DF_F0_FBIINFO0_HAS_MCA(dfe->adfe_info0)) { 517 dfe->adfe_flags |= AMDZEN_DFE_F_MCA; 518 } 519 dfe->adfe_subtype = 520 AMDZEN_DF_F0_FBIINFO0_SUBTYPE(dfe->adfe_info0); 521 522 dfe->adfe_inst_id = 523 AMDZEN_DF_F0_FBIINFO3_INSTID(dfe->adfe_info3); 524 dfe->adfe_fabric_id = 525 AMDZEN_DF_F0_FBIINFO3_FABID(dfe->adfe_info3); 526 } 527 528 df->adf_syscfg = amdzen_stub_get32(df->adf_funcs[1], 529 AMDZEN_DF_F1_SYSCFG); 530 df->adf_nodeid = AMDZEN_DF_F1_SYSCFG_NODEID(df->adf_syscfg); 531 df->adf_mask0 = amdzen_stub_get32(df->adf_funcs[1], 532 AMDZEN_DF_F1_FIDMASK0); 533 df->adf_mask1 = amdzen_stub_get32(df->adf_funcs[1], 534 AMDZEN_DF_F1_FIDMASK1); 535 } 536 537 static void 538 amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) 539 { 540 amdzen_stub_t *stub; 541 542 for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; 543 stub = list_next(&azn->azn_nb_stubs, stub)) { 544 if (stub->azns_bus == df->adf_nb_busno) { 545 df->adf_flags |= AMDZEN_DF_F_FOUND_NB; 546 df->adf_nb = stub; 547 return; 548 } 549 } 550 } 551 552 static void 553 amdzen_nexus_init(void *arg) 554 { 555 uint_t i; 556 amdzen_t *azn = arg; 557 558 /* 559 * First go through all of the stubs and assign the DF entries. 560 */ 561 mutex_enter(&azn->azn_mutex); 562 if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { 563 azn->azn_flags |= AMDZEN_F_MAP_ERROR; 564 goto done; 565 } 566 567 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 568 amdzen_df_t *df = &azn->azn_dfs[i]; 569 570 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) 571 continue; 572 amdzen_setup_df(azn, df); 573 amdzen_find_nb(azn, df); 574 } 575 576 /* 577 * Not all children may be installed. As such, we do not treat the 578 * failure of a child as fatal to the driver. 579 */ 580 mutex_exit(&azn->azn_mutex); 581 for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { 582 (void) amdzen_create_child(azn, &amdzen_children[i]); 583 } 584 mutex_enter(&azn->azn_mutex); 585 586 done: 587 azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; 588 azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; 589 azn->azn_taskqid = TASKQID_INVALID; 590 cv_broadcast(&azn->azn_cv); 591 mutex_exit(&azn->azn_mutex); 592 } 593 594 static int 595 amdzen_stub_scan_cb(dev_info_t *dip, void *arg) 596 { 597 amdzen_t *azn = arg; 598 uint16_t vid, did; 599 int *regs; 600 uint_t nregs, i; 601 boolean_t match = B_FALSE; 602 603 if (dip == ddi_root_node()) { 604 return (DDI_WALK_CONTINUE); 605 } 606 607 /* 608 * If a node in question is not a pci node, then we have no interest in 609 * it as all the stubs that we care about are related to pci devices. 610 */ 611 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 612 return (DDI_WALK_PRUNECHILD); 613 } 614 615 /* 616 * If we can't get a device or vendor ID and prove that this is an AMD 617 * part, then we don't care about it. 618 */ 619 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 620 "vendor-id", PCI_EINVAL16); 621 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 622 "device-id", PCI_EINVAL16); 623 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 624 return (DDI_WALK_CONTINUE); 625 } 626 627 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 628 return (DDI_WALK_CONTINUE); 629 } 630 631 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 632 if (amdzen_nb_ids[i] == did) { 633 match = B_TRUE; 634 } 635 } 636 637 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 638 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 639 return (DDI_WALK_CONTINUE); 640 } 641 642 if (nregs == 0) { 643 ddi_prop_free(regs); 644 return (DDI_WALK_CONTINUE); 645 } 646 647 if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && 648 PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { 649 match = B_TRUE; 650 } 651 652 ddi_prop_free(regs); 653 if (match) { 654 mutex_enter(&azn->azn_mutex); 655 azn->azn_nscanned++; 656 mutex_exit(&azn->azn_mutex); 657 } 658 659 return (DDI_WALK_CONTINUE); 660 } 661 662 static void 663 amdzen_stub_scan(void *arg) 664 { 665 amdzen_t *azn = arg; 666 667 mutex_enter(&azn->azn_mutex); 668 azn->azn_nscanned = 0; 669 mutex_exit(&azn->azn_mutex); 670 671 ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); 672 673 mutex_enter(&azn->azn_mutex); 674 azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; 675 azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; 676 677 if (azn->azn_nscanned == 0) { 678 azn->azn_flags |= AMDZEN_F_UNSUPPORTED; 679 azn->azn_taskqid = TASKQID_INVALID; 680 cv_broadcast(&azn->azn_cv); 681 } else if (azn->azn_npresent == azn->azn_nscanned) { 682 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 683 azn->azn_taskqid = taskq_dispatch(system_taskq, 684 amdzen_nexus_init, azn, TQ_SLEEP); 685 } 686 mutex_exit(&azn->azn_mutex); 687 } 688 689 /* 690 * Unfortunately we can't really let the stubs detach as we may need them to be 691 * available for client operations. We may be able to improve this if we know 692 * that the actual nexus is going away. However, as long as it's active, we need 693 * all the stubs. 694 */ 695 int 696 amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) 697 { 698 if (cmd == DDI_SUSPEND) { 699 return (DDI_SUCCESS); 700 } 701 702 return (DDI_FAILURE); 703 } 704 705 int 706 amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) 707 { 708 int *regs, reg; 709 uint_t nregs, i; 710 uint16_t vid, did; 711 amdzen_stub_t *stub; 712 amdzen_t *azn = amdzen_data; 713 boolean_t valid = B_FALSE; 714 boolean_t nb = B_FALSE; 715 716 if (cmd == DDI_RESUME) { 717 return (DDI_SUCCESS); 718 } else if (cmd != DDI_ATTACH) { 719 return (DDI_FAILURE); 720 } 721 722 /* 723 * Make sure that the stub that we've been asked to attach is a pci type 724 * device. If not, then there is no reason for us to proceed. 725 */ 726 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 727 dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " 728 "stub: %s", ddi_get_name(dip)); 729 return (DDI_FAILURE); 730 } 731 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 732 "vendor-id", PCI_EINVAL16); 733 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 734 "device-id", PCI_EINVAL16); 735 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 736 dev_err(dip, CE_WARN, "failed to get PCI ID properties"); 737 return (DDI_FAILURE); 738 } 739 740 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 741 dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", 742 cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? 743 AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); 744 return (DDI_FAILURE); 745 } 746 747 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 748 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 749 dev_err(dip, CE_WARN, "failed to get 'reg' property"); 750 return (DDI_FAILURE); 751 } 752 753 if (nregs == 0) { 754 ddi_prop_free(regs); 755 dev_err(dip, CE_WARN, "missing 'reg' property values"); 756 return (DDI_FAILURE); 757 } 758 reg = *regs; 759 ddi_prop_free(regs); 760 761 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 762 if (amdzen_nb_ids[i] == did) { 763 valid = B_TRUE; 764 nb = B_TRUE; 765 } 766 } 767 768 if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && 769 PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { 770 valid = B_TRUE; 771 nb = B_FALSE; 772 } 773 774 if (!valid) { 775 dev_err(dip, CE_WARN, "device %s didn't match the nexus list", 776 ddi_get_name(dip)); 777 return (DDI_FAILURE); 778 } 779 780 stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); 781 if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { 782 dev_err(dip, CE_WARN, "failed to set up config space"); 783 kmem_free(stub, sizeof (amdzen_stub_t)); 784 return (DDI_FAILURE); 785 } 786 787 stub->azns_dip = dip; 788 stub->azns_vid = vid; 789 stub->azns_did = did; 790 stub->azns_bus = PCI_REG_BUS_G(reg); 791 stub->azns_dev = PCI_REG_DEV_G(reg); 792 stub->azns_func = PCI_REG_FUNC_G(reg); 793 ddi_set_driver_private(dip, stub); 794 795 mutex_enter(&azn->azn_mutex); 796 azn->azn_npresent++; 797 if (nb) { 798 list_insert_tail(&azn->azn_nb_stubs, stub); 799 } else { 800 list_insert_tail(&azn->azn_df_stubs, stub); 801 } 802 803 if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && 804 azn->azn_nscanned == azn->azn_npresent) { 805 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 806 azn->azn_taskqid = taskq_dispatch(system_taskq, 807 amdzen_nexus_init, azn, TQ_SLEEP); 808 } 809 mutex_exit(&azn->azn_mutex); 810 811 return (DDI_SUCCESS); 812 } 813 814 static int 815 amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 816 void *arg, void *result) 817 { 818 char buf[32]; 819 dev_info_t *child; 820 const amdzen_child_data_t *acd; 821 822 switch (ctlop) { 823 case DDI_CTLOPS_REPORTDEV: 824 if (rdip == NULL) { 825 return (DDI_FAILURE); 826 } 827 cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", 828 ddi_node_name(rdip), ddi_get_name_addr(rdip), 829 ddi_driver_name(rdip), ddi_get_instance(rdip)); 830 break; 831 case DDI_CTLOPS_INITCHILD: 832 child = arg; 833 if (child == NULL) { 834 dev_err(dip, CE_WARN, "!no child passed for " 835 "DDI_CTLOPS_INITCHILD"); 836 } 837 838 acd = ddi_get_parent_data(child); 839 if (acd == NULL) { 840 dev_err(dip, CE_WARN, "!missing child parent data"); 841 return (DDI_FAILURE); 842 } 843 844 if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= 845 sizeof (buf)) { 846 dev_err(dip, CE_WARN, "!failed to construct device " 847 "addr due to overflow"); 848 return (DDI_FAILURE); 849 } 850 851 ddi_set_name_addr(child, buf); 852 break; 853 case DDI_CTLOPS_UNINITCHILD: 854 child = arg; 855 if (child == NULL) { 856 dev_err(dip, CE_WARN, "!no child passed for " 857 "DDI_CTLOPS_UNINITCHILD"); 858 } 859 860 ddi_set_name_addr(child, NULL); 861 break; 862 default: 863 return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 864 } 865 return (DDI_SUCCESS); 866 } 867 868 static int 869 amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 870 { 871 amdzen_t *azn = amdzen_data; 872 873 if (cmd == DDI_RESUME) { 874 return (DDI_SUCCESS); 875 } else if (cmd != DDI_ATTACH) { 876 return (DDI_FAILURE); 877 } 878 879 mutex_enter(&azn->azn_mutex); 880 if (azn->azn_dip != NULL) { 881 dev_err(dip, CE_WARN, "driver is already attached!"); 882 mutex_exit(&azn->azn_mutex); 883 return (DDI_FAILURE); 884 } 885 886 azn->azn_dip = dip; 887 azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, 888 azn, TQ_SLEEP); 889 azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; 890 mutex_exit(&azn->azn_mutex); 891 892 return (DDI_SUCCESS); 893 } 894 895 static int 896 amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 897 { 898 amdzen_t *azn = amdzen_data; 899 900 if (cmd == DDI_SUSPEND) { 901 return (DDI_SUCCESS); 902 } else if (cmd != DDI_DETACH) { 903 return (DDI_FAILURE); 904 } 905 906 mutex_enter(&azn->azn_mutex); 907 while (azn->azn_taskqid != TASKQID_INVALID) { 908 cv_wait(&azn->azn_cv, &azn->azn_mutex); 909 } 910 911 /* 912 * If we've attached any stub drivers, e.g. this platform is important 913 * for us, then we fail detach. 914 */ 915 if (!list_is_empty(&azn->azn_df_stubs) || 916 !list_is_empty(&azn->azn_nb_stubs)) { 917 mutex_exit(&azn->azn_mutex); 918 return (DDI_FAILURE); 919 } 920 921 azn->azn_dip = NULL; 922 mutex_exit(&azn->azn_mutex); 923 924 return (DDI_SUCCESS); 925 } 926 927 static void 928 amdzen_free(void) 929 { 930 if (amdzen_data == NULL) { 931 return; 932 } 933 934 VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); 935 list_destroy(&amdzen_data->azn_df_stubs); 936 VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); 937 list_destroy(&amdzen_data->azn_nb_stubs); 938 cv_destroy(&amdzen_data->azn_cv); 939 mutex_destroy(&amdzen_data->azn_mutex); 940 kmem_free(amdzen_data, sizeof (amdzen_t)); 941 amdzen_data = NULL; 942 } 943 944 static void 945 amdzen_alloc(void) 946 { 947 amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); 948 mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); 949 list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), 950 offsetof(amdzen_stub_t, azns_link)); 951 list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), 952 offsetof(amdzen_stub_t, azns_link)); 953 cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); 954 } 955 956 struct bus_ops amdzen_bus_ops = { 957 .busops_rev = BUSO_REV, 958 .bus_map = nullbusmap, 959 .bus_dma_map = ddi_no_dma_map, 960 .bus_dma_allochdl = ddi_no_dma_allochdl, 961 .bus_dma_freehdl = ddi_no_dma_freehdl, 962 .bus_dma_bindhdl = ddi_no_dma_bindhdl, 963 .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, 964 .bus_dma_flush = ddi_no_dma_flush, 965 .bus_dma_win = ddi_no_dma_win, 966 .bus_dma_ctl = ddi_no_dma_mctl, 967 .bus_prop_op = ddi_bus_prop_op, 968 .bus_ctl = amdzen_bus_ctl 969 }; 970 971 static struct dev_ops amdzen_dev_ops = { 972 .devo_rev = DEVO_REV, 973 .devo_refcnt = 0, 974 .devo_getinfo = nodev, 975 .devo_identify = nulldev, 976 .devo_probe = nulldev, 977 .devo_attach = amdzen_attach, 978 .devo_detach = amdzen_detach, 979 .devo_reset = nodev, 980 .devo_quiesce = ddi_quiesce_not_needed, 981 .devo_bus_ops = &amdzen_bus_ops 982 }; 983 984 static struct modldrv amdzen_modldrv = { 985 .drv_modops = &mod_driverops, 986 .drv_linkinfo = "AMD Zen Nexus Driver", 987 .drv_dev_ops = &amdzen_dev_ops 988 }; 989 990 static struct modlinkage amdzen_modlinkage = { 991 .ml_rev = MODREV_1, 992 .ml_linkage = { &amdzen_modldrv, NULL } 993 }; 994 995 int 996 _init(void) 997 { 998 int ret; 999 1000 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && 1001 cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { 1002 return (ENOTSUP); 1003 } 1004 1005 if ((ret = mod_install(&amdzen_modlinkage)) == 0) { 1006 amdzen_alloc(); 1007 } 1008 1009 return (ret); 1010 } 1011 1012 int 1013 _info(struct modinfo *modinfop) 1014 { 1015 return (mod_info(&amdzen_modlinkage, modinfop)); 1016 } 1017 1018 int 1019 _fini(void) 1020 { 1021 int ret; 1022 1023 if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { 1024 amdzen_free(); 1025 } 1026 1027 return (ret); 1028 } 1029