/* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2019, Joyent, Inc. * Copyright 2022 Oxide Computer Company */ /* * Nexus Driver for AMD Zen family systems. The purpose of this driver is to * provide access to the following resources in a single, centralized fashion: * * - The per-chip Data Fabric * - The North Bridge * - The System Management Network (SMN) * * This is a nexus driver as once we have attached to all the requisite * components, we will enumerate child devices which consume this functionality. * * ------------------------ * Mapping Devices Together * ------------------------ * * The operating system needs to expose things like temperature sensors and DRAM * configuration registers in terms that are meaningful to the system such as * logical CPUs, cores, etc. This driver attaches to the PCI IDs that represent * the northbridge and data fabric; however, there are multiple PCI devices (one * per die) that exist. This driver does manage to map all of these three things * together; however, it requires some acrobatics. Unfortunately, there's no * direct way to map a northbridge to its corresponding die. However, we can map * a CPU die to a data fabric PCI device and a data fabric PCI device to a * corresponding northbridge PCI device. * * In current Zen based products, there is a direct mapping between processor * nodes and a data fabric PCI device. All of the devices are on PCI Bus 0 and * start from Device 0x18. Device 0x18 maps to processor node 0, 0x19 to * processor node 1, etc. This means that to map a logical CPU to a data fabric * device, we take its processor node id, add it to 0x18 and find the PCI device * that is on bus 0, device 0x18. As each data fabric device is attached based * on its PCI ID, we add it to the global list, amd_nbdf_dfs that is in the * amd_f17nbdf_t structure. * * The northbridge PCI device has a defined device and function, but the PCI bus * that it's on can vary. Each die has its own series of PCI buses that are * assigned to it and the northbridge PCI device is on the first of die-specific * PCI bus for each die. This also means that the northbridge will not show up * on PCI bus 0, which is the PCI bus that all of the data fabric devices are * on. While conventionally the northbridge with the lowest PCI bus value * would correspond to processor node zero, hardware does not guarantee that at * all. Because we don't want to be at the mercy of firmware, we don't rely on * this ordering, even though we have yet to find a system that deviates from * this scheme. * * One of the registers in the data fabric device's function 0 * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to have the first PCI bus that is * associated with the processor node. This means that we can map a data fabric * device to a northbridge by finding the northbridge whose PCI bus matches the * value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. * * We can map a northbridge to a data fabric device and a data fabric device to * a die. Because these are generally 1:1 mappings, there is a transitive * relationship and therefore we know which northbridge is associated with which * processor die. This is summarized in the following image: * * +-------+ +-----------------------------------+ +--------------+ * | Die 0 |--->| Data Fabric PCI BDF 0/18/0 |------->| Northbridge | * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | * ... +-----------------------------------+ +--------------+ * +-------+ +------------------------------------+ +--------------+ * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |------->| Northbridge | * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | * +------------------------------------+ +--------------+ * * Note, the PCI buses used by the northbridges here are arbitrary. They do not * reflect the actual values by hardware; however, the bus/device/function (BDF) * of the data fabric accurately models hardware. All of the BDF values are in * hex. * * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), * AMD has multiple northbridges that exist on a given die. All of these * northbridges share the same data fabric and system management network port. * From our perspective this means that some of the northbridge devices will be * redundant and that we will no longer have a 1:1 mapping between the * northbridge and the data fabric devices. Every data fabric will have a * northbridge, but not every northbridge will have a data fabric device mapped. * Because we're always trying to map from a die to a northbridge and not the * reverse, the fact that there are extra northbridge devices hanging around * that we don't know about shouldn't be a problem. * * ------------------------------- * Attach and Detach Complications * ------------------------------- * * Because we need to map different PCI devices together, this means that we * have multiple dev_info_t structures that we need to manage. Each of these is * independently attached and detached. While this is easily managed for attach, * it is not for detach. Each of these devices is a 'stub'. * * Once a device has been detached it will only come back if we have an active * minor node that will be accessed. This means that if they are detached, * nothing would ever cause them to be reattached. The system also doesn't * provide us a way or any guarantees around making sure that we're attached to * all such devices before we detach. As a result, unfortunately, it's easier to * basically have detach always fail. * * --------------- * Exposed Devices * --------------- * * Rather than try and have all of the different functions that could be * provided by one driver, we instead have created a nexus driver that will * itself try and load children. Children are all pseudo-device drivers that * provide different pieces of functionality that use this. * * ------- * Locking * ------- * * The amdzen_data structure contains a single lock, azn_mutex. The various * client functions are intended for direct children of our nexus, but have been * designed in case someone else depends on this driver despite not being a * child. Once a DF has been discovered, the set of entities inside of it * (adf_nents, adf_ents[]) is considered static, constant data. This means that * iterating over it in and of itself does not require locking; however, the * discovery of the amd_df_t does. In addition, whenever performing register * accesses to the DF or SMN, those require locking. This means that one must * hold the lock in the following circumstances: * * o Looking up DF structures * o Reading or writing to DF registers * o Reading or writing to SMN registers * * In general, it is preferred that the lock be held across an entire client * operation if possible. The only time this becomes an issue are when we have * callbacks into our callers (ala amdzen_c_df_iter()) as they will likely * recursively call into us. */ #include #include #include #include #include #include #include #include #include #include #include #include "amdzen_client.h" #include "amdzen.h" amdzen_t *amdzen_data; /* * Array of northbridge IDs that we care about. */ static const uint16_t amdzen_nb_ids[] = { /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 0x1450, /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 0x15d0, /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 0x1480, /* Family 17h/19h Renoir, Cezanne, Van Gogh Zen 2/3 uarch */ 0x1630, /* Family 19h Genoa */ 0x14a4, /* Family 17h Mendocino, Family 19h Rembrandt */ 0x14b5, /* Family 19h Raphael */ 0x14d8 }; typedef struct { char *acd_name; amdzen_child_t acd_addr; } amdzen_child_data_t; static const amdzen_child_data_t amdzen_children[] = { { "smntemp", AMDZEN_C_SMNTEMP }, { "usmn", AMDZEN_C_USMN }, { "zen_udf", AMDZEN_C_ZEN_UDF }, { "zen_umc", AMDZEN_C_ZEN_UMC } }; static uint8_t amdzen_stub_get8(amdzen_stub_t *stub, off_t reg) { return (pci_config_get8(stub->azns_cfgspace, reg)); } static uint16_t amdzen_stub_get16(amdzen_stub_t *stub, off_t reg) { return (pci_config_get16(stub->azns_cfgspace, reg)); } static uint32_t amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) { return (pci_config_get32(stub->azns_cfgspace, reg)); } static uint64_t amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) { return (pci_config_get64(stub->azns_cfgspace, reg)); } static void amdzen_stub_put8(amdzen_stub_t *stub, off_t reg, uint8_t val) { pci_config_put8(stub->azns_cfgspace, reg, val); } static void amdzen_stub_put16(amdzen_stub_t *stub, off_t reg, uint16_t val) { pci_config_put16(stub->azns_cfgspace, reg, val); } static void amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) { pci_config_put32(stub->azns_cfgspace, reg, val); } static uint64_t amdzen_df_read_regdef(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def, uint8_t inst, boolean_t do_64) { df_reg_def_t ficaa; df_reg_def_t ficad; uint32_t val = 0; df_rev_t df_rev = azn->azn_dfs[0].adf_rev; VERIFY(MUTEX_HELD(&azn->azn_mutex)); ASSERT3U(def.drd_gens & df_rev, ==, df_rev); val = DF_FICAA_V2_SET_TARG_INST(val, 1); val = DF_FICAA_V2_SET_FUNC(val, def.drd_func); val = DF_FICAA_V2_SET_INST(val, inst); val = DF_FICAA_V2_SET_64B(val, do_64 ? 1 : 0); switch (df_rev) { case DF_REV_2: case DF_REV_3: case DF_REV_3P5: ficaa = DF_FICAA_V2; ficad = DF_FICAD_LO_V2; /* * Both here and in the DFv4 case, the register ignores the * lower 2 bits. That is we can only address and encode things * in units of 4 bytes. */ val = DF_FICAA_V2_SET_REG(val, def.drd_reg >> 2); break; case DF_REV_4: ficaa = DF_FICAA_V4; ficad = DF_FICAD_LO_V4; val = DF_FICAA_V4_SET_REG(val, def.drd_reg >> 2); break; default: panic("encountered unexpected DF rev: %u", df_rev); } amdzen_stub_put32(df->adf_funcs[ficaa.drd_func], ficaa.drd_reg, val); if (do_64) { return (amdzen_stub_get64(df->adf_funcs[ficad.drd_func], ficad.drd_reg)); } else { return (amdzen_stub_get32(df->adf_funcs[ficad.drd_func], ficad.drd_reg)); } } /* * Perform a targeted 32-bit indirect read to a specific instance and function. */ static uint32_t amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, const df_reg_def_t def) { return (amdzen_df_read_regdef(azn, df, def, inst, B_FALSE)); } /* * For a broadcast read, just go to the underlying PCI function and perform a * read. At this point in time, we don't believe we need to use the FICAA/FICAD * to access it (though it does have a broadcast mode). */ static uint32_t amdzen_df_read32_bcast(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def) { VERIFY(MUTEX_HELD(&azn->azn_mutex)); return (amdzen_stub_get32(df->adf_funcs[def.drd_func], def.drd_reg)); } static uint32_t amdzen_smn_read(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg) { const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); VERIFY(MUTEX_HELD(&azn->azn_mutex)); amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); switch (SMN_REG_SIZE(reg)) { case 1: return ((uint32_t)amdzen_stub_get8(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off)); case 2: return ((uint32_t)amdzen_stub_get16(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off)); case 4: return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); default: panic("unreachable invalid SMN register size %u", SMN_REG_SIZE(reg)); } } static void amdzen_smn_write(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg, const uint32_t val) { const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); VERIFY(SMN_REG_VALUE_FITS(reg, val)); VERIFY(MUTEX_HELD(&azn->azn_mutex)); amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); switch (SMN_REG_SIZE(reg)) { case 1: amdzen_stub_put8(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, (uint8_t)val); break; case 2: amdzen_stub_put16(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, (uint16_t)val); break; case 4: amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val); break; default: panic("unreachable invalid SMN register size %u", SMN_REG_SIZE(reg)); } } static amdzen_df_t * amdzen_df_find(amdzen_t *azn, uint_t dfno) { uint_t i; ASSERT(MUTEX_HELD(&azn->azn_mutex)); if (dfno >= azn->azn_ndfs) { return (NULL); } for (i = 0; i < azn->azn_ndfs; i++) { amdzen_df_t *df = &azn->azn_dfs[i]; if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { continue; } if (dfno == 0) { return (df); } dfno--; } return (NULL); } /* * Client functions that are used by nexus children. */ int amdzen_c_smn_read(uint_t dfno, const smn_reg_t reg, uint32_t *valp) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; if (!SMN_REG_SIZE_IS_VALID(reg)) return (EINVAL); if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) return (EINVAL); mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, dfno); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { mutex_exit(&azn->azn_mutex); return (ENXIO); } *valp = amdzen_smn_read(azn, df, reg); mutex_exit(&azn->azn_mutex); return (0); } int amdzen_c_smn_write(uint_t dfno, const smn_reg_t reg, const uint32_t val) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; if (!SMN_REG_SIZE_IS_VALID(reg)) return (EINVAL); if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) return (EINVAL); if (!SMN_REG_VALUE_FITS(reg, val)) return (EOVERFLOW); mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, dfno); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { mutex_exit(&azn->azn_mutex); return (ENXIO); } amdzen_smn_write(azn, df, reg, val); mutex_exit(&azn->azn_mutex); return (0); } uint_t amdzen_c_df_count(void) { uint_t ret; amdzen_t *azn = amdzen_data; mutex_enter(&azn->azn_mutex); ret = azn->azn_ndfs; mutex_exit(&azn->azn_mutex); return (ret); } df_rev_t amdzen_c_df_rev(void) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; df_rev_t rev; /* * Always use the first DF instance to determine what we're using. Our * current assumption, which seems to generally be true, is that the * given DF revisions are the same in a given system when the DFs are * directly connected. */ mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, 0); if (df == NULL) { rev = DF_REV_UNKNOWN; } else { rev = df->adf_rev; } mutex_exit(&azn->azn_mutex); return (rev); } int amdzen_c_df_read32(uint_t dfno, uint8_t inst, const df_reg_def_t def, uint32_t *valp) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, dfno); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } *valp = amdzen_df_read_regdef(azn, df, def, inst, B_FALSE); mutex_exit(&azn->azn_mutex); return (0); } int amdzen_c_df_read64(uint_t dfno, uint8_t inst, const df_reg_def_t def, uint64_t *valp) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, dfno); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } *valp = amdzen_df_read_regdef(azn, df, def, inst, B_TRUE); mutex_exit(&azn->azn_mutex); return (0); } int amdzen_c_df_iter(uint_t dfno, zen_df_type_t type, amdzen_c_iter_f func, void *arg) { amdzen_df_t *df; amdzen_t *azn = amdzen_data; df_type_t df_type; uint8_t df_subtype; /* * Unlike other calls here, we hold our lock only to find the DF here. * The main reason for this is the nature of the callback function. * Folks are iterating over instances so they can call back into us. If * you look at the locking statement, the thing that is most volatile * right here and what we need to protect is the DF itself and * subsequent register accesses to it. The actual data about which * entities exist is static and so once we have found a DF we should * hopefully be in good shape as they only come, but don't go. */ mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, dfno); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } mutex_exit(&azn->azn_mutex); switch (type) { case ZEN_DF_TYPE_CS_UMC: df_type = DF_TYPE_CS; /* * In the original Zeppelin DFv2 die there was no subtype field * used for the CS. The UMC is the only type and has a subtype * of zero. */ if (df->adf_rev != DF_REV_2) { df_subtype = DF_CS_SUBTYPE_UMC; } else { df_subtype = 0; } break; case ZEN_DF_TYPE_CCM_CPU: /* * While the wording of the PPR is a little weird, the CCM still * has subtype 0 in DFv4 systems; however, what's said to be for * the CPU appears to apply to the ACM. */ df_type = DF_TYPE_CCM; df_subtype = 0; break; default: return (EINVAL); } for (uint_t i = 0; i < df->adf_nents; i++) { amdzen_df_ent_t *ent = &df->adf_ents[i]; /* * Some DF components are not considered enabled and therefore * will end up having bogus values in their ID fields. If we do * not have an enable flag set, we must skip this node. */ if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) continue; if (ent->adfe_type == df_type && ent->adfe_subtype == df_subtype) { int ret = func(dfno, ent->adfe_fabric_id, ent->adfe_inst_id, arg); if (ret != 0) { return (ret); } } } return (0); } int amdzen_c_df_fabric_decomp(df_fabric_decomp_t *decomp) { const amdzen_df_t *df; amdzen_t *azn = amdzen_data; mutex_enter(&azn->azn_mutex); df = amdzen_df_find(azn, 0); if (df == NULL) { mutex_exit(&azn->azn_mutex); return (ENOENT); } *decomp = df->adf_decomp; mutex_exit(&azn->azn_mutex); return (0); } static boolean_t amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) { int ret; dev_info_t *child; if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " "dip for %s", acd->acd_name); return (B_FALSE); } ddi_set_parent_data(child, (void *)acd); if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { dev_err(azn->azn_dip, CE_WARN, "!failed to online child " "dip %s: %d", acd->acd_name, ret); return (B_FALSE); } return (B_TRUE); } static boolean_t amdzen_map_dfs(amdzen_t *azn) { amdzen_stub_t *stub; ASSERT(MUTEX_HELD(&azn->azn_mutex)); for (stub = list_head(&azn->azn_df_stubs); stub != NULL; stub = list_next(&azn->azn_df_stubs, stub)) { amdzen_df_t *df; uint_t dfno; dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; if (dfno > AMDZEN_MAX_DFS) { dev_err(stub->azns_dip, CE_WARN, "encountered df " "device with illegal DF PCI b/d/f: 0x%x/%x/%x", stub->azns_bus, stub->azns_dev, stub->azns_func); goto err; } df = &azn->azn_dfs[dfno]; if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { dev_err(stub->azns_dip, CE_WARN, "encountered df " "device with illegal DF PCI b/d/f: 0x%x/%x/%x", stub->azns_bus, stub->azns_dev, stub->azns_func); goto err; } if (df->adf_funcs[stub->azns_func] != NULL) { dev_err(stub->azns_dip, CE_WARN, "encountered " "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", stub->azns_bus, stub->azns_dev, stub->azns_func); goto err; } df->adf_funcs[stub->azns_func] = stub; } return (B_TRUE); err: azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; return (B_FALSE); } static boolean_t amdzen_check_dfs(amdzen_t *azn) { uint_t i; boolean_t ret = B_TRUE; for (i = 0; i < AMDZEN_MAX_DFS; i++) { amdzen_df_t *df = &azn->azn_dfs[i]; uint_t count = 0; /* * We require all platforms to have DFs functions 0-6. Not all * platforms have DF function 7. */ for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { if (df->adf_funcs[func] != NULL) { count++; } } if (count == 0) continue; if (count != 7) { ret = B_FALSE; dev_err(azn->azn_dip, CE_WARN, "df %u devices " "incomplete", i); } else { df->adf_flags |= AMDZEN_DF_F_VALID; azn->azn_ndfs++; } } return (ret); } static const uint8_t amdzen_df_rome_ids[0x2b] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48 }; /* * Check the first df entry to see if it belongs to Rome or Milan. If so, then * it uses the disjoint ID space. */ static boolean_t amdzen_is_rome_style(uint_t id) { return (id == 0x1490 || id == 0x1650); } /* * To be able to do most other things we want to do, we must first determine * what revision of the DF (data fabric) that we're using. * * Snapshot the df version. This was added explicitly in DFv4.0, around the Zen * 4 timeframe and allows us to tell apart different version of the DF register * set, most usefully when various subtypes were added. * * Older versions can theoretically be told apart based on usage of reserved * registers. We walk these in the following order, starting with the newest rev * and walking backwards to tell things apart: * * o v3.5 -> Check function 1, register 0x150. This was reserved prior * to this point. This is actually DF_FIDMASK0_V3P5. We are supposed * to check bits [7:0]. * * o v3.0 -> Check function 1, register 0x208. The low byte (7:0) was * changed to indicate a component mask. This is non-zero * in the 3.0 generation. This is actually DF_FIDMASK_V2. * * o v2.0 -> This is just the not that case. Presumably v1 wasn't part * of the Zen generation. * * Because we don't know what version we are yet, we do not use the normal * versioned register accesses which would check what DF version we are and * would want to use the normal indirect register accesses (which also require * us to know the version). We instead do direct broadcast reads. */ static void amdzen_determine_df_vers(amdzen_t *azn, amdzen_df_t *df) { uint32_t val; df_reg_def_t rd = DF_FBICNT; val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); df->adf_major = DF_FBICNT_V4_GET_MAJOR(val); df->adf_minor = DF_FBICNT_V4_GET_MINOR(val); if (df->adf_major == 0 && df->adf_minor == 0) { rd = DF_FIDMASK0_V3P5; val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); if (bitx32(val, 7, 0) != 0) { df->adf_major = 3; df->adf_minor = 5; df->adf_rev = DF_REV_3P5; } else { rd = DF_FIDMASK_V2; val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); if (bitx32(val, 7, 0) != 0) { df->adf_major = 3; df->adf_minor = 0; df->adf_rev = DF_REV_3; } else { df->adf_major = 2; df->adf_minor = 0; df->adf_rev = DF_REV_2; } } } else if (df->adf_major == 4 && df->adf_minor == 0) { df->adf_rev = DF_REV_4; } else { df->adf_rev = DF_REV_UNKNOWN; } } /* * All of the different versions of the DF have different ways of getting at and * answering the question of how do I break a fabric ID into a corresponding * socket, die, and component. Importantly the goal here is to obtain, cache, * and normalize: * * o The DF System Configuration * o The various Mask registers * o The Node ID */ static void amdzen_determine_fabric_decomp(amdzen_t *azn, amdzen_df_t *df) { uint32_t mask; df_fabric_decomp_t *decomp = &df->adf_decomp; switch (df->adf_rev) { case DF_REV_2: df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V2); switch (DF_SYSCFG_V2_GET_MY_TYPE(df->adf_syscfg)) { case DF_DIE_TYPE_CPU: mask = amdzen_df_read32_bcast(azn, df, DF_DIEMASK_CPU_V2); break; case DF_DIE_TYPE_APU: mask = amdzen_df_read32_bcast(azn, df, DF_DIEMASK_APU_V2); break; default: panic("DF thinks we're not on a CPU!"); } df->adf_mask0 = mask; /* * DFv2 is a bit different in how the fabric mask register is * phrased. Logically a fabric ID is broken into something that * uniquely identifies a "node" (a particular die on a socket) * and something that identifies a "component", e.g. a memory * controller. * * Starting with DFv3, these registers logically called out how * to separate the fabric ID first into a node and a component. * Then the node was then broken down into a socket and die. In * DFv2, there is no separate mask and shift of a node. Instead * the socket and die are absolute offsets into the fabric ID * rather than relative offsets into the node ID. As such, when * we encounter DFv2, we fake up a node mask and shift and make * it look like DFv3+. */ decomp->dfd_node_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) | DF_DIEMASK_V2_GET_DIE_MASK(mask); decomp->dfd_node_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask); decomp->dfd_comp_mask = DF_DIEMASK_V2_GET_COMP_MASK(mask); decomp->dfd_comp_shift = 0; decomp->dfd_sock_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) >> decomp->dfd_node_shift; decomp->dfd_die_mask = DF_DIEMASK_V2_GET_DIE_MASK(mask) >> decomp->dfd_node_shift; decomp->dfd_sock_shift = DF_DIEMASK_V2_GET_SOCK_SHIFT(mask) - decomp->dfd_node_shift; decomp->dfd_die_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask) - decomp->dfd_node_shift; ASSERT3U(decomp->dfd_die_shift, ==, 0); break; case DF_REV_3: df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3); df->adf_mask0 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK0_V3); df->adf_mask1 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK1_V3); decomp->dfd_sock_mask = DF_FIDMASK1_V3_GET_SOCK_MASK(df->adf_mask1); decomp->dfd_sock_shift = DF_FIDMASK1_V3_GET_SOCK_SHIFT(df->adf_mask1); decomp->dfd_die_mask = DF_FIDMASK1_V3_GET_DIE_MASK(df->adf_mask1); decomp->dfd_die_shift = 0; decomp->dfd_node_mask = DF_FIDMASK0_V3_GET_NODE_MASK(df->adf_mask0); decomp->dfd_node_shift = DF_FIDMASK1_V3_GET_NODE_SHIFT(df->adf_mask1); decomp->dfd_comp_mask = DF_FIDMASK0_V3_GET_COMP_MASK(df->adf_mask0); decomp->dfd_comp_shift = 0; break; case DF_REV_3P5: df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3P5); df->adf_mask0 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK0_V3P5); df->adf_mask1 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK1_V3P5); df->adf_mask2 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK2_V3P5); decomp->dfd_sock_mask = DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); decomp->dfd_sock_shift = DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); decomp->dfd_die_mask = DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); decomp->dfd_die_shift = 0; decomp->dfd_node_mask = DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); decomp->dfd_node_shift = DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); decomp->dfd_comp_mask = DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); decomp->dfd_comp_shift = 0; break; case DF_REV_4: df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V4); df->adf_mask0 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK0_V4); df->adf_mask1 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK1_V4); df->adf_mask2 = amdzen_df_read32_bcast(azn, df, DF_FIDMASK2_V4); /* * The DFv4 registers are at a different location in the DF; * however, the actual layout of fields is the same as DFv3.5. * This is why you see V3P5 below. */ decomp->dfd_sock_mask = DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); decomp->dfd_sock_shift = DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); decomp->dfd_die_mask = DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); decomp->dfd_die_shift = 0; decomp->dfd_node_mask = DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); decomp->dfd_node_shift = DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); decomp->dfd_comp_mask = DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); decomp->dfd_comp_shift = 0; break; default: panic("encountered suspicious, previously rejected DF " "rev: 0x%x", df->adf_rev); } } /* * Initialize our knowledge about a given series of nodes on the data fabric. */ static void amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) { uint_t i; uint32_t val; amdzen_determine_df_vers(azn, df); switch (df->adf_rev) { case DF_REV_2: case DF_REV_3: case DF_REV_3P5: val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V2); break; case DF_REV_4: val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V4); break; default: dev_err(azn->azn_dip, CE_WARN, "encountered unsupported DF " "revision: 0x%x", df->adf_rev); return; } df->adf_nb_busno = DF_CFG_ADDR_CTL_GET_BUS_NUM(val); val = amdzen_df_read32_bcast(azn, df, DF_FBICNT); df->adf_nents = DF_FBICNT_GET_COUNT(val); if (df->adf_nents == 0) return; df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, KM_SLEEP); for (i = 0; i < df->adf_nents; i++) { amdzen_df_ent_t *dfe = &df->adf_ents[i]; uint8_t inst = i; /* * Unfortunately, Rome uses a discontinuous instance ID pattern * while everything else we can find uses a contiguous instance * ID pattern. This means that for Rome, we need to adjust the * indexes that we iterate over, though the total number of * entries is right. This was carried over into Milan, but not * Genoa. */ if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { if (inst > ARRAY_SIZE(amdzen_df_rome_ids)) { dev_err(azn->azn_dip, CE_WARN, "Rome family " "processor reported more ids than the PPR, " "resetting %u to instance zero", inst); inst = 0; } else { inst = amdzen_df_rome_ids[inst]; } } dfe->adfe_drvid = inst; dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, DF_FBIINFO0); dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, DF_FBIINFO1); dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, DF_FBIINFO2); dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, DF_FBIINFO3); dfe->adfe_type = DF_FBIINFO0_GET_TYPE(dfe->adfe_info0); dfe->adfe_subtype = DF_FBIINFO0_GET_SUBTYPE(dfe->adfe_info0); /* * The enabled flag was not present in Zen 1. Simulate it by * checking for a non-zero register instead. */ if (DF_FBIINFO0_V3_GET_ENABLED(dfe->adfe_info0) || (df->adf_rev == DF_REV_2 && dfe->adfe_info0 != 0)) { dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; } if (DF_FBIINFO0_GET_HAS_MCA(dfe->adfe_info0)) { dfe->adfe_flags |= AMDZEN_DFE_F_MCA; } dfe->adfe_inst_id = DF_FBIINFO3_GET_INSTID(dfe->adfe_info3); switch (df->adf_rev) { case DF_REV_2: dfe->adfe_fabric_id = DF_FBIINFO3_V2_GET_BLOCKID(dfe->adfe_info3); break; case DF_REV_3: dfe->adfe_fabric_id = DF_FBIINFO3_V3_GET_BLOCKID(dfe->adfe_info3); break; case DF_REV_3P5: dfe->adfe_fabric_id = DF_FBIINFO3_V3P5_GET_BLOCKID(dfe->adfe_info3); break; case DF_REV_4: dfe->adfe_fabric_id = DF_FBIINFO3_V4_GET_BLOCKID(dfe->adfe_info3); break; default: panic("encountered suspicious, previously rejected DF " "rev: 0x%x", df->adf_rev); } } amdzen_determine_fabric_decomp(azn, df); } static void amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) { amdzen_stub_t *stub; for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; stub = list_next(&azn->azn_nb_stubs, stub)) { if (stub->azns_bus == df->adf_nb_busno) { df->adf_flags |= AMDZEN_DF_F_FOUND_NB; df->adf_nb = stub; return; } } } static void amdzen_nexus_init(void *arg) { uint_t i; amdzen_t *azn = arg; /* * First go through all of the stubs and assign the DF entries. */ mutex_enter(&azn->azn_mutex); if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { azn->azn_flags |= AMDZEN_F_MAP_ERROR; goto done; } for (i = 0; i < AMDZEN_MAX_DFS; i++) { amdzen_df_t *df = &azn->azn_dfs[i]; if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) continue; amdzen_setup_df(azn, df); amdzen_find_nb(azn, df); } /* * Not all children may be installed. As such, we do not treat the * failure of a child as fatal to the driver. */ mutex_exit(&azn->azn_mutex); for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { (void) amdzen_create_child(azn, &amdzen_children[i]); } mutex_enter(&azn->azn_mutex); done: azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; azn->azn_taskqid = TASKQID_INVALID; cv_broadcast(&azn->azn_cv); mutex_exit(&azn->azn_mutex); } static int amdzen_stub_scan_cb(dev_info_t *dip, void *arg) { amdzen_t *azn = arg; uint16_t vid, did; int *regs; uint_t nregs, i; boolean_t match = B_FALSE; if (dip == ddi_root_node()) { return (DDI_WALK_CONTINUE); } /* * If a node in question is not a pci node, then we have no interest in * it as all the stubs that we care about are related to pci devices. */ if (strncmp("pci", ddi_get_name(dip), 3) != 0) { return (DDI_WALK_PRUNECHILD); } /* * If we can't get a device or vendor ID and prove that this is an AMD * part, then we don't care about it. */ vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "vendor-id", PCI_EINVAL16); did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "device-id", PCI_EINVAL16); if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { return (DDI_WALK_CONTINUE); } if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { return (DDI_WALK_CONTINUE); } for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { if (amdzen_nb_ids[i] == did) { match = B_TRUE; } } if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { return (DDI_WALK_CONTINUE); } if (nregs == 0) { ddi_prop_free(regs); return (DDI_WALK_CONTINUE); } if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { match = B_TRUE; } ddi_prop_free(regs); if (match) { mutex_enter(&azn->azn_mutex); azn->azn_nscanned++; mutex_exit(&azn->azn_mutex); } return (DDI_WALK_CONTINUE); } static void amdzen_stub_scan(void *arg) { amdzen_t *azn = arg; mutex_enter(&azn->azn_mutex); azn->azn_nscanned = 0; mutex_exit(&azn->azn_mutex); ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); mutex_enter(&azn->azn_mutex); azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; if (azn->azn_nscanned == 0) { azn->azn_flags |= AMDZEN_F_UNSUPPORTED; azn->azn_taskqid = TASKQID_INVALID; cv_broadcast(&azn->azn_cv); } else if (azn->azn_npresent == azn->azn_nscanned) { azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_nexus_init, azn, TQ_SLEEP); } mutex_exit(&azn->azn_mutex); } /* * Unfortunately we can't really let the stubs detach as we may need them to be * available for client operations. We may be able to improve this if we know * that the actual nexus is going away. However, as long as it's active, we need * all the stubs. */ int amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) { if (cmd == DDI_SUSPEND) { return (DDI_SUCCESS); } return (DDI_FAILURE); } int amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) { int *regs, reg; uint_t nregs, i; uint16_t vid, did; amdzen_stub_t *stub; amdzen_t *azn = amdzen_data; boolean_t valid = B_FALSE; boolean_t nb = B_FALSE; if (cmd == DDI_RESUME) { return (DDI_SUCCESS); } else if (cmd != DDI_ATTACH) { return (DDI_FAILURE); } /* * Make sure that the stub that we've been asked to attach is a pci type * device. If not, then there is no reason for us to proceed. */ if (strncmp("pci", ddi_get_name(dip), 3) != 0) { dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " "stub: %s", ddi_get_name(dip)); return (DDI_FAILURE); } vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "vendor-id", PCI_EINVAL16); did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "device-id", PCI_EINVAL16); if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { dev_err(dip, CE_WARN, "failed to get PCI ID properties"); return (DDI_FAILURE); } if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); return (DDI_FAILURE); } if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { dev_err(dip, CE_WARN, "failed to get 'reg' property"); return (DDI_FAILURE); } if (nregs == 0) { ddi_prop_free(regs); dev_err(dip, CE_WARN, "missing 'reg' property values"); return (DDI_FAILURE); } reg = *regs; ddi_prop_free(regs); for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { if (amdzen_nb_ids[i] == did) { valid = B_TRUE; nb = B_TRUE; } } if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { valid = B_TRUE; nb = B_FALSE; } if (!valid) { dev_err(dip, CE_WARN, "device %s didn't match the nexus list", ddi_get_name(dip)); return (DDI_FAILURE); } stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { dev_err(dip, CE_WARN, "failed to set up config space"); kmem_free(stub, sizeof (amdzen_stub_t)); return (DDI_FAILURE); } stub->azns_dip = dip; stub->azns_vid = vid; stub->azns_did = did; stub->azns_bus = PCI_REG_BUS_G(reg); stub->azns_dev = PCI_REG_DEV_G(reg); stub->azns_func = PCI_REG_FUNC_G(reg); ddi_set_driver_private(dip, stub); mutex_enter(&azn->azn_mutex); azn->azn_npresent++; if (nb) { list_insert_tail(&azn->azn_nb_stubs, stub); } else { list_insert_tail(&azn->azn_df_stubs, stub); } if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && azn->azn_nscanned == azn->azn_npresent) { azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_nexus_init, azn, TQ_SLEEP); } mutex_exit(&azn->azn_mutex); return (DDI_SUCCESS); } static int amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, void *arg, void *result) { char buf[32]; dev_info_t *child; const amdzen_child_data_t *acd; switch (ctlop) { case DDI_CTLOPS_REPORTDEV: if (rdip == NULL) { return (DDI_FAILURE); } cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", ddi_node_name(rdip), ddi_get_name_addr(rdip), ddi_driver_name(rdip), ddi_get_instance(rdip)); break; case DDI_CTLOPS_INITCHILD: child = arg; if (child == NULL) { dev_err(dip, CE_WARN, "!no child passed for " "DDI_CTLOPS_INITCHILD"); } acd = ddi_get_parent_data(child); if (acd == NULL) { dev_err(dip, CE_WARN, "!missing child parent data"); return (DDI_FAILURE); } if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= sizeof (buf)) { dev_err(dip, CE_WARN, "!failed to construct device " "addr due to overflow"); return (DDI_FAILURE); } ddi_set_name_addr(child, buf); break; case DDI_CTLOPS_UNINITCHILD: child = arg; if (child == NULL) { dev_err(dip, CE_WARN, "!no child passed for " "DDI_CTLOPS_UNINITCHILD"); } ddi_set_name_addr(child, NULL); break; default: return (ddi_ctlops(dip, rdip, ctlop, arg, result)); } return (DDI_SUCCESS); } static int amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { amdzen_t *azn = amdzen_data; if (cmd == DDI_RESUME) { return (DDI_SUCCESS); } else if (cmd != DDI_ATTACH) { return (DDI_FAILURE); } mutex_enter(&azn->azn_mutex); if (azn->azn_dip != NULL) { dev_err(dip, CE_WARN, "driver is already attached!"); mutex_exit(&azn->azn_mutex); return (DDI_FAILURE); } azn->azn_dip = dip; azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, azn, TQ_SLEEP); azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; mutex_exit(&azn->azn_mutex); return (DDI_SUCCESS); } static int amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { amdzen_t *azn = amdzen_data; if (cmd == DDI_SUSPEND) { return (DDI_SUCCESS); } else if (cmd != DDI_DETACH) { return (DDI_FAILURE); } mutex_enter(&azn->azn_mutex); while (azn->azn_taskqid != TASKQID_INVALID) { cv_wait(&azn->azn_cv, &azn->azn_mutex); } /* * If we've attached any stub drivers, e.g. this platform is important * for us, then we fail detach. */ if (!list_is_empty(&azn->azn_df_stubs) || !list_is_empty(&azn->azn_nb_stubs)) { mutex_exit(&azn->azn_mutex); return (DDI_FAILURE); } azn->azn_dip = NULL; mutex_exit(&azn->azn_mutex); return (DDI_SUCCESS); } static void amdzen_free(void) { if (amdzen_data == NULL) { return; } VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); list_destroy(&amdzen_data->azn_df_stubs); VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); list_destroy(&amdzen_data->azn_nb_stubs); cv_destroy(&amdzen_data->azn_cv); mutex_destroy(&amdzen_data->azn_mutex); kmem_free(amdzen_data, sizeof (amdzen_t)); amdzen_data = NULL; } static void amdzen_alloc(void) { amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), offsetof(amdzen_stub_t, azns_link)); list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), offsetof(amdzen_stub_t, azns_link)); cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); } struct bus_ops amdzen_bus_ops = { .busops_rev = BUSO_REV, .bus_map = nullbusmap, .bus_dma_map = ddi_no_dma_map, .bus_dma_allochdl = ddi_no_dma_allochdl, .bus_dma_freehdl = ddi_no_dma_freehdl, .bus_dma_bindhdl = ddi_no_dma_bindhdl, .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, .bus_dma_flush = ddi_no_dma_flush, .bus_dma_win = ddi_no_dma_win, .bus_dma_ctl = ddi_no_dma_mctl, .bus_prop_op = ddi_bus_prop_op, .bus_ctl = amdzen_bus_ctl }; static struct dev_ops amdzen_dev_ops = { .devo_rev = DEVO_REV, .devo_refcnt = 0, .devo_getinfo = nodev, .devo_identify = nulldev, .devo_probe = nulldev, .devo_attach = amdzen_attach, .devo_detach = amdzen_detach, .devo_reset = nodev, .devo_quiesce = ddi_quiesce_not_needed, .devo_bus_ops = &amdzen_bus_ops }; static struct modldrv amdzen_modldrv = { .drv_modops = &mod_driverops, .drv_linkinfo = "AMD Zen Nexus Driver", .drv_dev_ops = &amdzen_dev_ops }; static struct modlinkage amdzen_modlinkage = { .ml_rev = MODREV_1, .ml_linkage = { &amdzen_modldrv, NULL } }; int _init(void) { int ret; if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { return (ENOTSUP); } if ((ret = mod_install(&amdzen_modlinkage)) == 0) { amdzen_alloc(); } return (ret); } int _info(struct modinfo *modinfop) { return (mod_info(&amdzen_modlinkage, modinfop)); } int _fini(void) { int ret; if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { amdzen_free(); } return (ret); }