1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * AMD Address Translation Library 4 * 5 * internal.h : Helper functions and common defines 6 * 7 * Copyright (c) 2023, Advanced Micro Devices, Inc. 8 * All Rights Reserved. 9 * 10 * Author: Yazen Ghannam <Yazen.Ghannam@amd.com> 11 */ 12 13 #ifndef __AMD_ATL_INTERNAL_H__ 14 #define __AMD_ATL_INTERNAL_H__ 15 16 #include <linux/bitfield.h> 17 #include <linux/bitops.h> 18 #include <linux/ras.h> 19 20 #include <asm/amd_nb.h> 21 22 #include "reg_fields.h" 23 24 /* Maximum possible number of Coherent Stations within a single Data Fabric. */ 25 #define MAX_COH_ST_CHANNELS 32 26 27 /* PCI ID for Zen4 Server DF Function 0. */ 28 #define DF_FUNC0_ID_ZEN4_SERVER 0x14AD1022 29 30 /* PCI IDs for MI300 DF Function 0. */ 31 #define DF_FUNC0_ID_MI300 0x15281022 32 33 /* Shift needed for adjusting register values to true values. */ 34 #define DF_DRAM_BASE_LIMIT_LSB 28 35 #define MI300_DRAM_LIMIT_LSB 20 36 37 enum df_revisions { 38 UNKNOWN, 39 DF2, 40 DF3, 41 DF3p5, 42 DF4, 43 DF4p5, 44 }; 45 46 /* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */ 47 enum intlv_modes { 48 NONE = 0x00, 49 NOHASH_2CHAN = 0x01, 50 NOHASH_4CHAN = 0x03, 51 NOHASH_8CHAN = 0x05, 52 DF3_6CHAN = 0x06, 53 NOHASH_16CHAN = 0x07, 54 NOHASH_32CHAN = 0x08, 55 DF3_COD4_2CHAN_HASH = 0x0C, 56 DF3_COD2_4CHAN_HASH = 0x0D, 57 DF3_COD1_8CHAN_HASH = 0x0E, 58 DF4_NPS4_2CHAN_HASH = 0x10, 59 DF4_NPS2_4CHAN_HASH = 0x11, 60 DF4_NPS1_8CHAN_HASH = 0x12, 61 DF4_NPS4_3CHAN_HASH = 0x13, 62 DF4_NPS2_6CHAN_HASH = 0x14, 63 DF4_NPS1_12CHAN_HASH = 0x15, 64 DF4_NPS2_5CHAN_HASH = 0x16, 65 DF4_NPS1_10CHAN_HASH = 0x17, 66 MI3_HASH_8CHAN = 0x18, 67 MI3_HASH_16CHAN = 0x19, 68 MI3_HASH_32CHAN = 0x1A, 69 DF2_2CHAN_HASH = 0x21, 70 /* DF4.5 modes are all IntLvNumChan + 0x20 */ 71 DF4p5_NPS1_16CHAN_1K_HASH = 0x2C, 72 DF4p5_NPS0_24CHAN_1K_HASH = 0x2E, 73 DF4p5_NPS4_2CHAN_1K_HASH = 0x30, 74 DF4p5_NPS2_4CHAN_1K_HASH = 0x31, 75 DF4p5_NPS1_8CHAN_1K_HASH = 0x32, 76 DF4p5_NPS4_3CHAN_1K_HASH = 0x33, 77 DF4p5_NPS2_6CHAN_1K_HASH = 0x34, 78 DF4p5_NPS1_12CHAN_1K_HASH = 0x35, 79 DF4p5_NPS2_5CHAN_1K_HASH = 0x36, 80 DF4p5_NPS1_10CHAN_1K_HASH = 0x37, 81 DF4p5_NPS4_2CHAN_2K_HASH = 0x40, 82 DF4p5_NPS2_4CHAN_2K_HASH = 0x41, 83 DF4p5_NPS1_8CHAN_2K_HASH = 0x42, 84 DF4p5_NPS1_16CHAN_2K_HASH = 0x43, 85 DF4p5_NPS4_3CHAN_2K_HASH = 0x44, 86 DF4p5_NPS2_6CHAN_2K_HASH = 0x45, 87 DF4p5_NPS1_12CHAN_2K_HASH = 0x46, 88 DF4p5_NPS0_24CHAN_2K_HASH = 0x47, 89 DF4p5_NPS2_5CHAN_2K_HASH = 0x48, 90 DF4p5_NPS1_10CHAN_2K_HASH = 0x49, 91 }; 92 93 struct df_flags { 94 __u8 legacy_ficaa : 1, 95 socket_id_shift_quirk : 1, 96 heterogeneous : 1, 97 __reserved_0 : 5; 98 }; 99 100 struct df_config { 101 enum df_revisions rev; 102 103 /* 104 * These masks operate on the 16-bit Coherent Station IDs, 105 * e.g. Instance, Fabric, Destination, etc. 106 */ 107 u16 component_id_mask; 108 u16 die_id_mask; 109 u16 node_id_mask; 110 u16 socket_id_mask; 111 112 /* 113 * Least-significant bit of Node ID portion of the 114 * system-wide Coherent Station Fabric ID. 115 */ 116 u8 node_id_shift; 117 118 /* 119 * Least-significant bit of Die portion of the Node ID. 120 * Adjusted to include the Node ID shift in order to apply 121 * to the Coherent Station Fabric ID. 122 */ 123 u8 die_id_shift; 124 125 /* 126 * Least-significant bit of Socket portion of the Node ID. 127 * Adjusted to include the Node ID shift in order to apply 128 * to the Coherent Station Fabric ID. 129 */ 130 u8 socket_id_shift; 131 132 /* Number of DRAM Address maps visible in a Coherent Station. */ 133 u8 num_coh_st_maps; 134 135 /* Global flags to handle special cases. */ 136 struct df_flags flags; 137 }; 138 139 extern struct df_config df_cfg; 140 141 struct dram_addr_map { 142 /* 143 * Each DRAM Address Map can operate independently 144 * in different interleaving modes. 145 */ 146 enum intlv_modes intlv_mode; 147 148 /* System-wide number for this address map. */ 149 u8 num; 150 151 /* Raw register values */ 152 u32 base; 153 u32 limit; 154 u32 ctl; 155 u32 intlv; 156 157 /* 158 * Logical to Physical Coherent Station Remapping array 159 * 160 * Index: Logical Coherent Station Instance ID 161 * Value: Physical Coherent Station Instance ID 162 * 163 * phys_coh_st_inst_id = remap_array[log_coh_st_inst_id] 164 */ 165 u8 remap_array[MAX_COH_ST_CHANNELS]; 166 167 /* 168 * Number of bits covering DRAM Address map 0 169 * when interleaving is non-power-of-2. 170 * 171 * Used only for DF3_6CHAN. 172 */ 173 u8 np2_bits; 174 175 /* Position of the 'interleave bit'. */ 176 u8 intlv_bit_pos; 177 /* Number of channels interleaved in this map. */ 178 u8 num_intlv_chan; 179 /* Number of dies interleaved in this map. */ 180 u8 num_intlv_dies; 181 /* Number of sockets interleaved in this map. */ 182 u8 num_intlv_sockets; 183 /* 184 * Total number of channels interleaved accounting 185 * for die and socket interleaving. 186 */ 187 u8 total_intlv_chan; 188 /* Total bits needed to cover 'total_intlv_chan'. */ 189 u8 total_intlv_bits; 190 }; 191 192 /* Original input values cached for debug printing. */ 193 struct addr_ctx_inputs { 194 u64 norm_addr; 195 u8 socket_id; 196 u8 die_id; 197 u8 coh_st_inst_id; 198 }; 199 200 struct addr_ctx { 201 u64 ret_addr; 202 203 struct addr_ctx_inputs inputs; 204 struct dram_addr_map map; 205 206 /* AMD Node ID calculated from Socket and Die IDs. */ 207 u8 node_id; 208 209 /* 210 * Coherent Station Instance ID 211 * Local ID used within a 'node'. 212 */ 213 u16 inst_id; 214 215 /* 216 * Coherent Station Fabric ID 217 * System-wide ID that includes 'node' bits. 218 */ 219 u16 coh_st_fabric_id; 220 }; 221 222 int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo); 223 int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo); 224 225 int get_df_system_info(void); 226 int determine_node_id(struct addr_ctx *ctx, u8 socket_num, u8 die_num); 227 int get_addr_hash_mi300(void); 228 229 int get_address_map(struct addr_ctx *ctx); 230 231 int denormalize_address(struct addr_ctx *ctx); 232 int dehash_address(struct addr_ctx *ctx); 233 234 unsigned long norm_to_sys_addr(u8 socket_id, u8 die_id, u8 coh_st_inst_id, unsigned long addr); 235 unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err); 236 237 /* 238 * Make a gap in @data that is @num_bits long starting at @bit_num. 239 * e.g. data = 11111111'b 240 * bit_num = 3 241 * num_bits = 2 242 * result = 1111100111'b 243 */ 244 static inline u64 expand_bits(u8 bit_num, u8 num_bits, u64 data) 245 { 246 u64 temp1, temp2; 247 248 if (!num_bits) 249 return data; 250 251 if (!bit_num) { 252 WARN_ON_ONCE(num_bits >= BITS_PER_LONG); 253 return data << num_bits; 254 } 255 256 WARN_ON_ONCE(bit_num >= BITS_PER_LONG); 257 258 temp1 = data & GENMASK_ULL(bit_num - 1, 0); 259 260 temp2 = data & GENMASK_ULL(63, bit_num); 261 temp2 <<= num_bits; 262 263 return temp1 | temp2; 264 } 265 266 /* 267 * Remove bits in @data between @low_bit and @high_bit inclusive. 268 * e.g. data = XXXYYZZZ'b 269 * low_bit = 3 270 * high_bit = 4 271 * result = XXXZZZ'b 272 */ 273 static inline u64 remove_bits(u8 low_bit, u8 high_bit, u64 data) 274 { 275 u64 temp1, temp2; 276 277 WARN_ON_ONCE(high_bit >= BITS_PER_LONG); 278 WARN_ON_ONCE(low_bit >= BITS_PER_LONG); 279 WARN_ON_ONCE(low_bit > high_bit); 280 281 if (!low_bit) 282 return data >> (high_bit++); 283 284 temp1 = GENMASK_ULL(low_bit - 1, 0) & data; 285 temp2 = GENMASK_ULL(63, high_bit + 1) & data; 286 temp2 >>= high_bit - low_bit + 1; 287 288 return temp1 | temp2; 289 } 290 291 #define atl_debug(ctx, fmt, arg...) \ 292 pr_debug("socket_id=%u die_id=%u coh_st_inst_id=%u norm_addr=0x%016llx: " fmt,\ 293 (ctx)->inputs.socket_id, (ctx)->inputs.die_id,\ 294 (ctx)->inputs.coh_st_inst_id, (ctx)->inputs.norm_addr, ##arg) 295 296 static inline void atl_debug_on_bad_df_rev(void) 297 { 298 pr_debug("Unrecognized DF rev: %u", df_cfg.rev); 299 } 300 301 static inline void atl_debug_on_bad_intlv_mode(struct addr_ctx *ctx) 302 { 303 atl_debug(ctx, "Unrecognized interleave mode: %u", ctx->map.intlv_mode); 304 } 305 306 #endif /* __AMD_ATL_INTERNAL_H__ */ 307