1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * AMD Address Translation Library 4 * 5 * internal.h : Helper functions and common defines 6 * 7 * Copyright (c) 2023, Advanced Micro Devices, Inc. 8 * All Rights Reserved. 9 * 10 * Author: Yazen Ghannam <Yazen.Ghannam@amd.com> 11 */ 12 13 #ifndef __AMD_ATL_INTERNAL_H__ 14 #define __AMD_ATL_INTERNAL_H__ 15 16 #include <linux/bitfield.h> 17 #include <linux/bitops.h> 18 #include <linux/ras.h> 19 20 #include <asm/amd_nb.h> 21 22 #include "reg_fields.h" 23 24 #undef pr_fmt 25 #define pr_fmt(fmt) "amd_atl: " fmt 26 27 /* Maximum possible number of Coherent Stations within a single Data Fabric. */ 28 #define MAX_COH_ST_CHANNELS 32 29 30 /* PCI ID for Zen4 Server DF Function 0. */ 31 #define DF_FUNC0_ID_ZEN4_SERVER 0x14AD1022 32 33 /* PCI IDs for MI300 DF Function 0. */ 34 #define DF_FUNC0_ID_MI300 0x15281022 35 36 /* Shift needed for adjusting register values to true values. */ 37 #define DF_DRAM_BASE_LIMIT_LSB 28 38 #define MI300_DRAM_LIMIT_LSB 20 39 40 enum df_revisions { 41 UNKNOWN, 42 DF2, 43 DF3, 44 DF3p5, 45 DF4, 46 DF4p5, 47 }; 48 49 /* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */ 50 enum intlv_modes { 51 NONE = 0x00, 52 NOHASH_2CHAN = 0x01, 53 NOHASH_4CHAN = 0x03, 54 NOHASH_8CHAN = 0x05, 55 DF3_6CHAN = 0x06, 56 NOHASH_16CHAN = 0x07, 57 NOHASH_32CHAN = 0x08, 58 DF3_COD4_2CHAN_HASH = 0x0C, 59 DF3_COD2_4CHAN_HASH = 0x0D, 60 DF3_COD1_8CHAN_HASH = 0x0E, 61 DF4_NPS4_2CHAN_HASH = 0x10, 62 DF4_NPS2_4CHAN_HASH = 0x11, 63 DF4_NPS1_8CHAN_HASH = 0x12, 64 DF4_NPS4_3CHAN_HASH = 0x13, 65 DF4_NPS2_6CHAN_HASH = 0x14, 66 DF4_NPS1_12CHAN_HASH = 0x15, 67 DF4_NPS2_5CHAN_HASH = 0x16, 68 DF4_NPS1_10CHAN_HASH = 0x17, 69 MI3_HASH_8CHAN = 0x18, 70 MI3_HASH_16CHAN = 0x19, 71 MI3_HASH_32CHAN = 0x1A, 72 DF2_2CHAN_HASH = 0x21, 73 /* DF4.5 modes are all IntLvNumChan + 0x20 */ 74 DF4p5_NPS1_16CHAN_1K_HASH = 0x2C, 75 DF4p5_NPS0_24CHAN_1K_HASH = 0x2E, 76 DF4p5_NPS4_2CHAN_1K_HASH = 0x30, 77 DF4p5_NPS2_4CHAN_1K_HASH = 0x31, 78 DF4p5_NPS1_8CHAN_1K_HASH = 0x32, 79 DF4p5_NPS4_3CHAN_1K_HASH = 0x33, 80 DF4p5_NPS2_6CHAN_1K_HASH = 0x34, 81 DF4p5_NPS1_12CHAN_1K_HASH = 0x35, 82 DF4p5_NPS2_5CHAN_1K_HASH = 0x36, 83 DF4p5_NPS1_10CHAN_1K_HASH = 0x37, 84 DF4p5_NPS4_2CHAN_2K_HASH = 0x40, 85 DF4p5_NPS2_4CHAN_2K_HASH = 0x41, 86 DF4p5_NPS1_8CHAN_2K_HASH = 0x42, 87 DF4p5_NPS1_16CHAN_2K_HASH = 0x43, 88 DF4p5_NPS4_3CHAN_2K_HASH = 0x44, 89 DF4p5_NPS2_6CHAN_2K_HASH = 0x45, 90 DF4p5_NPS1_12CHAN_2K_HASH = 0x46, 91 DF4p5_NPS0_24CHAN_2K_HASH = 0x47, 92 DF4p5_NPS2_5CHAN_2K_HASH = 0x48, 93 DF4p5_NPS1_10CHAN_2K_HASH = 0x49, 94 }; 95 96 struct df_flags { 97 __u8 legacy_ficaa : 1, 98 socket_id_shift_quirk : 1, 99 heterogeneous : 1, 100 __reserved_0 : 5; 101 }; 102 103 struct df_config { 104 enum df_revisions rev; 105 106 /* 107 * These masks operate on the 16-bit Coherent Station IDs, 108 * e.g. Instance, Fabric, Destination, etc. 109 */ 110 u16 component_id_mask; 111 u16 die_id_mask; 112 u16 node_id_mask; 113 u16 socket_id_mask; 114 115 /* 116 * Least-significant bit of Node ID portion of the 117 * system-wide Coherent Station Fabric ID. 118 */ 119 u8 node_id_shift; 120 121 /* 122 * Least-significant bit of Die portion of the Node ID. 123 * Adjusted to include the Node ID shift in order to apply 124 * to the Coherent Station Fabric ID. 125 */ 126 u8 die_id_shift; 127 128 /* 129 * Least-significant bit of Socket portion of the Node ID. 130 * Adjusted to include the Node ID shift in order to apply 131 * to the Coherent Station Fabric ID. 132 */ 133 u8 socket_id_shift; 134 135 /* Number of DRAM Address maps visible in a Coherent Station. */ 136 u8 num_coh_st_maps; 137 138 u32 dram_hole_base; 139 140 /* Global flags to handle special cases. */ 141 struct df_flags flags; 142 }; 143 144 extern struct df_config df_cfg; 145 146 struct dram_addr_map { 147 /* 148 * Each DRAM Address Map can operate independently 149 * in different interleaving modes. 150 */ 151 enum intlv_modes intlv_mode; 152 153 /* System-wide number for this address map. */ 154 u8 num; 155 156 /* Raw register values */ 157 u32 base; 158 u32 limit; 159 u32 ctl; 160 u32 intlv; 161 162 /* 163 * Logical to Physical Coherent Station Remapping array 164 * 165 * Index: Logical Coherent Station Instance ID 166 * Value: Physical Coherent Station Instance ID 167 * 168 * phys_coh_st_inst_id = remap_array[log_coh_st_inst_id] 169 */ 170 u8 remap_array[MAX_COH_ST_CHANNELS]; 171 172 /* 173 * Number of bits covering DRAM Address map 0 174 * when interleaving is non-power-of-2. 175 * 176 * Used only for DF3_6CHAN. 177 */ 178 u8 np2_bits; 179 180 /* Position of the 'interleave bit'. */ 181 u8 intlv_bit_pos; 182 /* Number of channels interleaved in this map. */ 183 u8 num_intlv_chan; 184 /* Number of dies interleaved in this map. */ 185 u8 num_intlv_dies; 186 /* Number of sockets interleaved in this map. */ 187 u8 num_intlv_sockets; 188 /* 189 * Total number of channels interleaved accounting 190 * for die and socket interleaving. 191 */ 192 u8 total_intlv_chan; 193 /* Total bits needed to cover 'total_intlv_chan'. */ 194 u8 total_intlv_bits; 195 }; 196 197 /* Original input values cached for debug printing. */ 198 struct addr_ctx_inputs { 199 u64 norm_addr; 200 u8 socket_id; 201 u8 die_id; 202 u8 coh_st_inst_id; 203 }; 204 205 struct addr_ctx { 206 u64 ret_addr; 207 208 struct addr_ctx_inputs inputs; 209 struct dram_addr_map map; 210 211 /* AMD Node ID calculated from Socket and Die IDs. */ 212 u8 node_id; 213 214 /* 215 * Coherent Station Instance ID 216 * Local ID used within a 'node'. 217 */ 218 u16 inst_id; 219 220 /* 221 * Coherent Station Fabric ID 222 * System-wide ID that includes 'node' bits. 223 */ 224 u16 coh_st_fabric_id; 225 }; 226 227 int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo); 228 int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo); 229 230 int get_df_system_info(void); 231 int determine_node_id(struct addr_ctx *ctx, u8 socket_num, u8 die_num); 232 int get_addr_hash_mi300(void); 233 234 int get_address_map(struct addr_ctx *ctx); 235 236 int denormalize_address(struct addr_ctx *ctx); 237 int dehash_address(struct addr_ctx *ctx); 238 239 unsigned long norm_to_sys_addr(u8 socket_id, u8 die_id, u8 coh_st_inst_id, unsigned long addr); 240 unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err); 241 242 /* 243 * Make a gap in @data that is @num_bits long starting at @bit_num. 244 * e.g. data = 11111111'b 245 * bit_num = 3 246 * num_bits = 2 247 * result = 1111100111'b 248 */ 249 static inline u64 expand_bits(u8 bit_num, u8 num_bits, u64 data) 250 { 251 u64 temp1, temp2; 252 253 if (!num_bits) 254 return data; 255 256 if (!bit_num) { 257 WARN_ON_ONCE(num_bits >= BITS_PER_LONG); 258 return data << num_bits; 259 } 260 261 WARN_ON_ONCE(bit_num >= BITS_PER_LONG); 262 263 temp1 = data & GENMASK_ULL(bit_num - 1, 0); 264 265 temp2 = data & GENMASK_ULL(63, bit_num); 266 temp2 <<= num_bits; 267 268 return temp1 | temp2; 269 } 270 271 /* 272 * Remove bits in @data between @low_bit and @high_bit inclusive. 273 * e.g. data = XXXYYZZZ'b 274 * low_bit = 3 275 * high_bit = 4 276 * result = XXXZZZ'b 277 */ 278 static inline u64 remove_bits(u8 low_bit, u8 high_bit, u64 data) 279 { 280 u64 temp1, temp2; 281 282 WARN_ON_ONCE(high_bit >= BITS_PER_LONG); 283 WARN_ON_ONCE(low_bit >= BITS_PER_LONG); 284 WARN_ON_ONCE(low_bit > high_bit); 285 286 if (!low_bit) 287 return data >> (high_bit++); 288 289 temp1 = GENMASK_ULL(low_bit - 1, 0) & data; 290 temp2 = GENMASK_ULL(63, high_bit + 1) & data; 291 temp2 >>= high_bit - low_bit + 1; 292 293 return temp1 | temp2; 294 } 295 296 #define atl_debug(ctx, fmt, arg...) \ 297 pr_debug("socket_id=%u die_id=%u coh_st_inst_id=%u norm_addr=0x%016llx: " fmt,\ 298 (ctx)->inputs.socket_id, (ctx)->inputs.die_id,\ 299 (ctx)->inputs.coh_st_inst_id, (ctx)->inputs.norm_addr, ##arg) 300 301 static inline void atl_debug_on_bad_df_rev(void) 302 { 303 pr_debug("Unrecognized DF rev: %u", df_cfg.rev); 304 } 305 306 static inline void atl_debug_on_bad_intlv_mode(struct addr_ctx *ctx) 307 { 308 atl_debug(ctx, "Unrecognized interleave mode: %u", ctx->map.intlv_mode); 309 } 310 311 #endif /* __AMD_ATL_INTERNAL_H__ */ 312