1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * AMD Address Translation Library 4 * 5 * umc.c : Unified Memory Controller (UMC) topology helpers 6 * 7 * Copyright (c) 2023, Advanced Micro Devices, Inc. 8 * All Rights Reserved. 9 * 10 * Author: Yazen Ghannam <Yazen.Ghannam@amd.com> 11 */ 12 13 #include "internal.h" 14 15 /* 16 * MI300 has a fixed, model-specific mapping between a UMC instance and 17 * its related Data Fabric Coherent Station instance. 18 * 19 * The MCA_IPID_UMC[InstanceId] field holds a unique identifier for the 20 * UMC instance within a Node. Use this to find the appropriate Coherent 21 * Station ID. 22 * 23 * Redundant bits were removed from the map below. 24 */ 25 static const u16 umc_coh_st_map[32] = { 26 0x393, 0x293, 0x193, 0x093, 27 0x392, 0x292, 0x192, 0x092, 28 0x391, 0x291, 0x191, 0x091, 29 0x390, 0x290, 0x190, 0x090, 30 0x793, 0x693, 0x593, 0x493, 31 0x792, 0x692, 0x592, 0x492, 32 0x791, 0x691, 0x591, 0x491, 33 0x790, 0x690, 0x590, 0x490, 34 }; 35 36 #define UMC_ID_MI300 GENMASK(23, 12) 37 static u8 get_coh_st_inst_id_mi300(struct atl_err *err) 38 { 39 u16 umc_id = FIELD_GET(UMC_ID_MI300, err->ipid); 40 u8 i; 41 42 for (i = 0; i < ARRAY_SIZE(umc_coh_st_map); i++) { 43 if (umc_id == umc_coh_st_map[i]) 44 break; 45 } 46 47 WARN_ON_ONCE(i >= ARRAY_SIZE(umc_coh_st_map)); 48 49 return i; 50 } 51 52 53 struct xor_bits { 54 bool xor_enable; 55 u16 col_xor; 56 u32 row_xor; 57 }; 58 59 #define NUM_BANK_BITS 4 60 #define NUM_COL_BITS 5 61 #define NUM_SID_BITS 2 62 63 static struct { 64 /* UMC::CH::AddrHashBank */ 65 struct xor_bits bank[NUM_BANK_BITS]; 66 67 /* UMC::CH::AddrHashPC */ 68 struct xor_bits pc; 69 70 /* UMC::CH::AddrHashPC2 */ 71 u8 bank_xor; 72 } addr_hash; 73 74 static struct { 75 u8 bank[NUM_BANK_BITS]; 76 u8 col[NUM_COL_BITS]; 77 u8 sid[NUM_SID_BITS]; 78 u8 num_row_lo; 79 u8 num_row_hi; 80 u8 row_lo; 81 u8 row_hi; 82 u8 pc; 83 } bit_shifts; 84 85 #define MI300_UMC_CH_BASE 0x90000 86 #define MI300_ADDR_CFG (MI300_UMC_CH_BASE + 0x30) 87 #define MI300_ADDR_SEL (MI300_UMC_CH_BASE + 0x40) 88 #define MI300_COL_SEL_LO (MI300_UMC_CH_BASE + 0x50) 89 #define MI300_ADDR_SEL_2 (MI300_UMC_CH_BASE + 0xA4) 90 #define MI300_ADDR_HASH_BANK0 (MI300_UMC_CH_BASE + 0xC8) 91 #define MI300_ADDR_HASH_PC (MI300_UMC_CH_BASE + 0xE0) 92 #define MI300_ADDR_HASH_PC2 (MI300_UMC_CH_BASE + 0xE4) 93 94 #define ADDR_HASH_XOR_EN BIT(0) 95 #define ADDR_HASH_COL_XOR GENMASK(13, 1) 96 #define ADDR_HASH_ROW_XOR GENMASK(31, 14) 97 #define ADDR_HASH_BANK_XOR GENMASK(5, 0) 98 99 #define ADDR_CFG_NUM_ROW_LO GENMASK(11, 8) 100 #define ADDR_CFG_NUM_ROW_HI GENMASK(15, 12) 101 102 #define ADDR_SEL_BANK0 GENMASK(3, 0) 103 #define ADDR_SEL_BANK1 GENMASK(7, 4) 104 #define ADDR_SEL_BANK2 GENMASK(11, 8) 105 #define ADDR_SEL_BANK3 GENMASK(15, 12) 106 #define ADDR_SEL_BANK4 GENMASK(20, 16) 107 #define ADDR_SEL_ROW_LO GENMASK(27, 24) 108 #define ADDR_SEL_ROW_HI GENMASK(31, 28) 109 110 #define COL_SEL_LO_COL0 GENMASK(3, 0) 111 #define COL_SEL_LO_COL1 GENMASK(7, 4) 112 #define COL_SEL_LO_COL2 GENMASK(11, 8) 113 #define COL_SEL_LO_COL3 GENMASK(15, 12) 114 #define COL_SEL_LO_COL4 GENMASK(19, 16) 115 116 #define ADDR_SEL_2_BANK5 GENMASK(4, 0) 117 #define ADDR_SEL_2_CHAN GENMASK(15, 12) 118 119 /* 120 * Read UMC::CH::AddrHash{Bank,PC,PC2} registers to get XOR bits used 121 * for hashing. 122 * 123 * Also, read UMC::CH::Addr{Cfg,Sel,Sel2} and UMC::CH:ColSelLo registers to 124 * get the values needed to reconstruct the normalized address. Apply additional 125 * offsets to the raw register values, as needed. 126 * 127 * Do this during module init, since the values will not change during run time. 128 * 129 * These registers are instantiated for each UMC across each AMD Node. 130 * However, they should be identically programmed due to the fixed hardware 131 * design of MI300 systems. So read the values from Node 0 UMC 0 and keep a 132 * single global structure for simplicity. 133 */ 134 int get_umc_info_mi300(void) 135 { 136 u32 temp; 137 int ret; 138 u8 i; 139 140 for (i = 0; i < NUM_BANK_BITS; i++) { 141 ret = amd_smn_read(0, MI300_ADDR_HASH_BANK0 + (i * 4), &temp); 142 if (ret) 143 return ret; 144 145 addr_hash.bank[i].xor_enable = FIELD_GET(ADDR_HASH_XOR_EN, temp); 146 addr_hash.bank[i].col_xor = FIELD_GET(ADDR_HASH_COL_XOR, temp); 147 addr_hash.bank[i].row_xor = FIELD_GET(ADDR_HASH_ROW_XOR, temp); 148 } 149 150 ret = amd_smn_read(0, MI300_ADDR_HASH_PC, &temp); 151 if (ret) 152 return ret; 153 154 addr_hash.pc.xor_enable = FIELD_GET(ADDR_HASH_XOR_EN, temp); 155 addr_hash.pc.col_xor = FIELD_GET(ADDR_HASH_COL_XOR, temp); 156 addr_hash.pc.row_xor = FIELD_GET(ADDR_HASH_ROW_XOR, temp); 157 158 ret = amd_smn_read(0, MI300_ADDR_HASH_PC2, &temp); 159 if (ret) 160 return ret; 161 162 addr_hash.bank_xor = FIELD_GET(ADDR_HASH_BANK_XOR, temp); 163 164 ret = amd_smn_read(0, MI300_ADDR_CFG, &temp); 165 if (ret) 166 return ret; 167 168 bit_shifts.num_row_hi = FIELD_GET(ADDR_CFG_NUM_ROW_HI, temp); 169 bit_shifts.num_row_lo = 10 + FIELD_GET(ADDR_CFG_NUM_ROW_LO, temp); 170 171 ret = amd_smn_read(0, MI300_ADDR_SEL, &temp); 172 if (ret) 173 return ret; 174 175 bit_shifts.bank[0] = 5 + FIELD_GET(ADDR_SEL_BANK0, temp); 176 bit_shifts.bank[1] = 5 + FIELD_GET(ADDR_SEL_BANK1, temp); 177 bit_shifts.bank[2] = 5 + FIELD_GET(ADDR_SEL_BANK2, temp); 178 bit_shifts.bank[3] = 5 + FIELD_GET(ADDR_SEL_BANK3, temp); 179 /* Use BankBit4 for the SID0 position. */ 180 bit_shifts.sid[0] = 5 + FIELD_GET(ADDR_SEL_BANK4, temp); 181 bit_shifts.row_lo = 12 + FIELD_GET(ADDR_SEL_ROW_LO, temp); 182 bit_shifts.row_hi = 24 + FIELD_GET(ADDR_SEL_ROW_HI, temp); 183 184 ret = amd_smn_read(0, MI300_COL_SEL_LO, &temp); 185 if (ret) 186 return ret; 187 188 bit_shifts.col[0] = 2 + FIELD_GET(COL_SEL_LO_COL0, temp); 189 bit_shifts.col[1] = 2 + FIELD_GET(COL_SEL_LO_COL1, temp); 190 bit_shifts.col[2] = 2 + FIELD_GET(COL_SEL_LO_COL2, temp); 191 bit_shifts.col[3] = 2 + FIELD_GET(COL_SEL_LO_COL3, temp); 192 bit_shifts.col[4] = 2 + FIELD_GET(COL_SEL_LO_COL4, temp); 193 194 ret = amd_smn_read(0, MI300_ADDR_SEL_2, &temp); 195 if (ret) 196 return ret; 197 198 /* Use BankBit5 for the SID1 position. */ 199 bit_shifts.sid[1] = 5 + FIELD_GET(ADDR_SEL_2_BANK5, temp); 200 bit_shifts.pc = 5 + FIELD_GET(ADDR_SEL_2_CHAN, temp); 201 202 return 0; 203 } 204 205 /* 206 * MI300 systems report a DRAM address in MCA_ADDR for DRAM ECC errors. This must 207 * be converted to the intermediate normalized address (NA) before translating to a 208 * system physical address. 209 * 210 * The DRAM address includes bank, row, and column. Also included are bits for 211 * pseudochannel (PC) and stack ID (SID). 212 * 213 * Abbreviations: (S)tack ID, (P)seudochannel, (R)ow, (B)ank, (C)olumn, (Z)ero 214 * 215 * The MCA address format is as follows: 216 * MCA_ADDR[27:0] = {S[1:0], P[0], R[14:0], B[3:0], C[4:0], Z[0]} 217 * 218 * Additionally, the PC and Bank bits may be hashed. This must be accounted for before 219 * reconstructing the normalized address. 220 */ 221 #define MI300_UMC_MCA_BANK GENMASK(9, 6) 222 #define MI300_UMC_MCA_ROW GENMASK(24, 10) 223 #define MI300_UMC_MCA_PC BIT(25) 224 #define MI300_UMC_MCA_SID GENMASK(27, 26) 225 226 static unsigned long convert_dram_to_norm_addr_mi300(unsigned long addr) 227 { 228 u16 i, col, row, bank, pc, sid; 229 u32 temp; 230 231 col = FIELD_GET(MI300_UMC_MCA_COL, addr); 232 bank = FIELD_GET(MI300_UMC_MCA_BANK, addr); 233 row = FIELD_GET(MI300_UMC_MCA_ROW, addr); 234 pc = FIELD_GET(MI300_UMC_MCA_PC, addr); 235 sid = FIELD_GET(MI300_UMC_MCA_SID, addr); 236 237 /* Calculate hash for each Bank bit. */ 238 for (i = 0; i < NUM_BANK_BITS; i++) { 239 if (!addr_hash.bank[i].xor_enable) 240 continue; 241 242 temp = hweight16(col & addr_hash.bank[i].col_xor) & 1; 243 temp ^= hweight16(row & addr_hash.bank[i].row_xor) & 1; 244 bank ^= temp << i; 245 } 246 247 /* Calculate hash for PC bit. */ 248 if (addr_hash.pc.xor_enable) { 249 temp = hweight16(col & addr_hash.pc.col_xor) & 1; 250 temp ^= hweight16(row & addr_hash.pc.row_xor) & 1; 251 /* Bits SID[1:0] act as Bank[5:4] for PC hash, so apply them here. */ 252 temp ^= hweight16((bank | sid << NUM_BANK_BITS) & addr_hash.bank_xor) & 1; 253 pc ^= temp; 254 } 255 256 /* Reconstruct the normalized address starting with NA[4:0] = 0 */ 257 addr = 0; 258 259 /* Column bits */ 260 for (i = 0; i < NUM_COL_BITS; i++) { 261 temp = (col >> i) & 0x1; 262 addr |= temp << bit_shifts.col[i]; 263 } 264 265 /* Bank bits */ 266 for (i = 0; i < NUM_BANK_BITS; i++) { 267 temp = (bank >> i) & 0x1; 268 addr |= temp << bit_shifts.bank[i]; 269 } 270 271 /* Row lo bits */ 272 for (i = 0; i < bit_shifts.num_row_lo; i++) { 273 temp = (row >> i) & 0x1; 274 addr |= temp << (i + bit_shifts.row_lo); 275 } 276 277 /* Row hi bits */ 278 for (i = 0; i < bit_shifts.num_row_hi; i++) { 279 temp = (row >> (i + bit_shifts.num_row_lo)) & 0x1; 280 addr |= temp << (i + bit_shifts.row_hi); 281 } 282 283 /* PC bit */ 284 addr |= pc << bit_shifts.pc; 285 286 /* SID bits */ 287 for (i = 0; i < NUM_SID_BITS; i++) { 288 temp = (sid >> i) & 0x1; 289 addr |= temp << bit_shifts.sid[i]; 290 } 291 292 pr_debug("Addr=0x%016lx", addr); 293 pr_debug("Bank=%u Row=%u Column=%u PC=%u SID=%u", bank, row, col, pc, sid); 294 295 return addr; 296 } 297 298 /* 299 * When a DRAM ECC error occurs on MI300 systems, it is recommended to retire 300 * all memory within that DRAM row. This applies to the memory with a DRAM 301 * bank. 302 * 303 * To find the memory addresses, loop through permutations of the DRAM column 304 * bits and find the System Physical address of each. The column bits are used 305 * to calculate the intermediate Normalized address, so all permutations should 306 * be checked. 307 * 308 * See amd_atl::convert_dram_to_norm_addr_mi300() for MI300 address formats. 309 */ 310 #define MI300_NUM_COL BIT(HWEIGHT(MI300_UMC_MCA_COL)) 311 static void _retire_row_mi300(struct atl_err *a_err) 312 { 313 unsigned long addr; 314 struct page *p; 315 u8 col; 316 317 for (col = 0; col < MI300_NUM_COL; col++) { 318 a_err->addr &= ~MI300_UMC_MCA_COL; 319 a_err->addr |= FIELD_PREP(MI300_UMC_MCA_COL, col); 320 321 addr = amd_convert_umc_mca_addr_to_sys_addr(a_err); 322 if (IS_ERR_VALUE(addr)) 323 continue; 324 325 addr = PHYS_PFN(addr); 326 327 /* 328 * Skip invalid or already poisoned pages to avoid unnecessary 329 * error messages from memory_failure(). 330 */ 331 p = pfn_to_online_page(addr); 332 if (!p) 333 continue; 334 335 if (PageHWPoison(p)) 336 continue; 337 338 memory_failure(addr, 0); 339 } 340 } 341 342 /* 343 * In addition to the column bits, the row[13] bit should also be included when 344 * calculating addresses affected by a physical row. 345 * 346 * Instead of running through another loop over a single bit, just run through 347 * the column bits twice and flip the row[13] bit in-between. 348 * 349 * See MI300_UMC_MCA_ROW for the row bits in MCA_ADDR_UMC value. 350 */ 351 static void retire_row_mi300(struct atl_err *a_err) 352 { 353 _retire_row_mi300(a_err); 354 a_err->addr ^= MI300_UMC_MCA_ROW13; 355 _retire_row_mi300(a_err); 356 } 357 358 void amd_retire_dram_row(struct atl_err *a_err) 359 { 360 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) 361 return retire_row_mi300(a_err); 362 } 363 EXPORT_SYMBOL_GPL(amd_retire_dram_row); 364 365 static unsigned long get_addr(unsigned long addr) 366 { 367 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) 368 return convert_dram_to_norm_addr_mi300(addr); 369 370 return addr; 371 } 372 373 #define MCA_IPID_INST_ID_HI GENMASK_ULL(47, 44) 374 static u8 get_die_id(struct atl_err *err) 375 { 376 /* 377 * AMD Node ID is provided in MCA_IPID[InstanceIdHi], and this 378 * needs to be divided by 4 to get the internal Die ID. 379 */ 380 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) { 381 u8 node_id = FIELD_GET(MCA_IPID_INST_ID_HI, err->ipid); 382 383 return node_id >> 2; 384 } 385 386 /* 387 * For CPUs, this is the AMD Node ID modulo the number 388 * of AMD Nodes per socket. 389 */ 390 return topology_amd_node_id(err->cpu) % topology_amd_nodes_per_pkg(); 391 } 392 393 #define UMC_CHANNEL_NUM GENMASK(31, 20) 394 static u8 get_coh_st_inst_id(struct atl_err *err) 395 { 396 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) 397 return get_coh_st_inst_id_mi300(err); 398 399 return FIELD_GET(UMC_CHANNEL_NUM, err->ipid); 400 } 401 402 unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err) 403 { 404 u8 socket_id = topology_physical_package_id(err->cpu); 405 u8 coh_st_inst_id = get_coh_st_inst_id(err); 406 unsigned long addr = get_addr(err->addr); 407 u8 die_id = get_die_id(err); 408 unsigned long ret_addr; 409 410 pr_debug("socket_id=0x%x die_id=0x%x coh_st_inst_id=0x%x addr=0x%016lx", 411 socket_id, die_id, coh_st_inst_id, addr); 412 413 ret_addr = prm_umc_norm_to_sys_addr(socket_id, err->ipid, addr); 414 if (!IS_ERR_VALUE(ret_addr) || df_cfg.flags.prm_only) 415 return ret_addr; 416 417 return norm_to_sys_addr(socket_id, die_id, coh_st_inst_id, addr); 418 } 419