1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module 3 * 4 * This driver supports the memory controllers found on the Intel 5 * processor family Sandy Bridge. 6 * 7 * Copyright (c) 2011 by: 8 * Mauro Carvalho Chehab 9 */ 10 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/pci.h> 14 #include <linux/pci_ids.h> 15 #include <linux/slab.h> 16 #include <linux/delay.h> 17 #include <linux/edac.h> 18 #include <linux/mmzone.h> 19 #include <linux/smp.h> 20 #include <linux/bitmap.h> 21 #include <linux/math64.h> 22 #include <asm/cpu_device_id.h> 23 #include <asm/intel-family.h> 24 #include <asm/processor.h> 25 #include <asm/mce.h> 26 27 #include "edac_module.h" 28 29 /* Static vars */ 30 static LIST_HEAD(sbridge_edac_list); 31 static char sb_msg[256]; 32 static char sb_msg_full[512]; 33 34 /* 35 * Alter this version for the module when modifications are made 36 */ 37 #define SBRIDGE_REVISION " Ver: 1.1.2 " 38 #define EDAC_MOD_STR "sb_edac" 39 40 /* 41 * Debug macros 42 */ 43 #define sbridge_printk(level, fmt, arg...) \ 44 edac_printk(level, "sbridge", fmt, ##arg) 45 46 #define sbridge_mc_printk(mci, level, fmt, arg...) \ 47 edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg) 48 49 /* 50 * Get a bit field at register value <v>, from bit <lo> to bit <hi> 51 */ 52 #define GET_BITFIELD(v, lo, hi) \ 53 (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 54 55 /* Devices 12 Function 6, Offsets 0x80 to 0xcc */ 56 static const u32 sbridge_dram_rule[] = { 57 0x80, 0x88, 0x90, 0x98, 0xa0, 58 0xa8, 0xb0, 0xb8, 0xc0, 0xc8, 59 }; 60 61 static const u32 ibridge_dram_rule[] = { 62 0x60, 0x68, 0x70, 0x78, 0x80, 63 0x88, 0x90, 0x98, 0xa0, 0xa8, 64 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, 65 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, 66 }; 67 68 static const u32 knl_dram_rule[] = { 69 0x60, 0x68, 0x70, 0x78, 0x80, /* 0-4 */ 70 0x88, 0x90, 0x98, 0xa0, 0xa8, /* 5-9 */ 71 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, /* 10-14 */ 72 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, /* 15-19 */ 73 0x100, 0x108, 0x110, 0x118, /* 20-23 */ 74 }; 75 76 #define DRAM_RULE_ENABLE(reg) GET_BITFIELD(reg, 0, 0) 77 #define A7MODE(reg) GET_BITFIELD(reg, 26, 26) 78 79 static char *show_dram_attr(u32 attr) 80 { 81 switch (attr) { 82 case 0: 83 return "DRAM"; 84 case 1: 85 return "MMCFG"; 86 case 2: 87 return "NXM"; 88 default: 89 return "unknown"; 90 } 91 } 92 93 static const u32 sbridge_interleave_list[] = { 94 0x84, 0x8c, 0x94, 0x9c, 0xa4, 95 0xac, 0xb4, 0xbc, 0xc4, 0xcc, 96 }; 97 98 static const u32 ibridge_interleave_list[] = { 99 0x64, 0x6c, 0x74, 0x7c, 0x84, 100 0x8c, 0x94, 0x9c, 0xa4, 0xac, 101 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, 102 0xdc, 0xe4, 0xec, 0xf4, 0xfc, 103 }; 104 105 static const u32 knl_interleave_list[] = { 106 0x64, 0x6c, 0x74, 0x7c, 0x84, /* 0-4 */ 107 0x8c, 0x94, 0x9c, 0xa4, 0xac, /* 5-9 */ 108 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, /* 10-14 */ 109 0xdc, 0xe4, 0xec, 0xf4, 0xfc, /* 15-19 */ 110 0x104, 0x10c, 0x114, 0x11c, /* 20-23 */ 111 }; 112 #define MAX_INTERLEAVE \ 113 (MAX_T(unsigned int, ARRAY_SIZE(sbridge_interleave_list), \ 114 MAX_T(unsigned int, ARRAY_SIZE(ibridge_interleave_list), \ 115 ARRAY_SIZE(knl_interleave_list)))) 116 117 struct interleave_pkg { 118 unsigned char start; 119 unsigned char end; 120 }; 121 122 static const struct interleave_pkg sbridge_interleave_pkg[] = { 123 { 0, 2 }, 124 { 3, 5 }, 125 { 8, 10 }, 126 { 11, 13 }, 127 { 16, 18 }, 128 { 19, 21 }, 129 { 24, 26 }, 130 { 27, 29 }, 131 }; 132 133 static const struct interleave_pkg ibridge_interleave_pkg[] = { 134 { 0, 3 }, 135 { 4, 7 }, 136 { 8, 11 }, 137 { 12, 15 }, 138 { 16, 19 }, 139 { 20, 23 }, 140 { 24, 27 }, 141 { 28, 31 }, 142 }; 143 144 static inline int sad_pkg(const struct interleave_pkg *table, u32 reg, 145 int interleave) 146 { 147 return GET_BITFIELD(reg, table[interleave].start, 148 table[interleave].end); 149 } 150 151 /* Devices 12 Function 7 */ 152 153 #define TOLM 0x80 154 #define TOHM 0x84 155 #define HASWELL_TOLM 0xd0 156 #define HASWELL_TOHM_0 0xd4 157 #define HASWELL_TOHM_1 0xd8 158 #define KNL_TOLM 0xd0 159 #define KNL_TOHM_0 0xd4 160 #define KNL_TOHM_1 0xd8 161 162 #define GET_TOLM(reg) ((GET_BITFIELD(reg, 0, 3) << 28) | 0x3ffffff) 163 #define GET_TOHM(reg) ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff) 164 165 /* Device 13 Function 6 */ 166 167 #define SAD_TARGET 0xf0 168 169 #define SOURCE_ID(reg) GET_BITFIELD(reg, 9, 11) 170 171 #define SOURCE_ID_KNL(reg) GET_BITFIELD(reg, 12, 14) 172 173 #define SAD_CONTROL 0xf4 174 175 /* Device 14 function 0 */ 176 177 static const u32 tad_dram_rule[] = { 178 0x40, 0x44, 0x48, 0x4c, 179 0x50, 0x54, 0x58, 0x5c, 180 0x60, 0x64, 0x68, 0x6c, 181 }; 182 #define MAX_TAD ARRAY_SIZE(tad_dram_rule) 183 184 #define TAD_LIMIT(reg) ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff) 185 #define TAD_SOCK(reg) GET_BITFIELD(reg, 10, 11) 186 #define TAD_CH(reg) GET_BITFIELD(reg, 8, 9) 187 #define TAD_TGT3(reg) GET_BITFIELD(reg, 6, 7) 188 #define TAD_TGT2(reg) GET_BITFIELD(reg, 4, 5) 189 #define TAD_TGT1(reg) GET_BITFIELD(reg, 2, 3) 190 #define TAD_TGT0(reg) GET_BITFIELD(reg, 0, 1) 191 192 /* Device 15, function 0 */ 193 194 #define MCMTR 0x7c 195 #define KNL_MCMTR 0x624 196 197 #define IS_ECC_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 2, 2) 198 #define IS_LOCKSTEP_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 1, 1) 199 #define IS_CLOSE_PG(mcmtr) GET_BITFIELD(mcmtr, 0, 0) 200 201 /* Device 15, function 1 */ 202 203 #define RASENABLES 0xac 204 #define IS_MIRROR_ENABLED(reg) GET_BITFIELD(reg, 0, 0) 205 206 /* Device 15, functions 2-5 */ 207 208 static const int mtr_regs[] = { 209 0x80, 0x84, 0x88, 210 }; 211 212 static const int knl_mtr_reg = 0xb60; 213 214 #define RANK_DISABLE(mtr) GET_BITFIELD(mtr, 16, 19) 215 #define IS_DIMM_PRESENT(mtr) GET_BITFIELD(mtr, 14, 14) 216 #define RANK_CNT_BITS(mtr) GET_BITFIELD(mtr, 12, 13) 217 #define RANK_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 2, 4) 218 #define COL_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 0, 1) 219 220 static const u32 tad_ch_nilv_offset[] = { 221 0x90, 0x94, 0x98, 0x9c, 222 0xa0, 0xa4, 0xa8, 0xac, 223 0xb0, 0xb4, 0xb8, 0xbc, 224 }; 225 #define CHN_IDX_OFFSET(reg) GET_BITFIELD(reg, 28, 29) 226 #define TAD_OFFSET(reg) (GET_BITFIELD(reg, 6, 25) << 26) 227 228 static const u32 rir_way_limit[] = { 229 0x108, 0x10c, 0x110, 0x114, 0x118, 230 }; 231 #define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit) 232 233 #define IS_RIR_VALID(reg) GET_BITFIELD(reg, 31, 31) 234 #define RIR_WAY(reg) GET_BITFIELD(reg, 28, 29) 235 236 #define MAX_RIR_WAY 8 237 238 static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = { 239 { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c }, 240 { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c }, 241 { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c }, 242 { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c }, 243 { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc }, 244 }; 245 246 #define RIR_RNK_TGT(type, reg) (((type) == BROADWELL) ? \ 247 GET_BITFIELD(reg, 20, 23) : GET_BITFIELD(reg, 16, 19)) 248 249 #define RIR_OFFSET(type, reg) (((type) == HASWELL || (type) == BROADWELL) ? \ 250 GET_BITFIELD(reg, 2, 15) : GET_BITFIELD(reg, 2, 14)) 251 252 /* Device 16, functions 2-7 */ 253 254 /* 255 * FIXME: Implement the error count reads directly 256 */ 257 258 #define RANK_ODD_OV(reg) GET_BITFIELD(reg, 31, 31) 259 #define RANK_ODD_ERR_CNT(reg) GET_BITFIELD(reg, 16, 30) 260 #define RANK_EVEN_OV(reg) GET_BITFIELD(reg, 15, 15) 261 #define RANK_EVEN_ERR_CNT(reg) GET_BITFIELD(reg, 0, 14) 262 263 #if 0 /* Currently unused*/ 264 static const u32 correrrcnt[] = { 265 0x104, 0x108, 0x10c, 0x110, 266 }; 267 268 static const u32 correrrthrsld[] = { 269 0x11c, 0x120, 0x124, 0x128, 270 }; 271 #endif 272 273 #define RANK_ODD_ERR_THRSLD(reg) GET_BITFIELD(reg, 16, 30) 274 #define RANK_EVEN_ERR_THRSLD(reg) GET_BITFIELD(reg, 0, 14) 275 276 277 /* Device 17, function 0 */ 278 279 #define SB_RANK_CFG_A 0x0328 280 281 #define IB_RANK_CFG_A 0x0320 282 283 /* 284 * sbridge structs 285 */ 286 287 #define NUM_CHANNELS 6 /* Max channels per MC */ 288 #define MAX_DIMMS 3 /* Max DIMMS per channel */ 289 #define KNL_MAX_CHAS 38 /* KNL max num. of Cache Home Agents */ 290 #define KNL_MAX_CHANNELS 6 /* KNL max num. of PCI channels */ 291 #define KNL_MAX_EDCS 8 /* Embedded DRAM controllers */ 292 #define CHANNEL_UNSPECIFIED 0xf /* Intel IA32 SDM 15-14 */ 293 294 enum type { 295 SANDY_BRIDGE, 296 IVY_BRIDGE, 297 HASWELL, 298 BROADWELL, 299 KNIGHTS_LANDING, 300 }; 301 302 enum domain { 303 IMC0 = 0, 304 IMC1, 305 SOCK, 306 }; 307 308 enum mirroring_mode { 309 NON_MIRRORING, 310 ADDR_RANGE_MIRRORING, 311 FULL_MIRRORING, 312 }; 313 314 struct sbridge_pvt; 315 struct sbridge_info { 316 enum type type; 317 u32 mcmtr; 318 u32 rankcfgr; 319 u64 (*get_tolm)(struct sbridge_pvt *pvt); 320 u64 (*get_tohm)(struct sbridge_pvt *pvt); 321 u64 (*rir_limit)(u32 reg); 322 u64 (*sad_limit)(u32 reg); 323 u32 (*interleave_mode)(u32 reg); 324 u32 (*dram_attr)(u32 reg); 325 const u32 *dram_rule; 326 const u32 *interleave_list; 327 const struct interleave_pkg *interleave_pkg; 328 u8 max_sad; 329 u8 (*get_node_id)(struct sbridge_pvt *pvt); 330 u8 (*get_ha)(u8 bank); 331 enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt); 332 enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr); 333 struct pci_dev *pci_vtd; 334 }; 335 336 struct sbridge_channel { 337 u32 ranks; 338 u32 dimms; 339 struct dimm { 340 u32 rowbits; 341 u32 colbits; 342 u32 bank_xor_enable; 343 u32 amap_fine; 344 } dimm[MAX_DIMMS]; 345 }; 346 347 struct pci_id_descr { 348 int dev_id; 349 int optional; 350 enum domain dom; 351 }; 352 353 struct pci_id_table { 354 const struct pci_id_descr *descr; 355 int n_devs_per_imc; 356 int n_devs_per_sock; 357 int n_imcs_per_sock; 358 enum type type; 359 }; 360 361 struct sbridge_dev { 362 struct list_head list; 363 int seg; 364 u8 bus, mc; 365 u8 node_id, source_id; 366 enum domain dom; 367 int n_devs; 368 int i_devs; 369 struct mem_ctl_info *mci; 370 struct pci_dev *pdev[] __counted_by(n_devs); 371 }; 372 373 struct knl_pvt { 374 struct pci_dev *pci_cha[KNL_MAX_CHAS]; 375 struct pci_dev *pci_channel[KNL_MAX_CHANNELS]; 376 struct pci_dev *pci_mc0; 377 struct pci_dev *pci_mc1; 378 struct pci_dev *pci_mc0_misc; 379 struct pci_dev *pci_mc1_misc; 380 struct pci_dev *pci_mc_info; /* tolm, tohm */ 381 }; 382 383 struct sbridge_pvt { 384 /* Devices per socket */ 385 struct pci_dev *pci_ddrio; 386 struct pci_dev *pci_sad0, *pci_sad1; 387 struct pci_dev *pci_br0, *pci_br1; 388 /* Devices per memory controller */ 389 struct pci_dev *pci_ha, *pci_ta, *pci_ras; 390 struct pci_dev *pci_tad[NUM_CHANNELS]; 391 392 struct sbridge_dev *sbridge_dev; 393 394 struct sbridge_info info; 395 struct sbridge_channel channel[NUM_CHANNELS]; 396 397 /* Memory type detection */ 398 bool is_cur_addr_mirrored, is_lockstep, is_close_pg; 399 bool is_chan_hash; 400 enum mirroring_mode mirror_mode; 401 402 /* Memory description */ 403 u64 tolm, tohm; 404 struct knl_pvt knl; 405 }; 406 407 #define PCI_DESCR(device_id, opt, domain) \ 408 .dev_id = (device_id), \ 409 .optional = opt, \ 410 .dom = domain 411 412 static const struct pci_id_descr pci_dev_descr_sbridge[] = { 413 /* Processor Home Agent */ 414 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0, IMC0) }, 415 416 /* Memory controller */ 417 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0, IMC0) }, 418 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0, IMC0) }, 419 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0, IMC0) }, 420 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0, IMC0) }, 421 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0, IMC0) }, 422 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0, IMC0) }, 423 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1, SOCK) }, 424 425 /* System Address Decoder */ 426 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0, SOCK) }, 427 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0, SOCK) }, 428 429 /* Broadcast Registers */ 430 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0, SOCK) }, 431 }; 432 433 #define PCI_ID_TABLE_ENTRY(A, N, M, T) { \ 434 .descr = A, \ 435 .n_devs_per_imc = N, \ 436 .n_devs_per_sock = ARRAY_SIZE(A), \ 437 .n_imcs_per_sock = M, \ 438 .type = T \ 439 } 440 441 static const struct pci_id_table pci_dev_descr_sbridge_table[] = { 442 PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge, ARRAY_SIZE(pci_dev_descr_sbridge), 1, SANDY_BRIDGE), 443 { NULL, } 444 }; 445 446 /* This changes depending if 1HA or 2HA: 447 * 1HA: 448 * 0x0eb8 (17.0) is DDRIO0 449 * 2HA: 450 * 0x0ebc (17.4) is DDRIO0 451 */ 452 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0 0x0eb8 453 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0 0x0ebc 454 455 /* pci ids */ 456 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0 0x0ea0 457 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA 0x0ea8 458 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS 0x0e71 459 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0 0x0eaa 460 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1 0x0eab 461 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2 0x0eac 462 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3 0x0ead 463 #define PCI_DEVICE_ID_INTEL_IBRIDGE_SAD 0x0ec8 464 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR0 0x0ec9 465 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR1 0x0eca 466 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1 0x0e60 467 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA 0x0e68 468 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS 0x0e79 469 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 0x0e6a 470 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1 0x0e6b 471 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2 0x0e6c 472 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3 0x0e6d 473 474 static const struct pci_id_descr pci_dev_descr_ibridge[] = { 475 /* Processor Home Agent */ 476 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0, 0, IMC0) }, 477 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1, 1, IMC1) }, 478 479 /* Memory controller */ 480 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA, 0, IMC0) }, 481 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS, 0, IMC0) }, 482 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0, 0, IMC0) }, 483 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1, 0, IMC0) }, 484 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2, 0, IMC0) }, 485 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3, 0, IMC0) }, 486 487 /* Optional, mode 2HA */ 488 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA, 1, IMC1) }, 489 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS, 1, IMC1) }, 490 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0, 1, IMC1) }, 491 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1, 1, IMC1) }, 492 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2, 1, IMC1) }, 493 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3, 1, IMC1) }, 494 495 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0, 1, SOCK) }, 496 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0, 1, SOCK) }, 497 498 /* System Address Decoder */ 499 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_SAD, 0, SOCK) }, 500 501 /* Broadcast Registers */ 502 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR0, 1, SOCK) }, 503 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR1, 0, SOCK) }, 504 505 }; 506 507 static const struct pci_id_table pci_dev_descr_ibridge_table[] = { 508 PCI_ID_TABLE_ENTRY(pci_dev_descr_ibridge, 12, 2, IVY_BRIDGE), 509 { NULL, } 510 }; 511 512 /* Haswell support */ 513 /* EN processor: 514 * - 1 IMC 515 * - 3 DDR3 channels, 2 DPC per channel 516 * EP processor: 517 * - 1 or 2 IMC 518 * - 4 DDR4 channels, 3 DPC per channel 519 * EP 4S processor: 520 * - 2 IMC 521 * - 4 DDR4 channels, 3 DPC per channel 522 * EX processor: 523 * - 2 IMC 524 * - each IMC interfaces with a SMI 2 channel 525 * - each SMI channel interfaces with a scalable memory buffer 526 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 527 */ 528 #define HASWELL_DDRCRCLKCONTROLS 0xa10 /* Ditto on Broadwell */ 529 #define HASWELL_HASYSDEFEATURE2 0x84 530 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC 0x2f28 531 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0 0x2fa0 532 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1 0x2f60 533 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA 0x2fa8 534 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM 0x2f71 535 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA 0x2f68 536 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM 0x2f79 537 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0 0x2ffc 538 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1 0x2ffd 539 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0 0x2faa 540 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1 0x2fab 541 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2 0x2fac 542 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3 0x2fad 543 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 0x2f6a 544 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1 0x2f6b 545 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c 546 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d 547 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd 548 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf 549 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9 550 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb 551 static const struct pci_id_descr pci_dev_descr_haswell[] = { 552 /* first item must be the HA */ 553 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0, IMC0) }, 554 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1, 1, IMC1) }, 555 556 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA, 0, IMC0) }, 557 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM, 0, IMC0) }, 558 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0, 0, IMC0) }, 559 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1, 0, IMC0) }, 560 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2, 1, IMC0) }, 561 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1, IMC0) }, 562 563 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1, IMC1) }, 564 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM, 1, IMC1) }, 565 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0, 1, IMC1) }, 566 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1, 1, IMC1) }, 567 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2, 1, IMC1) }, 568 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3, 1, IMC1) }, 569 570 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0, 0, SOCK) }, 571 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1, 0, SOCK) }, 572 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1, SOCK) }, 573 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1, SOCK) }, 574 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1, SOCK) }, 575 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1, SOCK) }, 576 }; 577 578 static const struct pci_id_table pci_dev_descr_haswell_table[] = { 579 PCI_ID_TABLE_ENTRY(pci_dev_descr_haswell, 13, 2, HASWELL), 580 { NULL, } 581 }; 582 583 /* Knight's Landing Support */ 584 /* 585 * KNL's memory channels are swizzled between memory controllers. 586 * MC0 is mapped to CH3,4,5 and MC1 is mapped to CH0,1,2 587 */ 588 #define knl_channel_remap(mc, chan) ((mc) ? (chan) : (chan) + 3) 589 590 /* Memory controller, TAD tables, error injection - 2-8-0, 2-9-0 (2 of these) */ 591 #define PCI_DEVICE_ID_INTEL_KNL_IMC_MC 0x7840 592 /* DRAM channel stuff; bank addrs, dimmmtr, etc.. 2-8-2 - 2-9-4 (6 of these) */ 593 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN 0x7843 594 /* kdrwdbu TAD limits/offsets, MCMTR - 2-10-1, 2-11-1 (2 of these) */ 595 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TA 0x7844 596 /* CHA broadcast registers, dram rules - 1-29-0 (1 of these) */ 597 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0 0x782a 598 /* SAD target - 1-29-1 (1 of these) */ 599 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1 0x782b 600 /* Caching / Home Agent */ 601 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHA 0x782c 602 /* Device with TOLM and TOHM, 0-5-0 (1 of these) */ 603 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM 0x7810 604 605 /* 606 * KNL differs from SB, IB, and Haswell in that it has multiple 607 * instances of the same device with the same device ID, so we handle that 608 * by creating as many copies in the table as we expect to find. 609 * (Like device ID must be grouped together.) 610 */ 611 612 static const struct pci_id_descr pci_dev_descr_knl[] = { 613 [0 ... 1] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_MC, 0, IMC0)}, 614 [2 ... 7] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN, 0, IMC0) }, 615 [8] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TA, 0, IMC0) }, 616 [9] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM, 0, IMC0) }, 617 [10] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0, 0, SOCK) }, 618 [11] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1, 0, SOCK) }, 619 [12 ... 49] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHA, 0, SOCK) }, 620 }; 621 622 static const struct pci_id_table pci_dev_descr_knl_table[] = { 623 PCI_ID_TABLE_ENTRY(pci_dev_descr_knl, ARRAY_SIZE(pci_dev_descr_knl), 1, KNIGHTS_LANDING), 624 { NULL, } 625 }; 626 627 /* 628 * Broadwell support 629 * 630 * DE processor: 631 * - 1 IMC 632 * - 2 DDR3 channels, 2 DPC per channel 633 * EP processor: 634 * - 1 or 2 IMC 635 * - 4 DDR4 channels, 3 DPC per channel 636 * EP 4S processor: 637 * - 2 IMC 638 * - 4 DDR4 channels, 3 DPC per channel 639 * EX processor: 640 * - 2 IMC 641 * - each IMC interfaces with a SMI 2 channel 642 * - each SMI channel interfaces with a scalable memory buffer 643 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 644 */ 645 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC 0x6f28 646 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0 0x6fa0 647 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1 0x6f60 648 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA 0x6fa8 649 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM 0x6f71 650 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA 0x6f68 651 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM 0x6f79 652 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0 0x6ffc 653 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1 0x6ffd 654 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0 0x6faa 655 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1 0x6fab 656 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2 0x6fac 657 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3 0x6fad 658 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 0x6f6a 659 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1 0x6f6b 660 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2 0x6f6c 661 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3 0x6f6d 662 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0 0x6faf 663 664 static const struct pci_id_descr pci_dev_descr_broadwell[] = { 665 /* first item must be the HA */ 666 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0, 0, IMC0) }, 667 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1, 1, IMC1) }, 668 669 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA, 0, IMC0) }, 670 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM, 0, IMC0) }, 671 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0, 0, IMC0) }, 672 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1, 0, IMC0) }, 673 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2, 1, IMC0) }, 674 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3, 1, IMC0) }, 675 676 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA, 1, IMC1) }, 677 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM, 1, IMC1) }, 678 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0, 1, IMC1) }, 679 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1, 1, IMC1) }, 680 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2, 1, IMC1) }, 681 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3, 1, IMC1) }, 682 683 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0, 0, SOCK) }, 684 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1, 0, SOCK) }, 685 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0, 1, SOCK) }, 686 }; 687 688 static const struct pci_id_table pci_dev_descr_broadwell_table[] = { 689 PCI_ID_TABLE_ENTRY(pci_dev_descr_broadwell, 10, 2, BROADWELL), 690 { NULL, } 691 }; 692 693 694 /**************************************************************************** 695 Ancillary status routines 696 ****************************************************************************/ 697 698 static inline int numrank(enum type type, u32 mtr) 699 { 700 int ranks = (1 << RANK_CNT_BITS(mtr)); 701 int max = 4; 702 703 if (type == HASWELL || type == BROADWELL || type == KNIGHTS_LANDING) 704 max = 8; 705 706 if (ranks > max) { 707 edac_dbg(0, "Invalid number of ranks: %d (max = %i) raw value = %x (%04x)\n", 708 ranks, max, (unsigned int)RANK_CNT_BITS(mtr), mtr); 709 return -EINVAL; 710 } 711 712 return ranks; 713 } 714 715 static inline int numrow(u32 mtr) 716 { 717 int rows = (RANK_WIDTH_BITS(mtr) + 12); 718 719 if (rows < 13 || rows > 18) { 720 edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n", 721 rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr); 722 return -EINVAL; 723 } 724 725 return 1 << rows; 726 } 727 728 static inline int numcol(u32 mtr) 729 { 730 int cols = (COL_WIDTH_BITS(mtr) + 10); 731 732 if (cols > 12) { 733 edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n", 734 cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr); 735 return -EINVAL; 736 } 737 738 return 1 << cols; 739 } 740 741 static struct sbridge_dev *get_sbridge_dev(int seg, u8 bus, enum domain dom, 742 int multi_bus, 743 struct sbridge_dev *prev) 744 { 745 struct sbridge_dev *sbridge_dev; 746 747 /* 748 * If we have devices scattered across several busses that pertain 749 * to the same memory controller, we'll lump them all together. 750 */ 751 if (multi_bus) { 752 return list_first_entry_or_null(&sbridge_edac_list, 753 struct sbridge_dev, list); 754 } 755 756 sbridge_dev = list_entry(prev ? prev->list.next 757 : sbridge_edac_list.next, struct sbridge_dev, list); 758 759 list_for_each_entry_from(sbridge_dev, &sbridge_edac_list, list) { 760 if ((sbridge_dev->seg == seg) && (sbridge_dev->bus == bus) && 761 (dom == SOCK || dom == sbridge_dev->dom)) 762 return sbridge_dev; 763 } 764 765 return NULL; 766 } 767 768 static struct sbridge_dev *alloc_sbridge_dev(int seg, u8 bus, enum domain dom, 769 const struct pci_id_table *table) 770 { 771 struct sbridge_dev *sbridge_dev; 772 773 sbridge_dev = kzalloc_flex(*sbridge_dev, pdev, table->n_devs_per_imc); 774 if (!sbridge_dev) 775 return NULL; 776 777 sbridge_dev->n_devs = table->n_devs_per_imc; 778 sbridge_dev->seg = seg; 779 sbridge_dev->bus = bus; 780 sbridge_dev->dom = dom; 781 list_add_tail(&sbridge_dev->list, &sbridge_edac_list); 782 783 return sbridge_dev; 784 } 785 786 static void free_sbridge_dev(struct sbridge_dev *sbridge_dev) 787 { 788 list_del(&sbridge_dev->list); 789 kfree(sbridge_dev); 790 } 791 792 static u64 sbridge_get_tolm(struct sbridge_pvt *pvt) 793 { 794 u32 reg; 795 796 /* Address range is 32:28 */ 797 pci_read_config_dword(pvt->pci_sad1, TOLM, ®); 798 return GET_TOLM(reg); 799 } 800 801 static u64 sbridge_get_tohm(struct sbridge_pvt *pvt) 802 { 803 u32 reg; 804 805 pci_read_config_dword(pvt->pci_sad1, TOHM, ®); 806 return GET_TOHM(reg); 807 } 808 809 static u64 ibridge_get_tolm(struct sbridge_pvt *pvt) 810 { 811 u32 reg; 812 813 pci_read_config_dword(pvt->pci_br1, TOLM, ®); 814 815 return GET_TOLM(reg); 816 } 817 818 static u64 ibridge_get_tohm(struct sbridge_pvt *pvt) 819 { 820 u32 reg; 821 822 pci_read_config_dword(pvt->pci_br1, TOHM, ®); 823 824 return GET_TOHM(reg); 825 } 826 827 static u64 rir_limit(u32 reg) 828 { 829 return ((u64)GET_BITFIELD(reg, 1, 10) << 29) | 0x1fffffff; 830 } 831 832 static u64 sad_limit(u32 reg) 833 { 834 return (GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff; 835 } 836 837 static u32 interleave_mode(u32 reg) 838 { 839 return GET_BITFIELD(reg, 1, 1); 840 } 841 842 static u32 dram_attr(u32 reg) 843 { 844 return GET_BITFIELD(reg, 2, 3); 845 } 846 847 static u64 knl_sad_limit(u32 reg) 848 { 849 return (GET_BITFIELD(reg, 7, 26) << 26) | 0x3ffffff; 850 } 851 852 static u32 knl_interleave_mode(u32 reg) 853 { 854 return GET_BITFIELD(reg, 1, 2); 855 } 856 857 static const char * const knl_intlv_mode[] = { 858 "[8:6]", "[10:8]", "[14:12]", "[32:30]" 859 }; 860 861 static const char *get_intlv_mode_str(u32 reg, enum type t) 862 { 863 if (t == KNIGHTS_LANDING) 864 return knl_intlv_mode[knl_interleave_mode(reg)]; 865 else 866 return interleave_mode(reg) ? "[8:6]" : "[8:6]XOR[18:16]"; 867 } 868 869 static u32 dram_attr_knl(u32 reg) 870 { 871 return GET_BITFIELD(reg, 3, 4); 872 } 873 874 875 static enum mem_type get_memory_type(struct sbridge_pvt *pvt) 876 { 877 u32 reg; 878 enum mem_type mtype; 879 880 if (pvt->pci_ddrio) { 881 pci_read_config_dword(pvt->pci_ddrio, pvt->info.rankcfgr, 882 ®); 883 if (GET_BITFIELD(reg, 11, 11)) 884 /* FIXME: Can also be LRDIMM */ 885 mtype = MEM_RDDR3; 886 else 887 mtype = MEM_DDR3; 888 } else 889 mtype = MEM_UNKNOWN; 890 891 return mtype; 892 } 893 894 static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt) 895 { 896 u32 reg; 897 bool registered = false; 898 enum mem_type mtype = MEM_UNKNOWN; 899 900 if (!pvt->pci_ddrio) 901 goto out; 902 903 pci_read_config_dword(pvt->pci_ddrio, 904 HASWELL_DDRCRCLKCONTROLS, ®); 905 /* Is_Rdimm */ 906 if (GET_BITFIELD(reg, 16, 16)) 907 registered = true; 908 909 pci_read_config_dword(pvt->pci_ta, MCMTR, ®); 910 if (GET_BITFIELD(reg, 14, 14)) { 911 if (registered) 912 mtype = MEM_RDDR4; 913 else 914 mtype = MEM_DDR4; 915 } else { 916 if (registered) 917 mtype = MEM_RDDR3; 918 else 919 mtype = MEM_DDR3; 920 } 921 922 out: 923 return mtype; 924 } 925 926 static enum dev_type knl_get_width(struct sbridge_pvt *pvt, u32 mtr) 927 { 928 /* for KNL value is fixed */ 929 return DEV_X16; 930 } 931 932 static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 933 { 934 /* there's no way to figure out */ 935 return DEV_UNKNOWN; 936 } 937 938 static enum dev_type __ibridge_get_width(u32 mtr) 939 { 940 enum dev_type type = DEV_UNKNOWN; 941 942 switch (mtr) { 943 case 2: 944 type = DEV_X16; 945 break; 946 case 1: 947 type = DEV_X8; 948 break; 949 case 0: 950 type = DEV_X4; 951 break; 952 } 953 954 return type; 955 } 956 957 static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 958 { 959 /* 960 * ddr3_width on the documentation but also valid for DDR4 on 961 * Haswell 962 */ 963 return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8)); 964 } 965 966 static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr) 967 { 968 /* ddr3_width on the documentation but also valid for DDR4 */ 969 return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9)); 970 } 971 972 static enum mem_type knl_get_memory_type(struct sbridge_pvt *pvt) 973 { 974 /* DDR4 RDIMMS and LRDIMMS are supported */ 975 return MEM_RDDR4; 976 } 977 978 static u8 get_node_id(struct sbridge_pvt *pvt) 979 { 980 u32 reg; 981 pci_read_config_dword(pvt->pci_br0, SAD_CONTROL, ®); 982 return GET_BITFIELD(reg, 0, 2); 983 } 984 985 static u8 haswell_get_node_id(struct sbridge_pvt *pvt) 986 { 987 u32 reg; 988 989 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 990 return GET_BITFIELD(reg, 0, 3); 991 } 992 993 static u8 knl_get_node_id(struct sbridge_pvt *pvt) 994 { 995 u32 reg; 996 997 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 998 return GET_BITFIELD(reg, 0, 2); 999 } 1000 1001 /* 1002 * Use the reporting bank number to determine which memory 1003 * controller (also known as "ha" for "home agent"). Sandy 1004 * Bridge only has one memory controller per socket, so the 1005 * answer is always zero. 1006 */ 1007 static u8 sbridge_get_ha(u8 bank) 1008 { 1009 return 0; 1010 } 1011 1012 /* 1013 * On Ivy Bridge, Haswell and Broadwell the error may be in a 1014 * home agent bank (7, 8), or one of the per-channel memory 1015 * controller banks (9 .. 16). 1016 */ 1017 static u8 ibridge_get_ha(u8 bank) 1018 { 1019 switch (bank) { 1020 case 7 ... 8: 1021 return bank - 7; 1022 case 9 ... 16: 1023 return (bank - 9) / 4; 1024 default: 1025 return 0xff; 1026 } 1027 } 1028 1029 /* Not used, but included for safety/symmetry */ 1030 static u8 knl_get_ha(u8 bank) 1031 { 1032 return 0xff; 1033 } 1034 1035 static u64 haswell_get_tolm(struct sbridge_pvt *pvt) 1036 { 1037 u32 reg; 1038 1039 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOLM, ®); 1040 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 1041 } 1042 1043 static u64 haswell_get_tohm(struct sbridge_pvt *pvt) 1044 { 1045 u64 rc; 1046 u32 reg; 1047 1048 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_0, ®); 1049 rc = GET_BITFIELD(reg, 26, 31); 1050 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, ®); 1051 rc = ((reg << 6) | rc) << 26; 1052 1053 return rc | 0x3ffffff; 1054 } 1055 1056 static u64 knl_get_tolm(struct sbridge_pvt *pvt) 1057 { 1058 u32 reg; 1059 1060 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOLM, ®); 1061 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 1062 } 1063 1064 static u64 knl_get_tohm(struct sbridge_pvt *pvt) 1065 { 1066 u64 rc; 1067 u32 reg_lo, reg_hi; 1068 1069 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_0, ®_lo); 1070 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_1, ®_hi); 1071 rc = ((u64)reg_hi << 32) | reg_lo; 1072 return rc | 0x3ffffff; 1073 } 1074 1075 1076 static u64 haswell_rir_limit(u32 reg) 1077 { 1078 return (((u64)GET_BITFIELD(reg, 1, 11) + 1) << 29) - 1; 1079 } 1080 1081 static inline u8 sad_pkg_socket(u8 pkg) 1082 { 1083 /* on Ivy Bridge, nodeID is SASS, where A is HA and S is node id */ 1084 return ((pkg >> 3) << 2) | (pkg & 0x3); 1085 } 1086 1087 static inline u8 sad_pkg_ha(u8 pkg) 1088 { 1089 return (pkg >> 2) & 0x1; 1090 } 1091 1092 static int haswell_chan_hash(int idx, u64 addr) 1093 { 1094 int i; 1095 1096 /* 1097 * XOR even bits from 12:26 to bit0 of idx, 1098 * odd bits from 13:27 to bit1 1099 */ 1100 for (i = 12; i < 28; i += 2) 1101 idx ^= (addr >> i) & 3; 1102 1103 return idx; 1104 } 1105 1106 /* Low bits of TAD limit, and some metadata. */ 1107 static const u32 knl_tad_dram_limit_lo[] = { 1108 0x400, 0x500, 0x600, 0x700, 1109 0x800, 0x900, 0xa00, 0xb00, 1110 }; 1111 1112 /* Low bits of TAD offset. */ 1113 static const u32 knl_tad_dram_offset_lo[] = { 1114 0x404, 0x504, 0x604, 0x704, 1115 0x804, 0x904, 0xa04, 0xb04, 1116 }; 1117 1118 /* High 16 bits of TAD limit and offset. */ 1119 static const u32 knl_tad_dram_hi[] = { 1120 0x408, 0x508, 0x608, 0x708, 1121 0x808, 0x908, 0xa08, 0xb08, 1122 }; 1123 1124 /* Number of ways a tad entry is interleaved. */ 1125 static const u32 knl_tad_ways[] = { 1126 8, 6, 4, 3, 2, 1, 1127 }; 1128 1129 /* 1130 * Retrieve the n'th Target Address Decode table entry 1131 * from the memory controller's TAD table. 1132 * 1133 * @pvt: driver private data 1134 * @entry: which entry you want to retrieve 1135 * @mc: which memory controller (0 or 1) 1136 * @offset: output tad range offset 1137 * @limit: output address of first byte above tad range 1138 * @ways: output number of interleave ways 1139 * 1140 * The offset value has curious semantics. It's a sort of running total 1141 * of the sizes of all the memory regions that aren't mapped in this 1142 * tad table. 1143 */ 1144 static int knl_get_tad(const struct sbridge_pvt *pvt, 1145 const int entry, 1146 const int mc, 1147 u64 *offset, 1148 u64 *limit, 1149 int *ways) 1150 { 1151 u32 reg_limit_lo, reg_offset_lo, reg_hi; 1152 struct pci_dev *pci_mc; 1153 int way_id; 1154 1155 switch (mc) { 1156 case 0: 1157 pci_mc = pvt->knl.pci_mc0; 1158 break; 1159 case 1: 1160 pci_mc = pvt->knl.pci_mc1; 1161 break; 1162 default: 1163 WARN_ON(1); 1164 return -EINVAL; 1165 } 1166 1167 pci_read_config_dword(pci_mc, 1168 knl_tad_dram_limit_lo[entry], ®_limit_lo); 1169 pci_read_config_dword(pci_mc, 1170 knl_tad_dram_offset_lo[entry], ®_offset_lo); 1171 pci_read_config_dword(pci_mc, 1172 knl_tad_dram_hi[entry], ®_hi); 1173 1174 /* Is this TAD entry enabled? */ 1175 if (!GET_BITFIELD(reg_limit_lo, 0, 0)) 1176 return -ENODEV; 1177 1178 way_id = GET_BITFIELD(reg_limit_lo, 3, 5); 1179 1180 if (way_id < ARRAY_SIZE(knl_tad_ways)) { 1181 *ways = knl_tad_ways[way_id]; 1182 } else { 1183 *ways = 0; 1184 sbridge_printk(KERN_ERR, 1185 "Unexpected value %d in mc_tad_limit_lo wayness field\n", 1186 way_id); 1187 return -ENODEV; 1188 } 1189 1190 /* 1191 * The least significant 6 bits of base and limit are truncated. 1192 * For limit, we fill the missing bits with 1s. 1193 */ 1194 *offset = ((u64) GET_BITFIELD(reg_offset_lo, 6, 31) << 6) | 1195 ((u64) GET_BITFIELD(reg_hi, 0, 15) << 32); 1196 *limit = ((u64) GET_BITFIELD(reg_limit_lo, 6, 31) << 6) | 63 | 1197 ((u64) GET_BITFIELD(reg_hi, 16, 31) << 32); 1198 1199 return 0; 1200 } 1201 1202 /* Determine which memory controller is responsible for a given channel. */ 1203 static int knl_channel_mc(int channel) 1204 { 1205 WARN_ON(channel < 0 || channel >= 6); 1206 1207 return channel < 3 ? 1 : 0; 1208 } 1209 1210 /* 1211 * Get the Nth entry from EDC_ROUTE_TABLE register. 1212 * (This is the per-tile mapping of logical interleave targets to 1213 * physical EDC modules.) 1214 * 1215 * entry 0: 0:2 1216 * 1: 3:5 1217 * 2: 6:8 1218 * 3: 9:11 1219 * 4: 12:14 1220 * 5: 15:17 1221 * 6: 18:20 1222 * 7: 21:23 1223 * reserved: 24:31 1224 */ 1225 static u32 knl_get_edc_route(int entry, u32 reg) 1226 { 1227 WARN_ON(entry >= KNL_MAX_EDCS); 1228 return GET_BITFIELD(reg, entry*3, (entry*3)+2); 1229 } 1230 1231 /* 1232 * Get the Nth entry from MC_ROUTE_TABLE register. 1233 * (This is the per-tile mapping of logical interleave targets to 1234 * physical DRAM channels modules.) 1235 * 1236 * entry 0: mc 0:2 channel 18:19 1237 * 1: mc 3:5 channel 20:21 1238 * 2: mc 6:8 channel 22:23 1239 * 3: mc 9:11 channel 24:25 1240 * 4: mc 12:14 channel 26:27 1241 * 5: mc 15:17 channel 28:29 1242 * reserved: 30:31 1243 * 1244 * Though we have 3 bits to identify the MC, we should only see 1245 * the values 0 or 1. 1246 */ 1247 1248 static u32 knl_get_mc_route(int entry, u32 reg) 1249 { 1250 int mc, chan; 1251 1252 WARN_ON(entry >= KNL_MAX_CHANNELS); 1253 1254 mc = GET_BITFIELD(reg, entry*3, (entry*3)+2); 1255 chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1); 1256 1257 return knl_channel_remap(mc, chan); 1258 } 1259 1260 /* 1261 * Render the EDC_ROUTE register in human-readable form. 1262 * Output string s should be at least KNL_MAX_EDCS*2 bytes. 1263 */ 1264 static void knl_show_edc_route(u32 reg, char *s) 1265 { 1266 int i; 1267 1268 for (i = 0; i < KNL_MAX_EDCS; i++) { 1269 s[i*2] = knl_get_edc_route(i, reg) + '0'; 1270 s[i*2+1] = '-'; 1271 } 1272 1273 s[KNL_MAX_EDCS*2 - 1] = '\0'; 1274 } 1275 1276 /* 1277 * Render the MC_ROUTE register in human-readable form. 1278 * Output string s should be at least KNL_MAX_CHANNELS*2 bytes. 1279 */ 1280 static void knl_show_mc_route(u32 reg, char *s) 1281 { 1282 int i; 1283 1284 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 1285 s[i*2] = knl_get_mc_route(i, reg) + '0'; 1286 s[i*2+1] = '-'; 1287 } 1288 1289 s[KNL_MAX_CHANNELS*2 - 1] = '\0'; 1290 } 1291 1292 #define KNL_EDC_ROUTE 0xb8 1293 #define KNL_MC_ROUTE 0xb4 1294 1295 /* Is this dram rule backed by regular DRAM in flat mode? */ 1296 #define KNL_EDRAM(reg) GET_BITFIELD(reg, 29, 29) 1297 1298 /* Is this dram rule cached? */ 1299 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1300 1301 /* Is this rule backed by edc ? */ 1302 #define KNL_EDRAM_ONLY(reg) GET_BITFIELD(reg, 29, 29) 1303 1304 /* Is this rule backed by DRAM, cacheable in EDRAM? */ 1305 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1306 1307 /* Is this rule mod3? */ 1308 #define KNL_MOD3(reg) GET_BITFIELD(reg, 27, 27) 1309 1310 /* 1311 * Figure out how big our RAM modules are. 1312 * 1313 * The DIMMMTR register in KNL doesn't tell us the size of the DIMMs, so we 1314 * have to figure this out from the SAD rules, interleave lists, route tables, 1315 * and TAD rules. 1316 * 1317 * SAD rules can have holes in them (e.g. the 3G-4G hole), so we have to 1318 * inspect the TAD rules to figure out how large the SAD regions really are. 1319 * 1320 * When we know the real size of a SAD region and how many ways it's 1321 * interleaved, we know the individual contribution of each channel to 1322 * TAD is size/ways. 1323 * 1324 * Finally, we have to check whether each channel participates in each SAD 1325 * region. 1326 * 1327 * Fortunately, KNL only supports one DIMM per channel, so once we know how 1328 * much memory the channel uses, we know the DIMM is at least that large. 1329 * (The BIOS might possibly choose not to map all available memory, in which 1330 * case we will underreport the size of the DIMM.) 1331 * 1332 * In theory, we could try to determine the EDC sizes as well, but that would 1333 * only work in flat mode, not in cache mode. 1334 * 1335 * @mc_sizes: Output sizes of channels (must have space for KNL_MAX_CHANNELS 1336 * elements) 1337 */ 1338 static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes) 1339 { 1340 u64 sad_base, sad_limit = 0; 1341 u64 tad_base, tad_size, tad_limit, tad_deadspace, tad_livespace; 1342 int sad_rule = 0; 1343 int tad_rule = 0; 1344 int intrlv_ways, tad_ways; 1345 u32 first_pkg, pkg; 1346 int i; 1347 u64 sad_actual_size[2]; /* sad size accounting for holes, per mc */ 1348 u32 dram_rule, interleave_reg; 1349 u32 mc_route_reg[KNL_MAX_CHAS]; 1350 u32 edc_route_reg[KNL_MAX_CHAS]; 1351 int edram_only; 1352 char edc_route_string[KNL_MAX_EDCS*2]; 1353 char mc_route_string[KNL_MAX_CHANNELS*2]; 1354 int cur_reg_start; 1355 int mc; 1356 int channel; 1357 int participants[KNL_MAX_CHANNELS]; 1358 1359 for (i = 0; i < KNL_MAX_CHANNELS; i++) 1360 mc_sizes[i] = 0; 1361 1362 /* Read the EDC route table in each CHA. */ 1363 cur_reg_start = 0; 1364 for (i = 0; i < KNL_MAX_CHAS; i++) { 1365 pci_read_config_dword(pvt->knl.pci_cha[i], 1366 KNL_EDC_ROUTE, &edc_route_reg[i]); 1367 1368 if (i > 0 && edc_route_reg[i] != edc_route_reg[i-1]) { 1369 knl_show_edc_route(edc_route_reg[i-1], 1370 edc_route_string); 1371 if (cur_reg_start == i-1) 1372 edac_dbg(0, "edc route table for CHA %d: %s\n", 1373 cur_reg_start, edc_route_string); 1374 else 1375 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1376 cur_reg_start, i-1, edc_route_string); 1377 cur_reg_start = i; 1378 } 1379 } 1380 knl_show_edc_route(edc_route_reg[i-1], edc_route_string); 1381 if (cur_reg_start == i-1) 1382 edac_dbg(0, "edc route table for CHA %d: %s\n", 1383 cur_reg_start, edc_route_string); 1384 else 1385 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1386 cur_reg_start, i-1, edc_route_string); 1387 1388 /* Read the MC route table in each CHA. */ 1389 cur_reg_start = 0; 1390 for (i = 0; i < KNL_MAX_CHAS; i++) { 1391 pci_read_config_dword(pvt->knl.pci_cha[i], 1392 KNL_MC_ROUTE, &mc_route_reg[i]); 1393 1394 if (i > 0 && mc_route_reg[i] != mc_route_reg[i-1]) { 1395 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1396 if (cur_reg_start == i-1) 1397 edac_dbg(0, "mc route table for CHA %d: %s\n", 1398 cur_reg_start, mc_route_string); 1399 else 1400 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1401 cur_reg_start, i-1, mc_route_string); 1402 cur_reg_start = i; 1403 } 1404 } 1405 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1406 if (cur_reg_start == i-1) 1407 edac_dbg(0, "mc route table for CHA %d: %s\n", 1408 cur_reg_start, mc_route_string); 1409 else 1410 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1411 cur_reg_start, i-1, mc_route_string); 1412 1413 /* Process DRAM rules */ 1414 for (sad_rule = 0; sad_rule < pvt->info.max_sad; sad_rule++) { 1415 /* previous limit becomes the new base */ 1416 sad_base = sad_limit; 1417 1418 pci_read_config_dword(pvt->pci_sad0, 1419 pvt->info.dram_rule[sad_rule], &dram_rule); 1420 1421 if (!DRAM_RULE_ENABLE(dram_rule)) 1422 break; 1423 1424 edram_only = KNL_EDRAM_ONLY(dram_rule); 1425 1426 sad_limit = pvt->info.sad_limit(dram_rule)+1; 1427 1428 pci_read_config_dword(pvt->pci_sad0, 1429 pvt->info.interleave_list[sad_rule], &interleave_reg); 1430 1431 /* 1432 * Find out how many ways this dram rule is interleaved. 1433 * We stop when we see the first channel again. 1434 */ 1435 first_pkg = sad_pkg(pvt->info.interleave_pkg, 1436 interleave_reg, 0); 1437 for (intrlv_ways = 1; intrlv_ways < 8; intrlv_ways++) { 1438 pkg = sad_pkg(pvt->info.interleave_pkg, 1439 interleave_reg, intrlv_ways); 1440 1441 if ((pkg & 0x8) == 0) { 1442 /* 1443 * 0 bit means memory is non-local, 1444 * which KNL doesn't support 1445 */ 1446 edac_dbg(0, "Unexpected interleave target %d\n", 1447 pkg); 1448 return -1; 1449 } 1450 1451 if (pkg == first_pkg) 1452 break; 1453 } 1454 if (KNL_MOD3(dram_rule)) 1455 intrlv_ways *= 3; 1456 1457 edac_dbg(3, "dram rule %d (base 0x%llx, limit 0x%llx), %d way interleave%s\n", 1458 sad_rule, 1459 sad_base, 1460 sad_limit, 1461 intrlv_ways, 1462 edram_only ? ", EDRAM" : ""); 1463 1464 /* 1465 * Find out how big the SAD region really is by iterating 1466 * over TAD tables (SAD regions may contain holes). 1467 * Each memory controller might have a different TAD table, so 1468 * we have to look at both. 1469 * 1470 * Livespace is the memory that's mapped in this TAD table, 1471 * deadspace is the holes (this could be the MMIO hole, or it 1472 * could be memory that's mapped by the other TAD table but 1473 * not this one). 1474 */ 1475 for (mc = 0; mc < 2; mc++) { 1476 sad_actual_size[mc] = 0; 1477 tad_livespace = 0; 1478 for (tad_rule = 0; 1479 tad_rule < ARRAY_SIZE( 1480 knl_tad_dram_limit_lo); 1481 tad_rule++) { 1482 if (knl_get_tad(pvt, 1483 tad_rule, 1484 mc, 1485 &tad_deadspace, 1486 &tad_limit, 1487 &tad_ways)) 1488 break; 1489 1490 tad_size = (tad_limit+1) - 1491 (tad_livespace + tad_deadspace); 1492 tad_livespace += tad_size; 1493 tad_base = (tad_limit+1) - tad_size; 1494 1495 if (tad_base < sad_base) { 1496 if (tad_limit > sad_base) 1497 edac_dbg(0, "TAD region overlaps lower SAD boundary -- TAD tables may be configured incorrectly.\n"); 1498 } else if (tad_base < sad_limit) { 1499 if (tad_limit+1 > sad_limit) { 1500 edac_dbg(0, "TAD region overlaps upper SAD boundary -- TAD tables may be configured incorrectly.\n"); 1501 } else { 1502 /* TAD region is completely inside SAD region */ 1503 edac_dbg(3, "TAD region %d 0x%llx - 0x%llx (%lld bytes) table%d\n", 1504 tad_rule, tad_base, 1505 tad_limit, tad_size, 1506 mc); 1507 sad_actual_size[mc] += tad_size; 1508 } 1509 } 1510 } 1511 } 1512 1513 for (mc = 0; mc < 2; mc++) { 1514 edac_dbg(3, " total TAD DRAM footprint in table%d : 0x%llx (%lld bytes)\n", 1515 mc, sad_actual_size[mc], sad_actual_size[mc]); 1516 } 1517 1518 /* Ignore EDRAM rule */ 1519 if (edram_only) 1520 continue; 1521 1522 /* Figure out which channels participate in interleave. */ 1523 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) 1524 participants[channel] = 0; 1525 1526 /* For each channel, does at least one CHA have 1527 * this channel mapped to the given target? 1528 */ 1529 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1530 int target; 1531 int cha; 1532 1533 for (target = 0; target < KNL_MAX_CHANNELS; target++) { 1534 for (cha = 0; cha < KNL_MAX_CHAS; cha++) { 1535 if (knl_get_mc_route(target, 1536 mc_route_reg[cha]) == channel 1537 && !participants[channel]) { 1538 participants[channel] = 1; 1539 break; 1540 } 1541 } 1542 } 1543 } 1544 1545 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1546 mc = knl_channel_mc(channel); 1547 if (participants[channel]) { 1548 edac_dbg(4, "mc channel %d contributes %lld bytes via sad entry %d\n", 1549 channel, 1550 sad_actual_size[mc]/intrlv_ways, 1551 sad_rule); 1552 mc_sizes[channel] += 1553 sad_actual_size[mc]/intrlv_ways; 1554 } 1555 } 1556 } 1557 1558 return 0; 1559 } 1560 1561 static void get_source_id(struct mem_ctl_info *mci) 1562 { 1563 struct sbridge_pvt *pvt = mci->pvt_info; 1564 u32 reg; 1565 1566 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL || 1567 pvt->info.type == KNIGHTS_LANDING) 1568 pci_read_config_dword(pvt->pci_sad1, SAD_TARGET, ®); 1569 else 1570 pci_read_config_dword(pvt->pci_br0, SAD_TARGET, ®); 1571 1572 if (pvt->info.type == KNIGHTS_LANDING) 1573 pvt->sbridge_dev->source_id = SOURCE_ID_KNL(reg); 1574 else 1575 pvt->sbridge_dev->source_id = SOURCE_ID(reg); 1576 } 1577 1578 static int __populate_dimms(struct mem_ctl_info *mci, 1579 u64 knl_mc_sizes[KNL_MAX_CHANNELS], 1580 enum edac_type mode) 1581 { 1582 struct sbridge_pvt *pvt = mci->pvt_info; 1583 int channels = pvt->info.type == KNIGHTS_LANDING ? KNL_MAX_CHANNELS 1584 : NUM_CHANNELS; 1585 unsigned int i, j, banks, ranks, rows, cols, npages; 1586 struct dimm_info *dimm; 1587 enum mem_type mtype; 1588 u64 size; 1589 1590 mtype = pvt->info.get_memory_type(pvt); 1591 if (mtype == MEM_RDDR3 || mtype == MEM_RDDR4) 1592 edac_dbg(0, "Memory is registered\n"); 1593 else if (mtype == MEM_UNKNOWN) 1594 edac_dbg(0, "Cannot determine memory type\n"); 1595 else 1596 edac_dbg(0, "Memory is unregistered\n"); 1597 1598 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4) 1599 banks = 16; 1600 else 1601 banks = 8; 1602 1603 for (i = 0; i < channels; i++) { 1604 u32 mtr, amap = 0; 1605 1606 int max_dimms_per_channel; 1607 1608 if (pvt->info.type == KNIGHTS_LANDING) { 1609 max_dimms_per_channel = 1; 1610 if (!pvt->knl.pci_channel[i]) 1611 continue; 1612 } else { 1613 max_dimms_per_channel = ARRAY_SIZE(mtr_regs); 1614 if (!pvt->pci_tad[i]) 1615 continue; 1616 pci_read_config_dword(pvt->pci_tad[i], 0x8c, &amap); 1617 } 1618 1619 for (j = 0; j < max_dimms_per_channel; j++) { 1620 dimm = edac_get_dimm(mci, i, j, 0); 1621 if (pvt->info.type == KNIGHTS_LANDING) { 1622 pci_read_config_dword(pvt->knl.pci_channel[i], 1623 knl_mtr_reg, &mtr); 1624 } else { 1625 pci_read_config_dword(pvt->pci_tad[i], 1626 mtr_regs[j], &mtr); 1627 } 1628 edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr); 1629 1630 if (IS_DIMM_PRESENT(mtr)) { 1631 if (!IS_ECC_ENABLED(pvt->info.mcmtr)) { 1632 sbridge_printk(KERN_ERR, "CPU SrcID #%d, Ha #%d, Channel #%d has DIMMs, but ECC is disabled\n", 1633 pvt->sbridge_dev->source_id, 1634 pvt->sbridge_dev->dom, i); 1635 return -ENODEV; 1636 } 1637 pvt->channel[i].dimms++; 1638 1639 ranks = numrank(pvt->info.type, mtr); 1640 1641 if (pvt->info.type == KNIGHTS_LANDING) { 1642 /* For DDR4, this is fixed. */ 1643 cols = 1 << 10; 1644 rows = knl_mc_sizes[i] / 1645 ((u64) cols * ranks * banks * 8); 1646 } else { 1647 rows = numrow(mtr); 1648 cols = numcol(mtr); 1649 } 1650 1651 size = ((u64)rows * cols * banks * ranks) >> (20 - 3); 1652 npages = MiB_TO_PAGES(size); 1653 1654 edac_dbg(0, "mc#%d: ha %d channel %d, dimm %d, %lld MiB (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n", 1655 pvt->sbridge_dev->mc, pvt->sbridge_dev->dom, i, j, 1656 size, npages, 1657 banks, ranks, rows, cols); 1658 1659 dimm->nr_pages = npages; 1660 dimm->grain = 32; 1661 dimm->dtype = pvt->info.get_width(pvt, mtr); 1662 dimm->mtype = mtype; 1663 dimm->edac_mode = mode; 1664 pvt->channel[i].dimm[j].rowbits = order_base_2(rows); 1665 pvt->channel[i].dimm[j].colbits = order_base_2(cols); 1666 pvt->channel[i].dimm[j].bank_xor_enable = 1667 GET_BITFIELD(pvt->info.mcmtr, 9, 9); 1668 pvt->channel[i].dimm[j].amap_fine = GET_BITFIELD(amap, 0, 0); 1669 snprintf(dimm->label, sizeof(dimm->label), 1670 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u", 1671 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom, i, j); 1672 } 1673 } 1674 } 1675 1676 return 0; 1677 } 1678 1679 static int get_dimm_config(struct mem_ctl_info *mci) 1680 { 1681 struct sbridge_pvt *pvt = mci->pvt_info; 1682 u64 knl_mc_sizes[KNL_MAX_CHANNELS]; 1683 enum edac_type mode; 1684 u32 reg; 1685 1686 pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt); 1687 edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n", 1688 pvt->sbridge_dev->mc, 1689 pvt->sbridge_dev->node_id, 1690 pvt->sbridge_dev->source_id); 1691 1692 /* KNL doesn't support mirroring or lockstep, 1693 * and is always closed page 1694 */ 1695 if (pvt->info.type == KNIGHTS_LANDING) { 1696 mode = EDAC_S4ECD4ED; 1697 pvt->mirror_mode = NON_MIRRORING; 1698 pvt->is_cur_addr_mirrored = false; 1699 1700 if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0) 1701 return -1; 1702 if (pci_read_config_dword(pvt->pci_ta, KNL_MCMTR, &pvt->info.mcmtr)) { 1703 edac_dbg(0, "Failed to read KNL_MCMTR register\n"); 1704 return -ENODEV; 1705 } 1706 } else { 1707 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 1708 if (pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®)) { 1709 edac_dbg(0, "Failed to read HASWELL_HASYSDEFEATURE2 register\n"); 1710 return -ENODEV; 1711 } 1712 pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21); 1713 if (GET_BITFIELD(reg, 28, 28)) { 1714 pvt->mirror_mode = ADDR_RANGE_MIRRORING; 1715 edac_dbg(0, "Address range partial memory mirroring is enabled\n"); 1716 goto next; 1717 } 1718 } 1719 if (pci_read_config_dword(pvt->pci_ras, RASENABLES, ®)) { 1720 edac_dbg(0, "Failed to read RASENABLES register\n"); 1721 return -ENODEV; 1722 } 1723 if (IS_MIRROR_ENABLED(reg)) { 1724 pvt->mirror_mode = FULL_MIRRORING; 1725 edac_dbg(0, "Full memory mirroring is enabled\n"); 1726 } else { 1727 pvt->mirror_mode = NON_MIRRORING; 1728 edac_dbg(0, "Memory mirroring is disabled\n"); 1729 } 1730 1731 next: 1732 if (pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr)) { 1733 edac_dbg(0, "Failed to read MCMTR register\n"); 1734 return -ENODEV; 1735 } 1736 if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) { 1737 edac_dbg(0, "Lockstep is enabled\n"); 1738 mode = EDAC_S8ECD8ED; 1739 pvt->is_lockstep = true; 1740 } else { 1741 edac_dbg(0, "Lockstep is disabled\n"); 1742 mode = EDAC_S4ECD4ED; 1743 pvt->is_lockstep = false; 1744 } 1745 if (IS_CLOSE_PG(pvt->info.mcmtr)) { 1746 edac_dbg(0, "address map is on closed page mode\n"); 1747 pvt->is_close_pg = true; 1748 } else { 1749 edac_dbg(0, "address map is on open page mode\n"); 1750 pvt->is_close_pg = false; 1751 } 1752 } 1753 1754 return __populate_dimms(mci, knl_mc_sizes, mode); 1755 } 1756 1757 static void get_memory_layout(const struct mem_ctl_info *mci) 1758 { 1759 struct sbridge_pvt *pvt = mci->pvt_info; 1760 int i, j, k, n_sads, n_tads, sad_interl; 1761 u32 reg; 1762 u64 limit, prv = 0; 1763 u64 tmp_mb; 1764 u32 gb, mb; 1765 u32 rir_way; 1766 1767 /* 1768 * Step 1) Get TOLM/TOHM ranges 1769 */ 1770 1771 pvt->tolm = pvt->info.get_tolm(pvt); 1772 tmp_mb = (1 + pvt->tolm) >> 20; 1773 1774 gb = div_u64_rem(tmp_mb, 1024, &mb); 1775 edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n", 1776 gb, (mb*1000)/1024, (u64)pvt->tolm); 1777 1778 /* Address range is already 45:25 */ 1779 pvt->tohm = pvt->info.get_tohm(pvt); 1780 tmp_mb = (1 + pvt->tohm) >> 20; 1781 1782 gb = div_u64_rem(tmp_mb, 1024, &mb); 1783 edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n", 1784 gb, (mb*1000)/1024, (u64)pvt->tohm); 1785 1786 /* 1787 * Step 2) Get SAD range and SAD Interleave list 1788 * TAD registers contain the interleave wayness. However, it 1789 * seems simpler to just discover it indirectly, with the 1790 * algorithm bellow. 1791 */ 1792 prv = 0; 1793 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 1794 /* SAD_LIMIT Address range is 45:26 */ 1795 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 1796 ®); 1797 limit = pvt->info.sad_limit(reg); 1798 1799 if (!DRAM_RULE_ENABLE(reg)) 1800 continue; 1801 1802 if (limit <= prv) 1803 break; 1804 1805 tmp_mb = (limit + 1) >> 20; 1806 gb = div_u64_rem(tmp_mb, 1024, &mb); 1807 edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n", 1808 n_sads, 1809 show_dram_attr(pvt->info.dram_attr(reg)), 1810 gb, (mb*1000)/1024, 1811 ((u64)tmp_mb) << 20L, 1812 get_intlv_mode_str(reg, pvt->info.type), 1813 reg); 1814 prv = limit; 1815 1816 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 1817 ®); 1818 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 1819 for (j = 0; j < 8; j++) { 1820 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, j); 1821 if (j > 0 && sad_interl == pkg) 1822 break; 1823 1824 edac_dbg(0, "SAD#%d, interleave #%d: %d\n", 1825 n_sads, j, pkg); 1826 } 1827 } 1828 1829 if (pvt->info.type == KNIGHTS_LANDING) 1830 return; 1831 1832 /* 1833 * Step 3) Get TAD range 1834 */ 1835 prv = 0; 1836 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 1837 pci_read_config_dword(pvt->pci_ha, tad_dram_rule[n_tads], ®); 1838 limit = TAD_LIMIT(reg); 1839 if (limit <= prv) 1840 break; 1841 tmp_mb = (limit + 1) >> 20; 1842 1843 gb = div_u64_rem(tmp_mb, 1024, &mb); 1844 edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n", 1845 n_tads, gb, (mb*1000)/1024, 1846 ((u64)tmp_mb) << 20L, 1847 (u32)(1 << TAD_SOCK(reg)), 1848 (u32)TAD_CH(reg) + 1, 1849 (u32)TAD_TGT0(reg), 1850 (u32)TAD_TGT1(reg), 1851 (u32)TAD_TGT2(reg), 1852 (u32)TAD_TGT3(reg), 1853 reg); 1854 prv = limit; 1855 } 1856 1857 /* 1858 * Step 4) Get TAD offsets, per each channel 1859 */ 1860 for (i = 0; i < NUM_CHANNELS; i++) { 1861 if (!pvt->channel[i].dimms) 1862 continue; 1863 for (j = 0; j < n_tads; j++) { 1864 pci_read_config_dword(pvt->pci_tad[i], 1865 tad_ch_nilv_offset[j], 1866 ®); 1867 tmp_mb = TAD_OFFSET(reg) >> 20; 1868 gb = div_u64_rem(tmp_mb, 1024, &mb); 1869 edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n", 1870 i, j, 1871 gb, (mb*1000)/1024, 1872 ((u64)tmp_mb) << 20L, 1873 reg); 1874 } 1875 } 1876 1877 /* 1878 * Step 6) Get RIR Wayness/Limit, per each channel 1879 */ 1880 for (i = 0; i < NUM_CHANNELS; i++) { 1881 if (!pvt->channel[i].dimms) 1882 continue; 1883 for (j = 0; j < MAX_RIR_RANGES; j++) { 1884 pci_read_config_dword(pvt->pci_tad[i], 1885 rir_way_limit[j], 1886 ®); 1887 1888 if (!IS_RIR_VALID(reg)) 1889 continue; 1890 1891 tmp_mb = pvt->info.rir_limit(reg) >> 20; 1892 rir_way = 1 << RIR_WAY(reg); 1893 gb = div_u64_rem(tmp_mb, 1024, &mb); 1894 edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n", 1895 i, j, 1896 gb, (mb*1000)/1024, 1897 ((u64)tmp_mb) << 20L, 1898 rir_way, 1899 reg); 1900 1901 for (k = 0; k < rir_way; k++) { 1902 pci_read_config_dword(pvt->pci_tad[i], 1903 rir_offset[j][k], 1904 ®); 1905 tmp_mb = RIR_OFFSET(pvt->info.type, reg) << 6; 1906 1907 gb = div_u64_rem(tmp_mb, 1024, &mb); 1908 edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n", 1909 i, j, k, 1910 gb, (mb*1000)/1024, 1911 ((u64)tmp_mb) << 20L, 1912 (u32)RIR_RNK_TGT(pvt->info.type, reg), 1913 reg); 1914 } 1915 } 1916 } 1917 } 1918 1919 static struct mem_ctl_info *get_mci_for_node_id(u8 node_id, u8 ha) 1920 { 1921 struct sbridge_dev *sbridge_dev; 1922 1923 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 1924 if (sbridge_dev->node_id == node_id && sbridge_dev->dom == ha) 1925 return sbridge_dev->mci; 1926 } 1927 return NULL; 1928 } 1929 1930 static u8 sb_close_row[] = { 1931 15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33 1932 }; 1933 1934 static u8 sb_close_column[] = { 1935 3, 4, 5, 14, 19, 23, 24, 25, 26, 27 1936 }; 1937 1938 static u8 sb_open_row[] = { 1939 14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33 1940 }; 1941 1942 static u8 sb_open_column[] = { 1943 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 1944 }; 1945 1946 static u8 sb_open_fine_column[] = { 1947 3, 4, 5, 7, 8, 9, 10, 11, 12, 13 1948 }; 1949 1950 static int sb_bits(u64 addr, int nbits, u8 *bits) 1951 { 1952 int i, res = 0; 1953 1954 for (i = 0; i < nbits; i++) 1955 res |= ((addr >> bits[i]) & 1) << i; 1956 return res; 1957 } 1958 1959 static int sb_bank_bits(u64 addr, int b0, int b1, int do_xor, int x0, int x1) 1960 { 1961 int ret = GET_BITFIELD(addr, b0, b0) | (GET_BITFIELD(addr, b1, b1) << 1); 1962 1963 if (do_xor) 1964 ret ^= GET_BITFIELD(addr, x0, x0) | (GET_BITFIELD(addr, x1, x1) << 1); 1965 1966 return ret; 1967 } 1968 1969 static bool sb_decode_ddr4(struct mem_ctl_info *mci, int ch, u8 rank, 1970 u64 rank_addr, char *msg) 1971 { 1972 int dimmno = 0; 1973 int row, col, bank_address, bank_group; 1974 struct sbridge_pvt *pvt; 1975 u32 bg0 = 0, rowbits = 0, colbits = 0; 1976 u32 amap_fine = 0, bank_xor_enable = 0; 1977 1978 dimmno = (rank < 12) ? rank / 4 : 2; 1979 pvt = mci->pvt_info; 1980 amap_fine = pvt->channel[ch].dimm[dimmno].amap_fine; 1981 bg0 = amap_fine ? 6 : 13; 1982 rowbits = pvt->channel[ch].dimm[dimmno].rowbits; 1983 colbits = pvt->channel[ch].dimm[dimmno].colbits; 1984 bank_xor_enable = pvt->channel[ch].dimm[dimmno].bank_xor_enable; 1985 1986 if (pvt->is_lockstep) { 1987 pr_warn_once("LockStep row/column decode is not supported yet!\n"); 1988 msg[0] = '\0'; 1989 return false; 1990 } 1991 1992 if (pvt->is_close_pg) { 1993 row = sb_bits(rank_addr, rowbits, sb_close_row); 1994 col = sb_bits(rank_addr, colbits, sb_close_column); 1995 col |= 0x400; /* C10 is autoprecharge, always set */ 1996 bank_address = sb_bank_bits(rank_addr, 8, 9, bank_xor_enable, 22, 28); 1997 bank_group = sb_bank_bits(rank_addr, 6, 7, bank_xor_enable, 20, 21); 1998 } else { 1999 row = sb_bits(rank_addr, rowbits, sb_open_row); 2000 if (amap_fine) 2001 col = sb_bits(rank_addr, colbits, sb_open_fine_column); 2002 else 2003 col = sb_bits(rank_addr, colbits, sb_open_column); 2004 bank_address = sb_bank_bits(rank_addr, 18, 19, bank_xor_enable, 22, 23); 2005 bank_group = sb_bank_bits(rank_addr, bg0, 17, bank_xor_enable, 20, 21); 2006 } 2007 2008 row &= (1u << rowbits) - 1; 2009 2010 sprintf(msg, "row:0x%x col:0x%x bank_addr:%d bank_group:%d", 2011 row, col, bank_address, bank_group); 2012 return true; 2013 } 2014 2015 static bool sb_decode_ddr3(struct mem_ctl_info *mci, int ch, u8 rank, 2016 u64 rank_addr, char *msg) 2017 { 2018 pr_warn_once("DDR3 row/column decode is not supported yet!\n"); 2019 msg[0] = '\0'; 2020 return false; 2021 } 2022 2023 static int get_memory_error_data(struct mem_ctl_info *mci, 2024 u64 addr, 2025 u8 *socket, u8 *ha, 2026 long *channel_mask, 2027 u8 *rank, 2028 char **area_type, char *msg) 2029 { 2030 struct mem_ctl_info *new_mci; 2031 struct sbridge_pvt *pvt = mci->pvt_info; 2032 struct pci_dev *pci_ha; 2033 int n_rir, n_sads, n_tads, sad_way, sck_xch; 2034 int sad_interl, idx, base_ch; 2035 int interleave_mode, shiftup = 0; 2036 unsigned int sad_interleave[MAX_INTERLEAVE]; 2037 u32 reg, dram_rule; 2038 u8 ch_way, sck_way, pkg, sad_ha = 0, rankid = 0; 2039 u32 tad_offset; 2040 u32 rir_way; 2041 u32 mb, gb; 2042 u64 ch_addr, offset, limit = 0, prv = 0; 2043 u64 rank_addr; 2044 enum mem_type mtype; 2045 2046 /* 2047 * Step 0) Check if the address is at special memory ranges 2048 * The check bellow is probably enough to fill all cases where 2049 * the error is not inside a memory, except for the legacy 2050 * range (e. g. VGA addresses). It is unlikely, however, that the 2051 * memory controller would generate an error on that range. 2052 */ 2053 if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) { 2054 sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr); 2055 return -EINVAL; 2056 } 2057 if (addr >= (u64)pvt->tohm) { 2058 sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr); 2059 return -EINVAL; 2060 } 2061 2062 /* 2063 * Step 1) Get socket 2064 */ 2065 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 2066 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 2067 ®); 2068 2069 if (!DRAM_RULE_ENABLE(reg)) 2070 continue; 2071 2072 limit = pvt->info.sad_limit(reg); 2073 if (limit <= prv) { 2074 sprintf(msg, "Can't discover the memory socket"); 2075 return -EINVAL; 2076 } 2077 if (addr <= limit) 2078 break; 2079 prv = limit; 2080 } 2081 if (n_sads == pvt->info.max_sad) { 2082 sprintf(msg, "Can't discover the memory socket"); 2083 return -EINVAL; 2084 } 2085 dram_rule = reg; 2086 *area_type = show_dram_attr(pvt->info.dram_attr(dram_rule)); 2087 interleave_mode = pvt->info.interleave_mode(dram_rule); 2088 2089 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 2090 ®); 2091 2092 if (pvt->info.type == SANDY_BRIDGE) { 2093 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 2094 for (sad_way = 0; sad_way < 8; sad_way++) { 2095 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, sad_way); 2096 if (sad_way > 0 && sad_interl == pkg) 2097 break; 2098 sad_interleave[sad_way] = pkg; 2099 edac_dbg(0, "SAD interleave #%d: %d\n", 2100 sad_way, sad_interleave[sad_way]); 2101 } 2102 edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n", 2103 pvt->sbridge_dev->mc, 2104 n_sads, 2105 addr, 2106 limit, 2107 sad_way + 7, 2108 !interleave_mode ? "" : "XOR[18:16]"); 2109 if (interleave_mode) 2110 idx = ((addr >> 6) ^ (addr >> 16)) & 7; 2111 else 2112 idx = (addr >> 6) & 7; 2113 switch (sad_way) { 2114 case 1: 2115 idx = 0; 2116 break; 2117 case 2: 2118 idx = idx & 1; 2119 break; 2120 case 4: 2121 idx = idx & 3; 2122 break; 2123 case 8: 2124 break; 2125 default: 2126 sprintf(msg, "Can't discover socket interleave"); 2127 return -EINVAL; 2128 } 2129 *socket = sad_interleave[idx]; 2130 edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n", 2131 idx, sad_way, *socket); 2132 } else if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 2133 int bits, a7mode = A7MODE(dram_rule); 2134 2135 if (a7mode) { 2136 /* A7 mode swaps P9 with P6 */ 2137 bits = GET_BITFIELD(addr, 7, 8) << 1; 2138 bits |= GET_BITFIELD(addr, 9, 9); 2139 } else 2140 bits = GET_BITFIELD(addr, 6, 8); 2141 2142 if (interleave_mode == 0) { 2143 /* interleave mode will XOR {8,7,6} with {18,17,16} */ 2144 idx = GET_BITFIELD(addr, 16, 18); 2145 idx ^= bits; 2146 } else 2147 idx = bits; 2148 2149 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2150 *socket = sad_pkg_socket(pkg); 2151 sad_ha = sad_pkg_ha(pkg); 2152 2153 if (a7mode) { 2154 /* MCChanShiftUpEnable */ 2155 pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®); 2156 shiftup = GET_BITFIELD(reg, 22, 22); 2157 } 2158 2159 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %i, shiftup: %i\n", 2160 idx, *socket, sad_ha, shiftup); 2161 } else { 2162 /* Ivy Bridge's SAD mode doesn't support XOR interleave mode */ 2163 idx = (addr >> 6) & 7; 2164 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2165 *socket = sad_pkg_socket(pkg); 2166 sad_ha = sad_pkg_ha(pkg); 2167 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %d\n", 2168 idx, *socket, sad_ha); 2169 } 2170 2171 *ha = sad_ha; 2172 2173 /* 2174 * Move to the proper node structure, in order to access the 2175 * right PCI registers 2176 */ 2177 new_mci = get_mci_for_node_id(*socket, sad_ha); 2178 if (!new_mci) { 2179 sprintf(msg, "Struct for socket #%u wasn't initialized", 2180 *socket); 2181 return -EINVAL; 2182 } 2183 mci = new_mci; 2184 pvt = mci->pvt_info; 2185 2186 /* 2187 * Step 2) Get memory channel 2188 */ 2189 prv = 0; 2190 pci_ha = pvt->pci_ha; 2191 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 2192 pci_read_config_dword(pci_ha, tad_dram_rule[n_tads], ®); 2193 limit = TAD_LIMIT(reg); 2194 if (limit <= prv) { 2195 sprintf(msg, "Can't discover the memory channel"); 2196 return -EINVAL; 2197 } 2198 if (addr <= limit) 2199 break; 2200 prv = limit; 2201 } 2202 if (n_tads == MAX_TAD) { 2203 sprintf(msg, "Can't discover the memory channel"); 2204 return -EINVAL; 2205 } 2206 2207 ch_way = TAD_CH(reg) + 1; 2208 sck_way = TAD_SOCK(reg); 2209 2210 if (ch_way == 3) 2211 idx = addr >> 6; 2212 else { 2213 idx = (addr >> (6 + sck_way + shiftup)) & 0x3; 2214 if (pvt->is_chan_hash) 2215 idx = haswell_chan_hash(idx, addr); 2216 } 2217 idx = idx % ch_way; 2218 2219 /* 2220 * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ??? 2221 */ 2222 switch (idx) { 2223 case 0: 2224 base_ch = TAD_TGT0(reg); 2225 break; 2226 case 1: 2227 base_ch = TAD_TGT1(reg); 2228 break; 2229 case 2: 2230 base_ch = TAD_TGT2(reg); 2231 break; 2232 case 3: 2233 base_ch = TAD_TGT3(reg); 2234 break; 2235 default: 2236 sprintf(msg, "Can't discover the TAD target"); 2237 return -EINVAL; 2238 } 2239 *channel_mask = 1 << base_ch; 2240 2241 pci_read_config_dword(pvt->pci_tad[base_ch], tad_ch_nilv_offset[n_tads], &tad_offset); 2242 2243 if (pvt->mirror_mode == FULL_MIRRORING || 2244 (pvt->mirror_mode == ADDR_RANGE_MIRRORING && n_tads == 0)) { 2245 *channel_mask |= 1 << ((base_ch + 2) % 4); 2246 switch(ch_way) { 2247 case 2: 2248 case 4: 2249 sck_xch = (1 << sck_way) * (ch_way >> 1); 2250 break; 2251 default: 2252 sprintf(msg, "Invalid mirror set. Can't decode addr"); 2253 return -EINVAL; 2254 } 2255 2256 pvt->is_cur_addr_mirrored = true; 2257 } else { 2258 sck_xch = (1 << sck_way) * ch_way; 2259 pvt->is_cur_addr_mirrored = false; 2260 } 2261 2262 if (pvt->is_lockstep) 2263 *channel_mask |= 1 << ((base_ch + 1) % 4); 2264 2265 offset = TAD_OFFSET(tad_offset); 2266 2267 edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n", 2268 n_tads, 2269 addr, 2270 limit, 2271 sck_way, 2272 ch_way, 2273 offset, 2274 idx, 2275 base_ch, 2276 *channel_mask); 2277 2278 /* Calculate channel address */ 2279 /* Remove the TAD offset */ 2280 2281 if (offset > addr) { 2282 sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!", 2283 offset, addr); 2284 return -EINVAL; 2285 } 2286 2287 ch_addr = addr - offset; 2288 ch_addr >>= (6 + shiftup); 2289 ch_addr /= sck_xch; 2290 ch_addr <<= (6 + shiftup); 2291 ch_addr |= addr & ((1 << (6 + shiftup)) - 1); 2292 2293 /* 2294 * Step 3) Decode rank 2295 */ 2296 for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) { 2297 pci_read_config_dword(pvt->pci_tad[base_ch], rir_way_limit[n_rir], ®); 2298 2299 if (!IS_RIR_VALID(reg)) 2300 continue; 2301 2302 limit = pvt->info.rir_limit(reg); 2303 gb = div_u64_rem(limit >> 20, 1024, &mb); 2304 edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n", 2305 n_rir, 2306 gb, (mb*1000)/1024, 2307 limit, 2308 1 << RIR_WAY(reg)); 2309 if (ch_addr <= limit) 2310 break; 2311 } 2312 if (n_rir == MAX_RIR_RANGES) { 2313 sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx", 2314 ch_addr); 2315 return -EINVAL; 2316 } 2317 rir_way = RIR_WAY(reg); 2318 2319 if (pvt->is_close_pg) 2320 idx = (ch_addr >> 6); 2321 else 2322 idx = (ch_addr >> 13); /* FIXME: Datasheet says to shift by 15 */ 2323 idx %= 1 << rir_way; 2324 2325 pci_read_config_dword(pvt->pci_tad[base_ch], rir_offset[n_rir][idx], ®); 2326 *rank = RIR_RNK_TGT(pvt->info.type, reg); 2327 2328 if (pvt->info.type == BROADWELL) { 2329 if (pvt->is_close_pg) 2330 shiftup = 6; 2331 else 2332 shiftup = 13; 2333 2334 rank_addr = ch_addr >> shiftup; 2335 rank_addr /= (1 << rir_way); 2336 rank_addr <<= shiftup; 2337 rank_addr |= ch_addr & GENMASK_ULL(shiftup - 1, 0); 2338 rank_addr -= RIR_OFFSET(pvt->info.type, reg); 2339 2340 mtype = pvt->info.get_memory_type(pvt); 2341 rankid = *rank; 2342 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4) 2343 sb_decode_ddr4(mci, base_ch, rankid, rank_addr, msg); 2344 else 2345 sb_decode_ddr3(mci, base_ch, rankid, rank_addr, msg); 2346 } else { 2347 msg[0] = '\0'; 2348 } 2349 2350 edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n", 2351 n_rir, 2352 ch_addr, 2353 limit, 2354 rir_way, 2355 idx); 2356 2357 return 0; 2358 } 2359 2360 static int get_memory_error_data_from_mce(struct mem_ctl_info *mci, 2361 const struct mce *m, u8 *socket, 2362 u8 *ha, long *channel_mask, 2363 char *msg) 2364 { 2365 u32 reg, channel = GET_BITFIELD(m->status, 0, 3); 2366 struct mem_ctl_info *new_mci; 2367 struct sbridge_pvt *pvt; 2368 struct pci_dev *pci_ha; 2369 bool tad0; 2370 2371 if (channel >= NUM_CHANNELS) { 2372 sprintf(msg, "Invalid channel 0x%x", channel); 2373 return -EINVAL; 2374 } 2375 2376 pvt = mci->pvt_info; 2377 if (!pvt->info.get_ha) { 2378 sprintf(msg, "No get_ha()"); 2379 return -EINVAL; 2380 } 2381 *ha = pvt->info.get_ha(m->bank); 2382 if (*ha != 0 && *ha != 1) { 2383 sprintf(msg, "Impossible bank %d", m->bank); 2384 return -EINVAL; 2385 } 2386 2387 *socket = m->socketid; 2388 new_mci = get_mci_for_node_id(*socket, *ha); 2389 if (!new_mci) { 2390 strcpy(msg, "mci socket got corrupted!"); 2391 return -EINVAL; 2392 } 2393 2394 pvt = new_mci->pvt_info; 2395 pci_ha = pvt->pci_ha; 2396 pci_read_config_dword(pci_ha, tad_dram_rule[0], ®); 2397 tad0 = m->addr <= TAD_LIMIT(reg); 2398 2399 *channel_mask = 1 << channel; 2400 if (pvt->mirror_mode == FULL_MIRRORING || 2401 (pvt->mirror_mode == ADDR_RANGE_MIRRORING && tad0)) { 2402 *channel_mask |= 1 << ((channel + 2) % 4); 2403 pvt->is_cur_addr_mirrored = true; 2404 } else { 2405 pvt->is_cur_addr_mirrored = false; 2406 } 2407 2408 if (pvt->is_lockstep) 2409 *channel_mask |= 1 << ((channel + 1) % 4); 2410 2411 return 0; 2412 } 2413 2414 /**************************************************************************** 2415 Device initialization routines: put/get, init/exit 2416 ****************************************************************************/ 2417 2418 /* 2419 * sbridge_put_all_devices 'put' all the devices that we have 2420 * reserved via 'get' 2421 */ 2422 static void sbridge_put_devices(struct sbridge_dev *sbridge_dev) 2423 { 2424 int i; 2425 2426 edac_dbg(0, "\n"); 2427 for (i = 0; i < sbridge_dev->n_devs; i++) { 2428 struct pci_dev *pdev = sbridge_dev->pdev[i]; 2429 if (!pdev) 2430 continue; 2431 edac_dbg(0, "Removing dev %02x:%02x.%d\n", 2432 pdev->bus->number, 2433 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); 2434 pci_dev_put(pdev); 2435 } 2436 } 2437 2438 static void sbridge_put_all_devices(void) 2439 { 2440 struct sbridge_dev *sbridge_dev, *tmp; 2441 2442 list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) { 2443 sbridge_put_devices(sbridge_dev); 2444 free_sbridge_dev(sbridge_dev); 2445 } 2446 } 2447 2448 static int sbridge_get_onedevice(struct pci_dev **prev, 2449 u8 *num_mc, 2450 const struct pci_id_table *table, 2451 const unsigned devno, 2452 const int multi_bus) 2453 { 2454 struct sbridge_dev *sbridge_dev = NULL; 2455 const struct pci_id_descr *dev_descr = &table->descr[devno]; 2456 struct pci_dev *pdev = NULL; 2457 int seg = 0; 2458 u8 bus = 0; 2459 int i = 0; 2460 2461 sbridge_printk(KERN_DEBUG, 2462 "Seeking for: PCI ID %04x:%04x\n", 2463 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2464 2465 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 2466 dev_descr->dev_id, *prev); 2467 2468 if (!pdev) { 2469 if (*prev) { 2470 *prev = pdev; 2471 return 0; 2472 } 2473 2474 if (dev_descr->optional) 2475 return 0; 2476 2477 /* if the HA wasn't found */ 2478 if (devno == 0) 2479 return -ENODEV; 2480 2481 sbridge_printk(KERN_INFO, 2482 "Device not found: %04x:%04x\n", 2483 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2484 2485 /* End of list, leave */ 2486 return -ENODEV; 2487 } 2488 seg = pci_domain_nr(pdev->bus); 2489 bus = pdev->bus->number; 2490 2491 next_imc: 2492 sbridge_dev = get_sbridge_dev(seg, bus, dev_descr->dom, 2493 multi_bus, sbridge_dev); 2494 if (!sbridge_dev) { 2495 /* If the HA1 wasn't found, don't create EDAC second memory controller */ 2496 if (dev_descr->dom == IMC1 && devno != 1) { 2497 edac_dbg(0, "Skip IMC1: %04x:%04x (since HA1 was absent)\n", 2498 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2499 pci_dev_put(pdev); 2500 return 0; 2501 } 2502 2503 if (dev_descr->dom == SOCK) 2504 goto out_imc; 2505 2506 sbridge_dev = alloc_sbridge_dev(seg, bus, dev_descr->dom, table); 2507 if (!sbridge_dev) { 2508 pci_dev_put(pdev); 2509 return -ENOMEM; 2510 } 2511 (*num_mc)++; 2512 } 2513 2514 if (sbridge_dev->pdev[sbridge_dev->i_devs]) { 2515 sbridge_printk(KERN_ERR, 2516 "Duplicated device for %04x:%04x\n", 2517 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2518 pci_dev_put(pdev); 2519 return -ENODEV; 2520 } 2521 2522 sbridge_dev->pdev[sbridge_dev->i_devs++] = pdev; 2523 2524 /* pdev belongs to more than one IMC, do extra gets */ 2525 if (++i > 1) 2526 pci_dev_get(pdev); 2527 2528 if (dev_descr->dom == SOCK && i < table->n_imcs_per_sock) 2529 goto next_imc; 2530 2531 out_imc: 2532 /* Be sure that the device is enabled */ 2533 if (unlikely(pci_enable_device(pdev) < 0)) { 2534 sbridge_printk(KERN_ERR, 2535 "Couldn't enable %04x:%04x\n", 2536 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2537 return -ENODEV; 2538 } 2539 2540 edac_dbg(0, "Detected %04x:%04x\n", 2541 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2542 2543 /* 2544 * As stated on drivers/pci/search.c, the reference count for 2545 * @from is always decremented if it is not %NULL. So, as we need 2546 * to get all devices up to null, we need to do a get for the device 2547 */ 2548 pci_dev_get(pdev); 2549 2550 *prev = pdev; 2551 2552 return 0; 2553 } 2554 2555 /* 2556 * sbridge_get_all_devices - Find and perform 'get' operation on the MCH's 2557 * devices we want to reference for this driver. 2558 * @num_mc: pointer to the memory controllers count, to be incremented in case 2559 * of success. 2560 * @table: model specific table 2561 * 2562 * returns 0 in case of success or error code 2563 */ 2564 static int sbridge_get_all_devices(u8 *num_mc, 2565 const struct pci_id_table *table) 2566 { 2567 int i, rc; 2568 struct pci_dev *pdev = NULL; 2569 int allow_dups = 0; 2570 int multi_bus = 0; 2571 2572 if (table->type == KNIGHTS_LANDING) 2573 allow_dups = multi_bus = 1; 2574 while (table && table->descr) { 2575 for (i = 0; i < table->n_devs_per_sock; i++) { 2576 if (!allow_dups || i == 0 || 2577 table->descr[i].dev_id != 2578 table->descr[i-1].dev_id) { 2579 pdev = NULL; 2580 } 2581 do { 2582 rc = sbridge_get_onedevice(&pdev, num_mc, 2583 table, i, multi_bus); 2584 if (rc < 0) { 2585 if (i == 0) { 2586 i = table->n_devs_per_sock; 2587 break; 2588 } 2589 sbridge_put_all_devices(); 2590 return -ENODEV; 2591 } 2592 } while (pdev && !allow_dups); 2593 } 2594 table++; 2595 } 2596 2597 return 0; 2598 } 2599 2600 /* 2601 * Device IDs for {SBRIDGE,IBRIDGE,HASWELL,BROADWELL}_IMC_HA0_TAD0 are in 2602 * the format: XXXa. So we can convert from a device to the corresponding 2603 * channel like this 2604 */ 2605 #define TAD_DEV_TO_CHAN(dev) (((dev) & 0xf) - 0xa) 2606 2607 static int sbridge_mci_bind_devs(struct mem_ctl_info *mci, 2608 struct sbridge_dev *sbridge_dev) 2609 { 2610 struct sbridge_pvt *pvt = mci->pvt_info; 2611 struct pci_dev *pdev; 2612 u8 saw_chan_mask = 0; 2613 int i; 2614 2615 for (i = 0; i < sbridge_dev->n_devs; i++) { 2616 pdev = sbridge_dev->pdev[i]; 2617 if (!pdev) 2618 continue; 2619 2620 switch (pdev->device) { 2621 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0: 2622 pvt->pci_sad0 = pdev; 2623 break; 2624 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1: 2625 pvt->pci_sad1 = pdev; 2626 break; 2627 case PCI_DEVICE_ID_INTEL_SBRIDGE_BR: 2628 pvt->pci_br0 = pdev; 2629 break; 2630 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0: 2631 pvt->pci_ha = pdev; 2632 break; 2633 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA: 2634 pvt->pci_ta = pdev; 2635 break; 2636 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS: 2637 pvt->pci_ras = pdev; 2638 break; 2639 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0: 2640 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1: 2641 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2: 2642 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3: 2643 { 2644 int id = TAD_DEV_TO_CHAN(pdev->device); 2645 pvt->pci_tad[id] = pdev; 2646 saw_chan_mask |= 1 << id; 2647 } 2648 break; 2649 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO: 2650 pvt->pci_ddrio = pdev; 2651 break; 2652 default: 2653 goto error; 2654 } 2655 2656 edac_dbg(0, "Associated PCI %02x:%02x, bus %d with dev = %p\n", 2657 pdev->vendor, pdev->device, 2658 sbridge_dev->bus, 2659 pdev); 2660 } 2661 2662 /* Check if everything were registered */ 2663 if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha || 2664 !pvt->pci_ras || !pvt->pci_ta) 2665 goto enodev; 2666 2667 if (saw_chan_mask != 0x0f) 2668 goto enodev; 2669 return 0; 2670 2671 enodev: 2672 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2673 return -ENODEV; 2674 2675 error: 2676 sbridge_printk(KERN_ERR, "Unexpected device %02x:%02x\n", 2677 PCI_VENDOR_ID_INTEL, pdev->device); 2678 return -EINVAL; 2679 } 2680 2681 static int ibridge_mci_bind_devs(struct mem_ctl_info *mci, 2682 struct sbridge_dev *sbridge_dev) 2683 { 2684 struct sbridge_pvt *pvt = mci->pvt_info; 2685 struct pci_dev *pdev; 2686 u8 saw_chan_mask = 0; 2687 int i; 2688 2689 for (i = 0; i < sbridge_dev->n_devs; i++) { 2690 pdev = sbridge_dev->pdev[i]; 2691 if (!pdev) 2692 continue; 2693 2694 switch (pdev->device) { 2695 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0: 2696 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1: 2697 pvt->pci_ha = pdev; 2698 break; 2699 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA: 2700 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA: 2701 pvt->pci_ta = pdev; 2702 break; 2703 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS: 2704 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS: 2705 pvt->pci_ras = pdev; 2706 break; 2707 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0: 2708 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1: 2709 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2: 2710 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3: 2711 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0: 2712 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1: 2713 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2: 2714 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3: 2715 { 2716 int id = TAD_DEV_TO_CHAN(pdev->device); 2717 pvt->pci_tad[id] = pdev; 2718 saw_chan_mask |= 1 << id; 2719 } 2720 break; 2721 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0: 2722 pvt->pci_ddrio = pdev; 2723 break; 2724 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0: 2725 pvt->pci_ddrio = pdev; 2726 break; 2727 case PCI_DEVICE_ID_INTEL_IBRIDGE_SAD: 2728 pvt->pci_sad0 = pdev; 2729 break; 2730 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR0: 2731 pvt->pci_br0 = pdev; 2732 break; 2733 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR1: 2734 pvt->pci_br1 = pdev; 2735 break; 2736 default: 2737 goto error; 2738 } 2739 2740 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2741 sbridge_dev->bus, 2742 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2743 pdev); 2744 } 2745 2746 /* Check if everything were registered */ 2747 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_br0 || 2748 !pvt->pci_br1 || !pvt->pci_ras || !pvt->pci_ta) 2749 goto enodev; 2750 2751 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2752 saw_chan_mask != 0x03) /* -EP */ 2753 goto enodev; 2754 return 0; 2755 2756 enodev: 2757 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2758 return -ENODEV; 2759 2760 error: 2761 sbridge_printk(KERN_ERR, 2762 "Unexpected device %02x:%02x\n", PCI_VENDOR_ID_INTEL, 2763 pdev->device); 2764 return -EINVAL; 2765 } 2766 2767 static int haswell_mci_bind_devs(struct mem_ctl_info *mci, 2768 struct sbridge_dev *sbridge_dev) 2769 { 2770 struct sbridge_pvt *pvt = mci->pvt_info; 2771 struct pci_dev *pdev; 2772 u8 saw_chan_mask = 0; 2773 int i; 2774 2775 /* there's only one device per system; not tied to any bus */ 2776 if (pvt->info.pci_vtd == NULL) 2777 /* result will be checked later */ 2778 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2779 PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC, 2780 NULL); 2781 2782 for (i = 0; i < sbridge_dev->n_devs; i++) { 2783 pdev = sbridge_dev->pdev[i]; 2784 if (!pdev) 2785 continue; 2786 2787 switch (pdev->device) { 2788 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0: 2789 pvt->pci_sad0 = pdev; 2790 break; 2791 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1: 2792 pvt->pci_sad1 = pdev; 2793 break; 2794 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0: 2795 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1: 2796 pvt->pci_ha = pdev; 2797 break; 2798 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA: 2799 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA: 2800 pvt->pci_ta = pdev; 2801 break; 2802 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM: 2803 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM: 2804 pvt->pci_ras = pdev; 2805 break; 2806 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0: 2807 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1: 2808 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2: 2809 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3: 2810 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0: 2811 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1: 2812 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2: 2813 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3: 2814 { 2815 int id = TAD_DEV_TO_CHAN(pdev->device); 2816 pvt->pci_tad[id] = pdev; 2817 saw_chan_mask |= 1 << id; 2818 } 2819 break; 2820 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0: 2821 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1: 2822 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2: 2823 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3: 2824 if (!pvt->pci_ddrio) 2825 pvt->pci_ddrio = pdev; 2826 break; 2827 default: 2828 break; 2829 } 2830 2831 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2832 sbridge_dev->bus, 2833 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2834 pdev); 2835 } 2836 2837 /* Check if everything were registered */ 2838 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2839 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2840 goto enodev; 2841 2842 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2843 saw_chan_mask != 0x03) /* -EP */ 2844 goto enodev; 2845 return 0; 2846 2847 enodev: 2848 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2849 return -ENODEV; 2850 } 2851 2852 static int broadwell_mci_bind_devs(struct mem_ctl_info *mci, 2853 struct sbridge_dev *sbridge_dev) 2854 { 2855 struct sbridge_pvt *pvt = mci->pvt_info; 2856 struct pci_dev *pdev; 2857 u8 saw_chan_mask = 0; 2858 int i; 2859 2860 /* there's only one device per system; not tied to any bus */ 2861 if (pvt->info.pci_vtd == NULL) 2862 /* result will be checked later */ 2863 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2864 PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC, 2865 NULL); 2866 2867 for (i = 0; i < sbridge_dev->n_devs; i++) { 2868 pdev = sbridge_dev->pdev[i]; 2869 if (!pdev) 2870 continue; 2871 2872 switch (pdev->device) { 2873 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0: 2874 pvt->pci_sad0 = pdev; 2875 break; 2876 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1: 2877 pvt->pci_sad1 = pdev; 2878 break; 2879 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0: 2880 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1: 2881 pvt->pci_ha = pdev; 2882 break; 2883 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA: 2884 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA: 2885 pvt->pci_ta = pdev; 2886 break; 2887 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM: 2888 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM: 2889 pvt->pci_ras = pdev; 2890 break; 2891 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0: 2892 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1: 2893 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2: 2894 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3: 2895 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0: 2896 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1: 2897 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2: 2898 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3: 2899 { 2900 int id = TAD_DEV_TO_CHAN(pdev->device); 2901 pvt->pci_tad[id] = pdev; 2902 saw_chan_mask |= 1 << id; 2903 } 2904 break; 2905 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0: 2906 pvt->pci_ddrio = pdev; 2907 break; 2908 default: 2909 break; 2910 } 2911 2912 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2913 sbridge_dev->bus, 2914 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2915 pdev); 2916 } 2917 2918 /* Check if everything were registered */ 2919 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2920 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2921 goto enodev; 2922 2923 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2924 saw_chan_mask != 0x03) /* -EP */ 2925 goto enodev; 2926 return 0; 2927 2928 enodev: 2929 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2930 return -ENODEV; 2931 } 2932 2933 static int knl_mci_bind_devs(struct mem_ctl_info *mci, 2934 struct sbridge_dev *sbridge_dev) 2935 { 2936 struct sbridge_pvt *pvt = mci->pvt_info; 2937 struct pci_dev *pdev; 2938 int dev, func; 2939 2940 int i; 2941 int devidx; 2942 2943 for (i = 0; i < sbridge_dev->n_devs; i++) { 2944 pdev = sbridge_dev->pdev[i]; 2945 if (!pdev) 2946 continue; 2947 2948 /* Extract PCI device and function. */ 2949 dev = (pdev->devfn >> 3) & 0x1f; 2950 func = pdev->devfn & 0x7; 2951 2952 switch (pdev->device) { 2953 case PCI_DEVICE_ID_INTEL_KNL_IMC_MC: 2954 if (dev == 8) 2955 pvt->knl.pci_mc0 = pdev; 2956 else if (dev == 9) 2957 pvt->knl.pci_mc1 = pdev; 2958 else { 2959 sbridge_printk(KERN_ERR, 2960 "Memory controller in unexpected place! (dev %d, fn %d)\n", 2961 dev, func); 2962 continue; 2963 } 2964 break; 2965 2966 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0: 2967 pvt->pci_sad0 = pdev; 2968 break; 2969 2970 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1: 2971 pvt->pci_sad1 = pdev; 2972 break; 2973 2974 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHA: 2975 /* There are one of these per tile, and range from 2976 * 1.14.0 to 1.18.5. 2977 */ 2978 devidx = ((dev-14)*8)+func; 2979 2980 if (devidx < 0 || devidx >= KNL_MAX_CHAS) { 2981 sbridge_printk(KERN_ERR, 2982 "Caching and Home Agent in unexpected place! (dev %d, fn %d)\n", 2983 dev, func); 2984 continue; 2985 } 2986 2987 WARN_ON(pvt->knl.pci_cha[devidx] != NULL); 2988 2989 pvt->knl.pci_cha[devidx] = pdev; 2990 break; 2991 2992 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN: 2993 devidx = -1; 2994 2995 /* 2996 * MC0 channels 0-2 are device 9 function 2-4, 2997 * MC1 channels 3-5 are device 8 function 2-4. 2998 */ 2999 3000 if (dev == 9) 3001 devidx = func-2; 3002 else if (dev == 8) 3003 devidx = 3 + (func-2); 3004 3005 if (devidx < 0 || devidx >= KNL_MAX_CHANNELS) { 3006 sbridge_printk(KERN_ERR, 3007 "DRAM Channel Registers in unexpected place! (dev %d, fn %d)\n", 3008 dev, func); 3009 continue; 3010 } 3011 3012 WARN_ON(pvt->knl.pci_channel[devidx] != NULL); 3013 pvt->knl.pci_channel[devidx] = pdev; 3014 break; 3015 3016 case PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM: 3017 pvt->knl.pci_mc_info = pdev; 3018 break; 3019 3020 case PCI_DEVICE_ID_INTEL_KNL_IMC_TA: 3021 pvt->pci_ta = pdev; 3022 break; 3023 3024 default: 3025 sbridge_printk(KERN_ERR, "Unexpected device %d\n", 3026 pdev->device); 3027 break; 3028 } 3029 } 3030 3031 if (!pvt->knl.pci_mc0 || !pvt->knl.pci_mc1 || 3032 !pvt->pci_sad0 || !pvt->pci_sad1 || 3033 !pvt->pci_ta) { 3034 goto enodev; 3035 } 3036 3037 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 3038 if (!pvt->knl.pci_channel[i]) { 3039 sbridge_printk(KERN_ERR, "Missing channel %d\n", i); 3040 goto enodev; 3041 } 3042 } 3043 3044 for (i = 0; i < KNL_MAX_CHAS; i++) { 3045 if (!pvt->knl.pci_cha[i]) { 3046 sbridge_printk(KERN_ERR, "Missing CHA %d\n", i); 3047 goto enodev; 3048 } 3049 } 3050 3051 return 0; 3052 3053 enodev: 3054 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 3055 return -ENODEV; 3056 } 3057 3058 /**************************************************************************** 3059 Error check routines 3060 ****************************************************************************/ 3061 3062 /* 3063 * While Sandy Bridge has error count registers, SMI BIOS read values from 3064 * and resets the counters. So, they are not reliable for the OS to read 3065 * from them. So, we have no option but to just trust on whatever MCE is 3066 * telling us about the errors. 3067 */ 3068 static void sbridge_mce_output_error(struct mem_ctl_info *mci, 3069 const struct mce *m) 3070 { 3071 struct mem_ctl_info *new_mci; 3072 struct sbridge_pvt *pvt = mci->pvt_info; 3073 enum hw_event_mc_err_type tp_event; 3074 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0); 3075 bool overflow = GET_BITFIELD(m->status, 62, 62); 3076 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61); 3077 bool recoverable; 3078 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); 3079 u32 mscod = GET_BITFIELD(m->status, 16, 31); 3080 u32 errcode = GET_BITFIELD(m->status, 0, 15); 3081 u32 channel = GET_BITFIELD(m->status, 0, 3); 3082 u32 optypenum = GET_BITFIELD(m->status, 4, 6); 3083 /* 3084 * Bits 5-0 of MCi_MISC give the least significant bit that is valid. 3085 * A value 6 is for cache line aligned address, a value 12 is for page 3086 * aligned address reported by patrol scrubber. 3087 */ 3088 u32 lsb = GET_BITFIELD(m->misc, 0, 5); 3089 char *optype, *area_type = "DRAM"; 3090 long channel_mask, first_channel; 3091 u8 rank = 0xff, socket, ha; 3092 int rc, dimm; 3093 3094 if (pvt->info.type != SANDY_BRIDGE) 3095 recoverable = true; 3096 else 3097 recoverable = GET_BITFIELD(m->status, 56, 56); 3098 3099 if (uncorrected_error) { 3100 core_err_cnt = 1; 3101 if (ripv) { 3102 tp_event = HW_EVENT_ERR_UNCORRECTED; 3103 } else { 3104 tp_event = HW_EVENT_ERR_FATAL; 3105 } 3106 } else { 3107 tp_event = HW_EVENT_ERR_CORRECTED; 3108 } 3109 3110 /* 3111 * According with Table 15-9 of the Intel Architecture spec vol 3A, 3112 * memory errors should fit in this mask: 3113 * 000f 0000 1mmm cccc (binary) 3114 * where: 3115 * f = Correction Report Filtering Bit. If 1, subsequent errors 3116 * won't be shown 3117 * mmm = error type 3118 * cccc = channel 3119 * If the mask doesn't match, report an error to the parsing logic 3120 */ 3121 switch (optypenum) { 3122 case 0: 3123 optype = "generic undef request error"; 3124 break; 3125 case 1: 3126 optype = "memory read error"; 3127 break; 3128 case 2: 3129 optype = "memory write error"; 3130 break; 3131 case 3: 3132 optype = "addr/cmd error"; 3133 break; 3134 case 4: 3135 optype = "memory scrubbing error"; 3136 break; 3137 default: 3138 optype = "reserved"; 3139 break; 3140 } 3141 3142 if (pvt->info.type == KNIGHTS_LANDING) { 3143 if (channel == 14) { 3144 edac_dbg(0, "%s%s err_code:%04x:%04x EDRAM bank %d\n", 3145 overflow ? " OVERFLOW" : "", 3146 (uncorrected_error && recoverable) 3147 ? " recoverable" : "", 3148 mscod, errcode, 3149 m->bank); 3150 } else { 3151 char A = *("A"); 3152 3153 /* 3154 * Reported channel is in range 0-2, so we can't map it 3155 * back to mc. To figure out mc we check machine check 3156 * bank register that reported this error. 3157 * bank15 means mc0 and bank16 means mc1. 3158 */ 3159 channel = knl_channel_remap(m->bank == 16, channel); 3160 channel_mask = 1 << channel; 3161 3162 snprintf(sb_msg, sizeof(sb_msg), 3163 "%s%s err_code:%04x:%04x channel:%d (DIMM_%c)", 3164 overflow ? " OVERFLOW" : "", 3165 (uncorrected_error && recoverable) 3166 ? " recoverable" : " ", 3167 mscod, errcode, channel, A + channel); 3168 edac_mc_handle_error(tp_event, mci, core_err_cnt, 3169 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 3170 channel, 0, -1, 3171 optype, sb_msg); 3172 } 3173 return; 3174 } else if (lsb < 12) { 3175 rc = get_memory_error_data(mci, m->addr, &socket, &ha, 3176 &channel_mask, &rank, 3177 &area_type, sb_msg); 3178 } else { 3179 rc = get_memory_error_data_from_mce(mci, m, &socket, &ha, 3180 &channel_mask, sb_msg); 3181 } 3182 3183 if (rc < 0) 3184 goto err_parsing; 3185 new_mci = get_mci_for_node_id(socket, ha); 3186 if (!new_mci) { 3187 strscpy(sb_msg, "Error: socket got corrupted!"); 3188 goto err_parsing; 3189 } 3190 mci = new_mci; 3191 pvt = mci->pvt_info; 3192 3193 first_channel = find_first_bit(&channel_mask, NUM_CHANNELS); 3194 3195 if (rank == 0xff) 3196 dimm = -1; 3197 else if (rank < 4) 3198 dimm = 0; 3199 else if (rank < 8) 3200 dimm = 1; 3201 else 3202 dimm = 2; 3203 3204 /* 3205 * FIXME: On some memory configurations (mirror, lockstep), the 3206 * Memory Controller can't point the error to a single DIMM. The 3207 * EDAC core should be handling the channel mask, in order to point 3208 * to the group of dimm's where the error may be happening. 3209 */ 3210 if (!pvt->is_lockstep && !pvt->is_cur_addr_mirrored && !pvt->is_close_pg) 3211 channel = first_channel; 3212 snprintf(sb_msg_full, sizeof(sb_msg_full), 3213 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d %s", 3214 overflow ? " OVERFLOW" : "", 3215 (uncorrected_error && recoverable) ? " recoverable" : "", 3216 area_type, 3217 mscod, errcode, 3218 socket, ha, 3219 channel_mask, 3220 rank, sb_msg); 3221 3222 edac_dbg(0, "%s\n", sb_msg_full); 3223 3224 /* FIXME: need support for channel mask */ 3225 3226 if (channel == CHANNEL_UNSPECIFIED) 3227 channel = -1; 3228 3229 /* Call the helper to output message */ 3230 edac_mc_handle_error(tp_event, mci, core_err_cnt, 3231 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 3232 channel, dimm, -1, 3233 optype, sb_msg_full); 3234 return; 3235 err_parsing: 3236 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, 3237 -1, -1, -1, 3238 sb_msg, ""); 3239 3240 } 3241 3242 /* 3243 * Check that logging is enabled and that this is the right type 3244 * of error for us to handle. 3245 */ 3246 static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, 3247 void *data) 3248 { 3249 struct mce *mce = (struct mce *)data; 3250 struct mem_ctl_info *mci; 3251 char *type; 3252 3253 if (mce->kflags & MCE_HANDLED_CEC) 3254 return NOTIFY_DONE; 3255 3256 /* 3257 * Just let mcelog handle it if the error is 3258 * outside the memory controller. A memory error 3259 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0. 3260 * bit 12 has an special meaning. 3261 */ 3262 if ((mce->status & 0xefff) >> 7 != 1) 3263 return NOTIFY_DONE; 3264 3265 /* Check ADDRV bit in STATUS */ 3266 if (!GET_BITFIELD(mce->status, 58, 58)) 3267 return NOTIFY_DONE; 3268 3269 /* Check MISCV bit in STATUS */ 3270 if (!GET_BITFIELD(mce->status, 59, 59)) 3271 return NOTIFY_DONE; 3272 3273 /* Check address type in MISC (physical address only) */ 3274 if (GET_BITFIELD(mce->misc, 6, 8) != 2) 3275 return NOTIFY_DONE; 3276 3277 mci = get_mci_for_node_id(mce->socketid, IMC0); 3278 if (!mci) 3279 return NOTIFY_DONE; 3280 3281 if (mce->mcgstatus & MCG_STATUS_MCIP) 3282 type = "Exception"; 3283 else 3284 type = "Event"; 3285 3286 sbridge_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n"); 3287 3288 sbridge_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx " 3289 "Bank %d: %016Lx\n", mce->extcpu, type, 3290 mce->mcgstatus, mce->bank, mce->status); 3291 sbridge_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc); 3292 sbridge_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr); 3293 sbridge_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc); 3294 3295 sbridge_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET " 3296 "%u APIC %x\n", mce->cpuvendor, mce->cpuid, 3297 mce->time, mce->socketid, mce->apicid); 3298 3299 sbridge_mce_output_error(mci, mce); 3300 3301 /* Advice mcelog that the error were handled */ 3302 mce->kflags |= MCE_HANDLED_EDAC; 3303 return NOTIFY_OK; 3304 } 3305 3306 static struct notifier_block sbridge_mce_dec = { 3307 .notifier_call = sbridge_mce_check_error, 3308 .priority = MCE_PRIO_EDAC, 3309 }; 3310 3311 /**************************************************************************** 3312 EDAC register/unregister logic 3313 ****************************************************************************/ 3314 3315 static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev) 3316 { 3317 struct mem_ctl_info *mci = sbridge_dev->mci; 3318 3319 if (unlikely(!mci || !mci->pvt_info)) { 3320 edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev); 3321 3322 sbridge_printk(KERN_ERR, "Couldn't find mci handler\n"); 3323 return; 3324 } 3325 3326 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3327 mci, &sbridge_dev->pdev[0]->dev); 3328 3329 /* Remove MC sysfs nodes */ 3330 edac_mc_del_mc(mci->pdev); 3331 3332 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 3333 kfree(mci->ctl_name); 3334 edac_mc_free(mci); 3335 sbridge_dev->mci = NULL; 3336 } 3337 3338 static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) 3339 { 3340 struct mem_ctl_info *mci; 3341 struct edac_mc_layer layers[2]; 3342 struct sbridge_pvt *pvt; 3343 struct pci_dev *pdev = sbridge_dev->pdev[0]; 3344 int rc; 3345 3346 /* allocate a new MC control structure */ 3347 layers[0].type = EDAC_MC_LAYER_CHANNEL; 3348 layers[0].size = type == KNIGHTS_LANDING ? 3349 KNL_MAX_CHANNELS : NUM_CHANNELS; 3350 layers[0].is_virt_csrow = false; 3351 layers[1].type = EDAC_MC_LAYER_SLOT; 3352 layers[1].size = type == KNIGHTS_LANDING ? 1 : MAX_DIMMS; 3353 layers[1].is_virt_csrow = true; 3354 mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers, 3355 sizeof(*pvt)); 3356 3357 if (unlikely(!mci)) 3358 return -ENOMEM; 3359 3360 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3361 mci, &pdev->dev); 3362 3363 pvt = mci->pvt_info; 3364 memset(pvt, 0, sizeof(*pvt)); 3365 3366 /* Associate sbridge_dev and mci for future usage */ 3367 pvt->sbridge_dev = sbridge_dev; 3368 sbridge_dev->mci = mci; 3369 3370 mci->mtype_cap = type == KNIGHTS_LANDING ? 3371 MEM_FLAG_DDR4 : MEM_FLAG_DDR3; 3372 mci->edac_ctl_cap = EDAC_FLAG_NONE; 3373 mci->edac_cap = EDAC_FLAG_NONE; 3374 mci->mod_name = EDAC_MOD_STR; 3375 mci->dev_name = pci_name(pdev); 3376 mci->ctl_page_to_phys = NULL; 3377 3378 pvt->info.type = type; 3379 switch (type) { 3380 case IVY_BRIDGE: 3381 pvt->info.rankcfgr = IB_RANK_CFG_A; 3382 pvt->info.get_tolm = ibridge_get_tolm; 3383 pvt->info.get_tohm = ibridge_get_tohm; 3384 pvt->info.dram_rule = ibridge_dram_rule; 3385 pvt->info.get_memory_type = get_memory_type; 3386 pvt->info.get_node_id = get_node_id; 3387 pvt->info.get_ha = ibridge_get_ha; 3388 pvt->info.rir_limit = rir_limit; 3389 pvt->info.sad_limit = sad_limit; 3390 pvt->info.interleave_mode = interleave_mode; 3391 pvt->info.dram_attr = dram_attr; 3392 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3393 pvt->info.interleave_list = ibridge_interleave_list; 3394 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3395 pvt->info.get_width = ibridge_get_width; 3396 3397 /* Store pci devices at mci for faster access */ 3398 rc = ibridge_mci_bind_devs(mci, sbridge_dev); 3399 if (unlikely(rc < 0)) 3400 goto fail0; 3401 get_source_id(mci); 3402 mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge SrcID#%d_Ha#%d", 3403 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3404 break; 3405 case SANDY_BRIDGE: 3406 pvt->info.rankcfgr = SB_RANK_CFG_A; 3407 pvt->info.get_tolm = sbridge_get_tolm; 3408 pvt->info.get_tohm = sbridge_get_tohm; 3409 pvt->info.dram_rule = sbridge_dram_rule; 3410 pvt->info.get_memory_type = get_memory_type; 3411 pvt->info.get_node_id = get_node_id; 3412 pvt->info.get_ha = sbridge_get_ha; 3413 pvt->info.rir_limit = rir_limit; 3414 pvt->info.sad_limit = sad_limit; 3415 pvt->info.interleave_mode = interleave_mode; 3416 pvt->info.dram_attr = dram_attr; 3417 pvt->info.max_sad = ARRAY_SIZE(sbridge_dram_rule); 3418 pvt->info.interleave_list = sbridge_interleave_list; 3419 pvt->info.interleave_pkg = sbridge_interleave_pkg; 3420 pvt->info.get_width = sbridge_get_width; 3421 3422 /* Store pci devices at mci for faster access */ 3423 rc = sbridge_mci_bind_devs(mci, sbridge_dev); 3424 if (unlikely(rc < 0)) 3425 goto fail0; 3426 get_source_id(mci); 3427 mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge SrcID#%d_Ha#%d", 3428 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3429 break; 3430 case HASWELL: 3431 /* rankcfgr isn't used */ 3432 pvt->info.get_tolm = haswell_get_tolm; 3433 pvt->info.get_tohm = haswell_get_tohm; 3434 pvt->info.dram_rule = ibridge_dram_rule; 3435 pvt->info.get_memory_type = haswell_get_memory_type; 3436 pvt->info.get_node_id = haswell_get_node_id; 3437 pvt->info.get_ha = ibridge_get_ha; 3438 pvt->info.rir_limit = haswell_rir_limit; 3439 pvt->info.sad_limit = sad_limit; 3440 pvt->info.interleave_mode = interleave_mode; 3441 pvt->info.dram_attr = dram_attr; 3442 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3443 pvt->info.interleave_list = ibridge_interleave_list; 3444 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3445 pvt->info.get_width = ibridge_get_width; 3446 3447 /* Store pci devices at mci for faster access */ 3448 rc = haswell_mci_bind_devs(mci, sbridge_dev); 3449 if (unlikely(rc < 0)) 3450 goto fail0; 3451 get_source_id(mci); 3452 mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell SrcID#%d_Ha#%d", 3453 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3454 break; 3455 case BROADWELL: 3456 /* rankcfgr isn't used */ 3457 pvt->info.get_tolm = haswell_get_tolm; 3458 pvt->info.get_tohm = haswell_get_tohm; 3459 pvt->info.dram_rule = ibridge_dram_rule; 3460 pvt->info.get_memory_type = haswell_get_memory_type; 3461 pvt->info.get_node_id = haswell_get_node_id; 3462 pvt->info.get_ha = ibridge_get_ha; 3463 pvt->info.rir_limit = haswell_rir_limit; 3464 pvt->info.sad_limit = sad_limit; 3465 pvt->info.interleave_mode = interleave_mode; 3466 pvt->info.dram_attr = dram_attr; 3467 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3468 pvt->info.interleave_list = ibridge_interleave_list; 3469 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3470 pvt->info.get_width = broadwell_get_width; 3471 3472 /* Store pci devices at mci for faster access */ 3473 rc = broadwell_mci_bind_devs(mci, sbridge_dev); 3474 if (unlikely(rc < 0)) 3475 goto fail0; 3476 get_source_id(mci); 3477 mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell SrcID#%d_Ha#%d", 3478 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3479 break; 3480 case KNIGHTS_LANDING: 3481 /* pvt->info.rankcfgr == ??? */ 3482 pvt->info.get_tolm = knl_get_tolm; 3483 pvt->info.get_tohm = knl_get_tohm; 3484 pvt->info.dram_rule = knl_dram_rule; 3485 pvt->info.get_memory_type = knl_get_memory_type; 3486 pvt->info.get_node_id = knl_get_node_id; 3487 pvt->info.get_ha = knl_get_ha; 3488 pvt->info.rir_limit = NULL; 3489 pvt->info.sad_limit = knl_sad_limit; 3490 pvt->info.interleave_mode = knl_interleave_mode; 3491 pvt->info.dram_attr = dram_attr_knl; 3492 pvt->info.max_sad = ARRAY_SIZE(knl_dram_rule); 3493 pvt->info.interleave_list = knl_interleave_list; 3494 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3495 pvt->info.get_width = knl_get_width; 3496 3497 rc = knl_mci_bind_devs(mci, sbridge_dev); 3498 if (unlikely(rc < 0)) 3499 goto fail0; 3500 get_source_id(mci); 3501 mci->ctl_name = kasprintf(GFP_KERNEL, "Knights Landing SrcID#%d_Ha#%d", 3502 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3503 break; 3504 } 3505 3506 if (!mci->ctl_name) { 3507 rc = -ENOMEM; 3508 goto fail0; 3509 } 3510 3511 /* Get dimm basic config and the memory layout */ 3512 rc = get_dimm_config(mci); 3513 if (rc < 0) { 3514 edac_dbg(0, "MC: failed to get_dimm_config()\n"); 3515 goto fail; 3516 } 3517 get_memory_layout(mci); 3518 3519 /* record ptr to the generic device */ 3520 mci->pdev = &pdev->dev; 3521 3522 /* add this new MC control structure to EDAC's list of MCs */ 3523 if (unlikely(edac_mc_add_mc(mci))) { 3524 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 3525 rc = -EINVAL; 3526 goto fail; 3527 } 3528 3529 return 0; 3530 3531 fail: 3532 kfree(mci->ctl_name); 3533 fail0: 3534 edac_mc_free(mci); 3535 sbridge_dev->mci = NULL; 3536 return rc; 3537 } 3538 3539 static const struct x86_cpu_id sbridge_cpuids[] = { 3540 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &pci_dev_descr_sbridge_table), 3541 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &pci_dev_descr_ibridge_table), 3542 X86_MATCH_VFM(INTEL_HASWELL_X, &pci_dev_descr_haswell_table), 3543 X86_MATCH_VFM(INTEL_BROADWELL_X, &pci_dev_descr_broadwell_table), 3544 X86_MATCH_VFM(INTEL_BROADWELL_D, &pci_dev_descr_broadwell_table), 3545 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &pci_dev_descr_knl_table), 3546 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &pci_dev_descr_knl_table), 3547 { } 3548 }; 3549 MODULE_DEVICE_TABLE(x86cpu, sbridge_cpuids); 3550 3551 /* 3552 * sbridge_probe Get all devices and register memory controllers 3553 * present. 3554 * return: 3555 * 0 for FOUND a device 3556 * < 0 for error code 3557 */ 3558 3559 static int sbridge_probe(const struct x86_cpu_id *id) 3560 { 3561 int rc; 3562 u8 mc, num_mc = 0; 3563 struct sbridge_dev *sbridge_dev; 3564 struct pci_id_table *ptable = (struct pci_id_table *)id->driver_data; 3565 3566 /* get the pci devices we want to reserve for our use */ 3567 rc = sbridge_get_all_devices(&num_mc, ptable); 3568 3569 if (unlikely(rc < 0)) { 3570 edac_dbg(0, "couldn't get all devices\n"); 3571 goto fail0; 3572 } 3573 3574 mc = 0; 3575 3576 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 3577 edac_dbg(0, "Registering MC#%d (%d of %d)\n", 3578 mc, mc + 1, num_mc); 3579 3580 sbridge_dev->mc = mc++; 3581 rc = sbridge_register_mci(sbridge_dev, ptable->type); 3582 if (unlikely(rc < 0)) 3583 goto fail1; 3584 } 3585 3586 sbridge_printk(KERN_INFO, "%s\n", SBRIDGE_REVISION); 3587 3588 return 0; 3589 3590 fail1: 3591 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3592 sbridge_unregister_mci(sbridge_dev); 3593 3594 sbridge_put_all_devices(); 3595 fail0: 3596 return rc; 3597 } 3598 3599 /* 3600 * sbridge_remove cleanup 3601 * 3602 */ 3603 static void sbridge_remove(void) 3604 { 3605 struct sbridge_dev *sbridge_dev; 3606 3607 edac_dbg(0, "\n"); 3608 3609 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3610 sbridge_unregister_mci(sbridge_dev); 3611 3612 /* Release PCI resources */ 3613 sbridge_put_all_devices(); 3614 } 3615 3616 /* 3617 * sbridge_init Module entry function 3618 * Try to initialize this module for its devices 3619 */ 3620 static int __init sbridge_init(void) 3621 { 3622 const struct x86_cpu_id *id; 3623 const char *owner; 3624 int rc; 3625 3626 edac_dbg(2, "\n"); 3627 3628 if (ghes_get_devices()) 3629 return -EBUSY; 3630 3631 owner = edac_get_owner(); 3632 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 3633 return -EBUSY; 3634 3635 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 3636 return -ENODEV; 3637 3638 id = x86_match_cpu(sbridge_cpuids); 3639 if (!id) 3640 return -ENODEV; 3641 3642 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 3643 opstate_init(); 3644 3645 rc = sbridge_probe(id); 3646 3647 if (rc >= 0) { 3648 mce_register_decode_chain(&sbridge_mce_dec); 3649 return 0; 3650 } 3651 3652 sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n", 3653 rc); 3654 3655 return rc; 3656 } 3657 3658 /* 3659 * sbridge_exit() Module exit function 3660 * Unregister the driver 3661 */ 3662 static void __exit sbridge_exit(void) 3663 { 3664 edac_dbg(2, "\n"); 3665 sbridge_remove(); 3666 mce_unregister_decode_chain(&sbridge_mce_dec); 3667 } 3668 3669 module_init(sbridge_init); 3670 module_exit(sbridge_exit); 3671 3672 module_param(edac_op_state, int, 0444); 3673 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 3674 3675 MODULE_LICENSE("GPL"); 3676 MODULE_AUTHOR("Mauro Carvalho Chehab"); 3677 MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)"); 3678 MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - " 3679 SBRIDGE_REVISION); 3680