1 /* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module 2 * 3 * This driver supports the memory controllers found on the Intel 4 * processor family Sandy Bridge. 5 * 6 * This file may be distributed under the terms of the 7 * GNU General Public License version 2 only. 8 * 9 * Copyright (c) 2011 by: 10 * Mauro Carvalho Chehab 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/pci.h> 16 #include <linux/pci_ids.h> 17 #include <linux/slab.h> 18 #include <linux/delay.h> 19 #include <linux/edac.h> 20 #include <linux/mmzone.h> 21 #include <linux/smp.h> 22 #include <linux/bitmap.h> 23 #include <linux/math64.h> 24 #include <linux/mod_devicetable.h> 25 #include <asm/cpu_device_id.h> 26 #include <asm/intel-family.h> 27 #include <asm/processor.h> 28 #include <asm/mce.h> 29 30 #include "edac_module.h" 31 32 /* Static vars */ 33 static LIST_HEAD(sbridge_edac_list); 34 35 /* 36 * Alter this version for the module when modifications are made 37 */ 38 #define SBRIDGE_REVISION " Ver: 1.1.2 " 39 #define EDAC_MOD_STR "sb_edac" 40 41 /* 42 * Debug macros 43 */ 44 #define sbridge_printk(level, fmt, arg...) \ 45 edac_printk(level, "sbridge", fmt, ##arg) 46 47 #define sbridge_mc_printk(mci, level, fmt, arg...) \ 48 edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg) 49 50 /* 51 * Get a bit field at register value <v>, from bit <lo> to bit <hi> 52 */ 53 #define GET_BITFIELD(v, lo, hi) \ 54 (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 55 56 /* Devices 12 Function 6, Offsets 0x80 to 0xcc */ 57 static const u32 sbridge_dram_rule[] = { 58 0x80, 0x88, 0x90, 0x98, 0xa0, 59 0xa8, 0xb0, 0xb8, 0xc0, 0xc8, 60 }; 61 62 static const u32 ibridge_dram_rule[] = { 63 0x60, 0x68, 0x70, 0x78, 0x80, 64 0x88, 0x90, 0x98, 0xa0, 0xa8, 65 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, 66 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, 67 }; 68 69 static const u32 knl_dram_rule[] = { 70 0x60, 0x68, 0x70, 0x78, 0x80, /* 0-4 */ 71 0x88, 0x90, 0x98, 0xa0, 0xa8, /* 5-9 */ 72 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, /* 10-14 */ 73 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, /* 15-19 */ 74 0x100, 0x108, 0x110, 0x118, /* 20-23 */ 75 }; 76 77 #define DRAM_RULE_ENABLE(reg) GET_BITFIELD(reg, 0, 0) 78 #define A7MODE(reg) GET_BITFIELD(reg, 26, 26) 79 80 static char *show_dram_attr(u32 attr) 81 { 82 switch (attr) { 83 case 0: 84 return "DRAM"; 85 case 1: 86 return "MMCFG"; 87 case 2: 88 return "NXM"; 89 default: 90 return "unknown"; 91 } 92 } 93 94 static const u32 sbridge_interleave_list[] = { 95 0x84, 0x8c, 0x94, 0x9c, 0xa4, 96 0xac, 0xb4, 0xbc, 0xc4, 0xcc, 97 }; 98 99 static const u32 ibridge_interleave_list[] = { 100 0x64, 0x6c, 0x74, 0x7c, 0x84, 101 0x8c, 0x94, 0x9c, 0xa4, 0xac, 102 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, 103 0xdc, 0xe4, 0xec, 0xf4, 0xfc, 104 }; 105 106 static const u32 knl_interleave_list[] = { 107 0x64, 0x6c, 0x74, 0x7c, 0x84, /* 0-4 */ 108 0x8c, 0x94, 0x9c, 0xa4, 0xac, /* 5-9 */ 109 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, /* 10-14 */ 110 0xdc, 0xe4, 0xec, 0xf4, 0xfc, /* 15-19 */ 111 0x104, 0x10c, 0x114, 0x11c, /* 20-23 */ 112 }; 113 #define MAX_INTERLEAVE \ 114 (max_t(unsigned int, ARRAY_SIZE(sbridge_interleave_list), \ 115 max_t(unsigned int, ARRAY_SIZE(ibridge_interleave_list), \ 116 ARRAY_SIZE(knl_interleave_list)))) 117 118 struct interleave_pkg { 119 unsigned char start; 120 unsigned char end; 121 }; 122 123 static const struct interleave_pkg sbridge_interleave_pkg[] = { 124 { 0, 2 }, 125 { 3, 5 }, 126 { 8, 10 }, 127 { 11, 13 }, 128 { 16, 18 }, 129 { 19, 21 }, 130 { 24, 26 }, 131 { 27, 29 }, 132 }; 133 134 static const struct interleave_pkg ibridge_interleave_pkg[] = { 135 { 0, 3 }, 136 { 4, 7 }, 137 { 8, 11 }, 138 { 12, 15 }, 139 { 16, 19 }, 140 { 20, 23 }, 141 { 24, 27 }, 142 { 28, 31 }, 143 }; 144 145 static inline int sad_pkg(const struct interleave_pkg *table, u32 reg, 146 int interleave) 147 { 148 return GET_BITFIELD(reg, table[interleave].start, 149 table[interleave].end); 150 } 151 152 /* Devices 12 Function 7 */ 153 154 #define TOLM 0x80 155 #define TOHM 0x84 156 #define HASWELL_TOLM 0xd0 157 #define HASWELL_TOHM_0 0xd4 158 #define HASWELL_TOHM_1 0xd8 159 #define KNL_TOLM 0xd0 160 #define KNL_TOHM_0 0xd4 161 #define KNL_TOHM_1 0xd8 162 163 #define GET_TOLM(reg) ((GET_BITFIELD(reg, 0, 3) << 28) | 0x3ffffff) 164 #define GET_TOHM(reg) ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff) 165 166 /* Device 13 Function 6 */ 167 168 #define SAD_TARGET 0xf0 169 170 #define SOURCE_ID(reg) GET_BITFIELD(reg, 9, 11) 171 172 #define SOURCE_ID_KNL(reg) GET_BITFIELD(reg, 12, 14) 173 174 #define SAD_CONTROL 0xf4 175 176 /* Device 14 function 0 */ 177 178 static const u32 tad_dram_rule[] = { 179 0x40, 0x44, 0x48, 0x4c, 180 0x50, 0x54, 0x58, 0x5c, 181 0x60, 0x64, 0x68, 0x6c, 182 }; 183 #define MAX_TAD ARRAY_SIZE(tad_dram_rule) 184 185 #define TAD_LIMIT(reg) ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff) 186 #define TAD_SOCK(reg) GET_BITFIELD(reg, 10, 11) 187 #define TAD_CH(reg) GET_BITFIELD(reg, 8, 9) 188 #define TAD_TGT3(reg) GET_BITFIELD(reg, 6, 7) 189 #define TAD_TGT2(reg) GET_BITFIELD(reg, 4, 5) 190 #define TAD_TGT1(reg) GET_BITFIELD(reg, 2, 3) 191 #define TAD_TGT0(reg) GET_BITFIELD(reg, 0, 1) 192 193 /* Device 15, function 0 */ 194 195 #define MCMTR 0x7c 196 #define KNL_MCMTR 0x624 197 198 #define IS_ECC_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 2, 2) 199 #define IS_LOCKSTEP_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 1, 1) 200 #define IS_CLOSE_PG(mcmtr) GET_BITFIELD(mcmtr, 0, 0) 201 202 /* Device 15, function 1 */ 203 204 #define RASENABLES 0xac 205 #define IS_MIRROR_ENABLED(reg) GET_BITFIELD(reg, 0, 0) 206 207 /* Device 15, functions 2-5 */ 208 209 static const int mtr_regs[] = { 210 0x80, 0x84, 0x88, 211 }; 212 213 static const int knl_mtr_reg = 0xb60; 214 215 #define RANK_DISABLE(mtr) GET_BITFIELD(mtr, 16, 19) 216 #define IS_DIMM_PRESENT(mtr) GET_BITFIELD(mtr, 14, 14) 217 #define RANK_CNT_BITS(mtr) GET_BITFIELD(mtr, 12, 13) 218 #define RANK_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 2, 4) 219 #define COL_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 0, 1) 220 221 static const u32 tad_ch_nilv_offset[] = { 222 0x90, 0x94, 0x98, 0x9c, 223 0xa0, 0xa4, 0xa8, 0xac, 224 0xb0, 0xb4, 0xb8, 0xbc, 225 }; 226 #define CHN_IDX_OFFSET(reg) GET_BITFIELD(reg, 28, 29) 227 #define TAD_OFFSET(reg) (GET_BITFIELD(reg, 6, 25) << 26) 228 229 static const u32 rir_way_limit[] = { 230 0x108, 0x10c, 0x110, 0x114, 0x118, 231 }; 232 #define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit) 233 234 #define IS_RIR_VALID(reg) GET_BITFIELD(reg, 31, 31) 235 #define RIR_WAY(reg) GET_BITFIELD(reg, 28, 29) 236 237 #define MAX_RIR_WAY 8 238 239 static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = { 240 { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c }, 241 { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c }, 242 { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c }, 243 { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c }, 244 { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc }, 245 }; 246 247 #define RIR_RNK_TGT(type, reg) (((type) == BROADWELL) ? \ 248 GET_BITFIELD(reg, 20, 23) : GET_BITFIELD(reg, 16, 19)) 249 250 #define RIR_OFFSET(type, reg) (((type) == HASWELL || (type) == BROADWELL) ? \ 251 GET_BITFIELD(reg, 2, 15) : GET_BITFIELD(reg, 2, 14)) 252 253 /* Device 16, functions 2-7 */ 254 255 /* 256 * FIXME: Implement the error count reads directly 257 */ 258 259 static const u32 correrrcnt[] = { 260 0x104, 0x108, 0x10c, 0x110, 261 }; 262 263 #define RANK_ODD_OV(reg) GET_BITFIELD(reg, 31, 31) 264 #define RANK_ODD_ERR_CNT(reg) GET_BITFIELD(reg, 16, 30) 265 #define RANK_EVEN_OV(reg) GET_BITFIELD(reg, 15, 15) 266 #define RANK_EVEN_ERR_CNT(reg) GET_BITFIELD(reg, 0, 14) 267 268 static const u32 correrrthrsld[] = { 269 0x11c, 0x120, 0x124, 0x128, 270 }; 271 272 #define RANK_ODD_ERR_THRSLD(reg) GET_BITFIELD(reg, 16, 30) 273 #define RANK_EVEN_ERR_THRSLD(reg) GET_BITFIELD(reg, 0, 14) 274 275 276 /* Device 17, function 0 */ 277 278 #define SB_RANK_CFG_A 0x0328 279 280 #define IB_RANK_CFG_A 0x0320 281 282 /* 283 * sbridge structs 284 */ 285 286 #define NUM_CHANNELS 6 /* Max channels per MC */ 287 #define MAX_DIMMS 3 /* Max DIMMS per channel */ 288 #define KNL_MAX_CHAS 38 /* KNL max num. of Cache Home Agents */ 289 #define KNL_MAX_CHANNELS 6 /* KNL max num. of PCI channels */ 290 #define KNL_MAX_EDCS 8 /* Embedded DRAM controllers */ 291 #define CHANNEL_UNSPECIFIED 0xf /* Intel IA32 SDM 15-14 */ 292 293 enum type { 294 SANDY_BRIDGE, 295 IVY_BRIDGE, 296 HASWELL, 297 BROADWELL, 298 KNIGHTS_LANDING, 299 }; 300 301 enum domain { 302 IMC0 = 0, 303 IMC1, 304 SOCK, 305 }; 306 307 enum mirroring_mode { 308 NON_MIRRORING, 309 ADDR_RANGE_MIRRORING, 310 FULL_MIRRORING, 311 }; 312 313 struct sbridge_pvt; 314 struct sbridge_info { 315 enum type type; 316 u32 mcmtr; 317 u32 rankcfgr; 318 u64 (*get_tolm)(struct sbridge_pvt *pvt); 319 u64 (*get_tohm)(struct sbridge_pvt *pvt); 320 u64 (*rir_limit)(u32 reg); 321 u64 (*sad_limit)(u32 reg); 322 u32 (*interleave_mode)(u32 reg); 323 u32 (*dram_attr)(u32 reg); 324 const u32 *dram_rule; 325 const u32 *interleave_list; 326 const struct interleave_pkg *interleave_pkg; 327 u8 max_sad; 328 u8 (*get_node_id)(struct sbridge_pvt *pvt); 329 enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt); 330 enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr); 331 struct pci_dev *pci_vtd; 332 }; 333 334 struct sbridge_channel { 335 u32 ranks; 336 u32 dimms; 337 }; 338 339 struct pci_id_descr { 340 int dev_id; 341 int optional; 342 enum domain dom; 343 }; 344 345 struct pci_id_table { 346 const struct pci_id_descr *descr; 347 int n_devs_per_imc; 348 int n_devs_per_sock; 349 int n_imcs_per_sock; 350 enum type type; 351 }; 352 353 struct sbridge_dev { 354 struct list_head list; 355 int seg; 356 u8 bus, mc; 357 u8 node_id, source_id; 358 struct pci_dev **pdev; 359 enum domain dom; 360 int n_devs; 361 int i_devs; 362 struct mem_ctl_info *mci; 363 }; 364 365 struct knl_pvt { 366 struct pci_dev *pci_cha[KNL_MAX_CHAS]; 367 struct pci_dev *pci_channel[KNL_MAX_CHANNELS]; 368 struct pci_dev *pci_mc0; 369 struct pci_dev *pci_mc1; 370 struct pci_dev *pci_mc0_misc; 371 struct pci_dev *pci_mc1_misc; 372 struct pci_dev *pci_mc_info; /* tolm, tohm */ 373 }; 374 375 struct sbridge_pvt { 376 /* Devices per socket */ 377 struct pci_dev *pci_ddrio; 378 struct pci_dev *pci_sad0, *pci_sad1; 379 struct pci_dev *pci_br0, *pci_br1; 380 /* Devices per memory controller */ 381 struct pci_dev *pci_ha, *pci_ta, *pci_ras; 382 struct pci_dev *pci_tad[NUM_CHANNELS]; 383 384 struct sbridge_dev *sbridge_dev; 385 386 struct sbridge_info info; 387 struct sbridge_channel channel[NUM_CHANNELS]; 388 389 /* Memory type detection */ 390 bool is_cur_addr_mirrored, is_lockstep, is_close_pg; 391 bool is_chan_hash; 392 enum mirroring_mode mirror_mode; 393 394 /* Memory description */ 395 u64 tolm, tohm; 396 struct knl_pvt knl; 397 }; 398 399 #define PCI_DESCR(device_id, opt, domain) \ 400 .dev_id = (device_id), \ 401 .optional = opt, \ 402 .dom = domain 403 404 static const struct pci_id_descr pci_dev_descr_sbridge[] = { 405 /* Processor Home Agent */ 406 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0, IMC0) }, 407 408 /* Memory controller */ 409 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0, IMC0) }, 410 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0, IMC0) }, 411 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0, IMC0) }, 412 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0, IMC0) }, 413 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0, IMC0) }, 414 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0, IMC0) }, 415 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1, SOCK) }, 416 417 /* System Address Decoder */ 418 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0, SOCK) }, 419 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0, SOCK) }, 420 421 /* Broadcast Registers */ 422 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0, SOCK) }, 423 }; 424 425 #define PCI_ID_TABLE_ENTRY(A, N, M, T) { \ 426 .descr = A, \ 427 .n_devs_per_imc = N, \ 428 .n_devs_per_sock = ARRAY_SIZE(A), \ 429 .n_imcs_per_sock = M, \ 430 .type = T \ 431 } 432 433 static const struct pci_id_table pci_dev_descr_sbridge_table[] = { 434 PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge, ARRAY_SIZE(pci_dev_descr_sbridge), 1, SANDY_BRIDGE), 435 {0,} /* 0 terminated list. */ 436 }; 437 438 /* This changes depending if 1HA or 2HA: 439 * 1HA: 440 * 0x0eb8 (17.0) is DDRIO0 441 * 2HA: 442 * 0x0ebc (17.4) is DDRIO0 443 */ 444 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0 0x0eb8 445 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0 0x0ebc 446 447 /* pci ids */ 448 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0 0x0ea0 449 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA 0x0ea8 450 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS 0x0e71 451 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0 0x0eaa 452 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1 0x0eab 453 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2 0x0eac 454 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3 0x0ead 455 #define PCI_DEVICE_ID_INTEL_IBRIDGE_SAD 0x0ec8 456 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR0 0x0ec9 457 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR1 0x0eca 458 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1 0x0e60 459 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA 0x0e68 460 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS 0x0e79 461 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 0x0e6a 462 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1 0x0e6b 463 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2 0x0e6c 464 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3 0x0e6d 465 466 static const struct pci_id_descr pci_dev_descr_ibridge[] = { 467 /* Processor Home Agent */ 468 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0, 0, IMC0) }, 469 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1, 1, IMC1) }, 470 471 /* Memory controller */ 472 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA, 0, IMC0) }, 473 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS, 0, IMC0) }, 474 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0, 0, IMC0) }, 475 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1, 0, IMC0) }, 476 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2, 0, IMC0) }, 477 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3, 0, IMC0) }, 478 479 /* Optional, mode 2HA */ 480 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA, 1, IMC1) }, 481 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS, 1, IMC1) }, 482 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0, 1, IMC1) }, 483 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1, 1, IMC1) }, 484 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2, 1, IMC1) }, 485 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3, 1, IMC1) }, 486 487 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0, 1, SOCK) }, 488 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0, 1, SOCK) }, 489 490 /* System Address Decoder */ 491 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_SAD, 0, SOCK) }, 492 493 /* Broadcast Registers */ 494 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR0, 1, SOCK) }, 495 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR1, 0, SOCK) }, 496 497 }; 498 499 static const struct pci_id_table pci_dev_descr_ibridge_table[] = { 500 PCI_ID_TABLE_ENTRY(pci_dev_descr_ibridge, 12, 2, IVY_BRIDGE), 501 {0,} /* 0 terminated list. */ 502 }; 503 504 /* Haswell support */ 505 /* EN processor: 506 * - 1 IMC 507 * - 3 DDR3 channels, 2 DPC per channel 508 * EP processor: 509 * - 1 or 2 IMC 510 * - 4 DDR4 channels, 3 DPC per channel 511 * EP 4S processor: 512 * - 2 IMC 513 * - 4 DDR4 channels, 3 DPC per channel 514 * EX processor: 515 * - 2 IMC 516 * - each IMC interfaces with a SMI 2 channel 517 * - each SMI channel interfaces with a scalable memory buffer 518 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 519 */ 520 #define HASWELL_DDRCRCLKCONTROLS 0xa10 /* Ditto on Broadwell */ 521 #define HASWELL_HASYSDEFEATURE2 0x84 522 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC 0x2f28 523 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0 0x2fa0 524 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1 0x2f60 525 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA 0x2fa8 526 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM 0x2f71 527 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA 0x2f68 528 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM 0x2f79 529 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0 0x2ffc 530 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1 0x2ffd 531 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0 0x2faa 532 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1 0x2fab 533 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2 0x2fac 534 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3 0x2fad 535 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 0x2f6a 536 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1 0x2f6b 537 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c 538 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d 539 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd 540 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf 541 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9 542 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb 543 static const struct pci_id_descr pci_dev_descr_haswell[] = { 544 /* first item must be the HA */ 545 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0, IMC0) }, 546 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1, 1, IMC1) }, 547 548 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA, 0, IMC0) }, 549 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM, 0, IMC0) }, 550 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0, 0, IMC0) }, 551 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1, 0, IMC0) }, 552 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2, 1, IMC0) }, 553 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1, IMC0) }, 554 555 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1, IMC1) }, 556 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM, 1, IMC1) }, 557 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0, 1, IMC1) }, 558 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1, 1, IMC1) }, 559 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2, 1, IMC1) }, 560 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3, 1, IMC1) }, 561 562 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0, 0, SOCK) }, 563 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1, 0, SOCK) }, 564 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1, SOCK) }, 565 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1, SOCK) }, 566 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1, SOCK) }, 567 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1, SOCK) }, 568 }; 569 570 static const struct pci_id_table pci_dev_descr_haswell_table[] = { 571 PCI_ID_TABLE_ENTRY(pci_dev_descr_haswell, 13, 2, HASWELL), 572 {0,} /* 0 terminated list. */ 573 }; 574 575 /* Knight's Landing Support */ 576 /* 577 * KNL's memory channels are swizzled between memory controllers. 578 * MC0 is mapped to CH3,4,5 and MC1 is mapped to CH0,1,2 579 */ 580 #define knl_channel_remap(mc, chan) ((mc) ? (chan) : (chan) + 3) 581 582 /* Memory controller, TAD tables, error injection - 2-8-0, 2-9-0 (2 of these) */ 583 #define PCI_DEVICE_ID_INTEL_KNL_IMC_MC 0x7840 584 /* DRAM channel stuff; bank addrs, dimmmtr, etc.. 2-8-2 - 2-9-4 (6 of these) */ 585 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN 0x7843 586 /* kdrwdbu TAD limits/offsets, MCMTR - 2-10-1, 2-11-1 (2 of these) */ 587 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TA 0x7844 588 /* CHA broadcast registers, dram rules - 1-29-0 (1 of these) */ 589 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0 0x782a 590 /* SAD target - 1-29-1 (1 of these) */ 591 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1 0x782b 592 /* Caching / Home Agent */ 593 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHA 0x782c 594 /* Device with TOLM and TOHM, 0-5-0 (1 of these) */ 595 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM 0x7810 596 597 /* 598 * KNL differs from SB, IB, and Haswell in that it has multiple 599 * instances of the same device with the same device ID, so we handle that 600 * by creating as many copies in the table as we expect to find. 601 * (Like device ID must be grouped together.) 602 */ 603 604 static const struct pci_id_descr pci_dev_descr_knl[] = { 605 [0 ... 1] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_MC, 0, IMC0)}, 606 [2 ... 7] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN, 0, IMC0) }, 607 [8] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TA, 0, IMC0) }, 608 [9] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM, 0, IMC0) }, 609 [10] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0, 0, SOCK) }, 610 [11] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1, 0, SOCK) }, 611 [12 ... 49] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHA, 0, SOCK) }, 612 }; 613 614 static const struct pci_id_table pci_dev_descr_knl_table[] = { 615 PCI_ID_TABLE_ENTRY(pci_dev_descr_knl, ARRAY_SIZE(pci_dev_descr_knl), 1, KNIGHTS_LANDING), 616 {0,} 617 }; 618 619 /* 620 * Broadwell support 621 * 622 * DE processor: 623 * - 1 IMC 624 * - 2 DDR3 channels, 2 DPC per channel 625 * EP processor: 626 * - 1 or 2 IMC 627 * - 4 DDR4 channels, 3 DPC per channel 628 * EP 4S processor: 629 * - 2 IMC 630 * - 4 DDR4 channels, 3 DPC per channel 631 * EX processor: 632 * - 2 IMC 633 * - each IMC interfaces with a SMI 2 channel 634 * - each SMI channel interfaces with a scalable memory buffer 635 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 636 */ 637 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC 0x6f28 638 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0 0x6fa0 639 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1 0x6f60 640 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA 0x6fa8 641 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM 0x6f71 642 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA 0x6f68 643 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM 0x6f79 644 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0 0x6ffc 645 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1 0x6ffd 646 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0 0x6faa 647 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1 0x6fab 648 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2 0x6fac 649 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3 0x6fad 650 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 0x6f6a 651 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1 0x6f6b 652 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2 0x6f6c 653 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3 0x6f6d 654 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0 0x6faf 655 656 static const struct pci_id_descr pci_dev_descr_broadwell[] = { 657 /* first item must be the HA */ 658 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0, 0, IMC0) }, 659 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1, 1, IMC1) }, 660 661 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA, 0, IMC0) }, 662 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM, 0, IMC0) }, 663 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0, 0, IMC0) }, 664 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1, 0, IMC0) }, 665 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2, 1, IMC0) }, 666 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3, 1, IMC0) }, 667 668 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA, 1, IMC1) }, 669 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM, 1, IMC1) }, 670 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0, 1, IMC1) }, 671 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1, 1, IMC1) }, 672 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2, 1, IMC1) }, 673 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3, 1, IMC1) }, 674 675 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0, 0, SOCK) }, 676 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1, 0, SOCK) }, 677 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0, 1, SOCK) }, 678 }; 679 680 static const struct pci_id_table pci_dev_descr_broadwell_table[] = { 681 PCI_ID_TABLE_ENTRY(pci_dev_descr_broadwell, 10, 2, BROADWELL), 682 {0,} /* 0 terminated list. */ 683 }; 684 685 686 /**************************************************************************** 687 Ancillary status routines 688 ****************************************************************************/ 689 690 static inline int numrank(enum type type, u32 mtr) 691 { 692 int ranks = (1 << RANK_CNT_BITS(mtr)); 693 int max = 4; 694 695 if (type == HASWELL || type == BROADWELL || type == KNIGHTS_LANDING) 696 max = 8; 697 698 if (ranks > max) { 699 edac_dbg(0, "Invalid number of ranks: %d (max = %i) raw value = %x (%04x)\n", 700 ranks, max, (unsigned int)RANK_CNT_BITS(mtr), mtr); 701 return -EINVAL; 702 } 703 704 return ranks; 705 } 706 707 static inline int numrow(u32 mtr) 708 { 709 int rows = (RANK_WIDTH_BITS(mtr) + 12); 710 711 if (rows < 13 || rows > 18) { 712 edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n", 713 rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr); 714 return -EINVAL; 715 } 716 717 return 1 << rows; 718 } 719 720 static inline int numcol(u32 mtr) 721 { 722 int cols = (COL_WIDTH_BITS(mtr) + 10); 723 724 if (cols > 12) { 725 edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n", 726 cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr); 727 return -EINVAL; 728 } 729 730 return 1 << cols; 731 } 732 733 static struct sbridge_dev *get_sbridge_dev(int seg, u8 bus, enum domain dom, 734 int multi_bus, 735 struct sbridge_dev *prev) 736 { 737 struct sbridge_dev *sbridge_dev; 738 739 /* 740 * If we have devices scattered across several busses that pertain 741 * to the same memory controller, we'll lump them all together. 742 */ 743 if (multi_bus) { 744 return list_first_entry_or_null(&sbridge_edac_list, 745 struct sbridge_dev, list); 746 } 747 748 sbridge_dev = list_entry(prev ? prev->list.next 749 : sbridge_edac_list.next, struct sbridge_dev, list); 750 751 list_for_each_entry_from(sbridge_dev, &sbridge_edac_list, list) { 752 if ((sbridge_dev->seg == seg) && (sbridge_dev->bus == bus) && 753 (dom == SOCK || dom == sbridge_dev->dom)) 754 return sbridge_dev; 755 } 756 757 return NULL; 758 } 759 760 static struct sbridge_dev *alloc_sbridge_dev(int seg, u8 bus, enum domain dom, 761 const struct pci_id_table *table) 762 { 763 struct sbridge_dev *sbridge_dev; 764 765 sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL); 766 if (!sbridge_dev) 767 return NULL; 768 769 sbridge_dev->pdev = kcalloc(table->n_devs_per_imc, 770 sizeof(*sbridge_dev->pdev), 771 GFP_KERNEL); 772 if (!sbridge_dev->pdev) { 773 kfree(sbridge_dev); 774 return NULL; 775 } 776 777 sbridge_dev->seg = seg; 778 sbridge_dev->bus = bus; 779 sbridge_dev->dom = dom; 780 sbridge_dev->n_devs = table->n_devs_per_imc; 781 list_add_tail(&sbridge_dev->list, &sbridge_edac_list); 782 783 return sbridge_dev; 784 } 785 786 static void free_sbridge_dev(struct sbridge_dev *sbridge_dev) 787 { 788 list_del(&sbridge_dev->list); 789 kfree(sbridge_dev->pdev); 790 kfree(sbridge_dev); 791 } 792 793 static u64 sbridge_get_tolm(struct sbridge_pvt *pvt) 794 { 795 u32 reg; 796 797 /* Address range is 32:28 */ 798 pci_read_config_dword(pvt->pci_sad1, TOLM, ®); 799 return GET_TOLM(reg); 800 } 801 802 static u64 sbridge_get_tohm(struct sbridge_pvt *pvt) 803 { 804 u32 reg; 805 806 pci_read_config_dword(pvt->pci_sad1, TOHM, ®); 807 return GET_TOHM(reg); 808 } 809 810 static u64 ibridge_get_tolm(struct sbridge_pvt *pvt) 811 { 812 u32 reg; 813 814 pci_read_config_dword(pvt->pci_br1, TOLM, ®); 815 816 return GET_TOLM(reg); 817 } 818 819 static u64 ibridge_get_tohm(struct sbridge_pvt *pvt) 820 { 821 u32 reg; 822 823 pci_read_config_dword(pvt->pci_br1, TOHM, ®); 824 825 return GET_TOHM(reg); 826 } 827 828 static u64 rir_limit(u32 reg) 829 { 830 return ((u64)GET_BITFIELD(reg, 1, 10) << 29) | 0x1fffffff; 831 } 832 833 static u64 sad_limit(u32 reg) 834 { 835 return (GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff; 836 } 837 838 static u32 interleave_mode(u32 reg) 839 { 840 return GET_BITFIELD(reg, 1, 1); 841 } 842 843 static u32 dram_attr(u32 reg) 844 { 845 return GET_BITFIELD(reg, 2, 3); 846 } 847 848 static u64 knl_sad_limit(u32 reg) 849 { 850 return (GET_BITFIELD(reg, 7, 26) << 26) | 0x3ffffff; 851 } 852 853 static u32 knl_interleave_mode(u32 reg) 854 { 855 return GET_BITFIELD(reg, 1, 2); 856 } 857 858 static const char * const knl_intlv_mode[] = { 859 "[8:6]", "[10:8]", "[14:12]", "[32:30]" 860 }; 861 862 static const char *get_intlv_mode_str(u32 reg, enum type t) 863 { 864 if (t == KNIGHTS_LANDING) 865 return knl_intlv_mode[knl_interleave_mode(reg)]; 866 else 867 return interleave_mode(reg) ? "[8:6]" : "[8:6]XOR[18:16]"; 868 } 869 870 static u32 dram_attr_knl(u32 reg) 871 { 872 return GET_BITFIELD(reg, 3, 4); 873 } 874 875 876 static enum mem_type get_memory_type(struct sbridge_pvt *pvt) 877 { 878 u32 reg; 879 enum mem_type mtype; 880 881 if (pvt->pci_ddrio) { 882 pci_read_config_dword(pvt->pci_ddrio, pvt->info.rankcfgr, 883 ®); 884 if (GET_BITFIELD(reg, 11, 11)) 885 /* FIXME: Can also be LRDIMM */ 886 mtype = MEM_RDDR3; 887 else 888 mtype = MEM_DDR3; 889 } else 890 mtype = MEM_UNKNOWN; 891 892 return mtype; 893 } 894 895 static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt) 896 { 897 u32 reg; 898 bool registered = false; 899 enum mem_type mtype = MEM_UNKNOWN; 900 901 if (!pvt->pci_ddrio) 902 goto out; 903 904 pci_read_config_dword(pvt->pci_ddrio, 905 HASWELL_DDRCRCLKCONTROLS, ®); 906 /* Is_Rdimm */ 907 if (GET_BITFIELD(reg, 16, 16)) 908 registered = true; 909 910 pci_read_config_dword(pvt->pci_ta, MCMTR, ®); 911 if (GET_BITFIELD(reg, 14, 14)) { 912 if (registered) 913 mtype = MEM_RDDR4; 914 else 915 mtype = MEM_DDR4; 916 } else { 917 if (registered) 918 mtype = MEM_RDDR3; 919 else 920 mtype = MEM_DDR3; 921 } 922 923 out: 924 return mtype; 925 } 926 927 static enum dev_type knl_get_width(struct sbridge_pvt *pvt, u32 mtr) 928 { 929 /* for KNL value is fixed */ 930 return DEV_X16; 931 } 932 933 static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 934 { 935 /* there's no way to figure out */ 936 return DEV_UNKNOWN; 937 } 938 939 static enum dev_type __ibridge_get_width(u32 mtr) 940 { 941 enum dev_type type; 942 943 switch (mtr) { 944 case 3: 945 type = DEV_UNKNOWN; 946 break; 947 case 2: 948 type = DEV_X16; 949 break; 950 case 1: 951 type = DEV_X8; 952 break; 953 case 0: 954 type = DEV_X4; 955 break; 956 } 957 958 return type; 959 } 960 961 static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 962 { 963 /* 964 * ddr3_width on the documentation but also valid for DDR4 on 965 * Haswell 966 */ 967 return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8)); 968 } 969 970 static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr) 971 { 972 /* ddr3_width on the documentation but also valid for DDR4 */ 973 return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9)); 974 } 975 976 static enum mem_type knl_get_memory_type(struct sbridge_pvt *pvt) 977 { 978 /* DDR4 RDIMMS and LRDIMMS are supported */ 979 return MEM_RDDR4; 980 } 981 982 static u8 get_node_id(struct sbridge_pvt *pvt) 983 { 984 u32 reg; 985 pci_read_config_dword(pvt->pci_br0, SAD_CONTROL, ®); 986 return GET_BITFIELD(reg, 0, 2); 987 } 988 989 static u8 haswell_get_node_id(struct sbridge_pvt *pvt) 990 { 991 u32 reg; 992 993 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 994 return GET_BITFIELD(reg, 0, 3); 995 } 996 997 static u8 knl_get_node_id(struct sbridge_pvt *pvt) 998 { 999 u32 reg; 1000 1001 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 1002 return GET_BITFIELD(reg, 0, 2); 1003 } 1004 1005 1006 static u64 haswell_get_tolm(struct sbridge_pvt *pvt) 1007 { 1008 u32 reg; 1009 1010 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOLM, ®); 1011 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 1012 } 1013 1014 static u64 haswell_get_tohm(struct sbridge_pvt *pvt) 1015 { 1016 u64 rc; 1017 u32 reg; 1018 1019 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_0, ®); 1020 rc = GET_BITFIELD(reg, 26, 31); 1021 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, ®); 1022 rc = ((reg << 6) | rc) << 26; 1023 1024 return rc | 0x1ffffff; 1025 } 1026 1027 static u64 knl_get_tolm(struct sbridge_pvt *pvt) 1028 { 1029 u32 reg; 1030 1031 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOLM, ®); 1032 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 1033 } 1034 1035 static u64 knl_get_tohm(struct sbridge_pvt *pvt) 1036 { 1037 u64 rc; 1038 u32 reg_lo, reg_hi; 1039 1040 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_0, ®_lo); 1041 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_1, ®_hi); 1042 rc = ((u64)reg_hi << 32) | reg_lo; 1043 return rc | 0x3ffffff; 1044 } 1045 1046 1047 static u64 haswell_rir_limit(u32 reg) 1048 { 1049 return (((u64)GET_BITFIELD(reg, 1, 11) + 1) << 29) - 1; 1050 } 1051 1052 static inline u8 sad_pkg_socket(u8 pkg) 1053 { 1054 /* on Ivy Bridge, nodeID is SASS, where A is HA and S is node id */ 1055 return ((pkg >> 3) << 2) | (pkg & 0x3); 1056 } 1057 1058 static inline u8 sad_pkg_ha(u8 pkg) 1059 { 1060 return (pkg >> 2) & 0x1; 1061 } 1062 1063 static int haswell_chan_hash(int idx, u64 addr) 1064 { 1065 int i; 1066 1067 /* 1068 * XOR even bits from 12:26 to bit0 of idx, 1069 * odd bits from 13:27 to bit1 1070 */ 1071 for (i = 12; i < 28; i += 2) 1072 idx ^= (addr >> i) & 3; 1073 1074 return idx; 1075 } 1076 1077 /* Low bits of TAD limit, and some metadata. */ 1078 static const u32 knl_tad_dram_limit_lo[] = { 1079 0x400, 0x500, 0x600, 0x700, 1080 0x800, 0x900, 0xa00, 0xb00, 1081 }; 1082 1083 /* Low bits of TAD offset. */ 1084 static const u32 knl_tad_dram_offset_lo[] = { 1085 0x404, 0x504, 0x604, 0x704, 1086 0x804, 0x904, 0xa04, 0xb04, 1087 }; 1088 1089 /* High 16 bits of TAD limit and offset. */ 1090 static const u32 knl_tad_dram_hi[] = { 1091 0x408, 0x508, 0x608, 0x708, 1092 0x808, 0x908, 0xa08, 0xb08, 1093 }; 1094 1095 /* Number of ways a tad entry is interleaved. */ 1096 static const u32 knl_tad_ways[] = { 1097 8, 6, 4, 3, 2, 1, 1098 }; 1099 1100 /* 1101 * Retrieve the n'th Target Address Decode table entry 1102 * from the memory controller's TAD table. 1103 * 1104 * @pvt: driver private data 1105 * @entry: which entry you want to retrieve 1106 * @mc: which memory controller (0 or 1) 1107 * @offset: output tad range offset 1108 * @limit: output address of first byte above tad range 1109 * @ways: output number of interleave ways 1110 * 1111 * The offset value has curious semantics. It's a sort of running total 1112 * of the sizes of all the memory regions that aren't mapped in this 1113 * tad table. 1114 */ 1115 static int knl_get_tad(const struct sbridge_pvt *pvt, 1116 const int entry, 1117 const int mc, 1118 u64 *offset, 1119 u64 *limit, 1120 int *ways) 1121 { 1122 u32 reg_limit_lo, reg_offset_lo, reg_hi; 1123 struct pci_dev *pci_mc; 1124 int way_id; 1125 1126 switch (mc) { 1127 case 0: 1128 pci_mc = pvt->knl.pci_mc0; 1129 break; 1130 case 1: 1131 pci_mc = pvt->knl.pci_mc1; 1132 break; 1133 default: 1134 WARN_ON(1); 1135 return -EINVAL; 1136 } 1137 1138 pci_read_config_dword(pci_mc, 1139 knl_tad_dram_limit_lo[entry], ®_limit_lo); 1140 pci_read_config_dword(pci_mc, 1141 knl_tad_dram_offset_lo[entry], ®_offset_lo); 1142 pci_read_config_dword(pci_mc, 1143 knl_tad_dram_hi[entry], ®_hi); 1144 1145 /* Is this TAD entry enabled? */ 1146 if (!GET_BITFIELD(reg_limit_lo, 0, 0)) 1147 return -ENODEV; 1148 1149 way_id = GET_BITFIELD(reg_limit_lo, 3, 5); 1150 1151 if (way_id < ARRAY_SIZE(knl_tad_ways)) { 1152 *ways = knl_tad_ways[way_id]; 1153 } else { 1154 *ways = 0; 1155 sbridge_printk(KERN_ERR, 1156 "Unexpected value %d in mc_tad_limit_lo wayness field\n", 1157 way_id); 1158 return -ENODEV; 1159 } 1160 1161 /* 1162 * The least significant 6 bits of base and limit are truncated. 1163 * For limit, we fill the missing bits with 1s. 1164 */ 1165 *offset = ((u64) GET_BITFIELD(reg_offset_lo, 6, 31) << 6) | 1166 ((u64) GET_BITFIELD(reg_hi, 0, 15) << 32); 1167 *limit = ((u64) GET_BITFIELD(reg_limit_lo, 6, 31) << 6) | 63 | 1168 ((u64) GET_BITFIELD(reg_hi, 16, 31) << 32); 1169 1170 return 0; 1171 } 1172 1173 /* Determine which memory controller is responsible for a given channel. */ 1174 static int knl_channel_mc(int channel) 1175 { 1176 WARN_ON(channel < 0 || channel >= 6); 1177 1178 return channel < 3 ? 1 : 0; 1179 } 1180 1181 /* 1182 * Get the Nth entry from EDC_ROUTE_TABLE register. 1183 * (This is the per-tile mapping of logical interleave targets to 1184 * physical EDC modules.) 1185 * 1186 * entry 0: 0:2 1187 * 1: 3:5 1188 * 2: 6:8 1189 * 3: 9:11 1190 * 4: 12:14 1191 * 5: 15:17 1192 * 6: 18:20 1193 * 7: 21:23 1194 * reserved: 24:31 1195 */ 1196 static u32 knl_get_edc_route(int entry, u32 reg) 1197 { 1198 WARN_ON(entry >= KNL_MAX_EDCS); 1199 return GET_BITFIELD(reg, entry*3, (entry*3)+2); 1200 } 1201 1202 /* 1203 * Get the Nth entry from MC_ROUTE_TABLE register. 1204 * (This is the per-tile mapping of logical interleave targets to 1205 * physical DRAM channels modules.) 1206 * 1207 * entry 0: mc 0:2 channel 18:19 1208 * 1: mc 3:5 channel 20:21 1209 * 2: mc 6:8 channel 22:23 1210 * 3: mc 9:11 channel 24:25 1211 * 4: mc 12:14 channel 26:27 1212 * 5: mc 15:17 channel 28:29 1213 * reserved: 30:31 1214 * 1215 * Though we have 3 bits to identify the MC, we should only see 1216 * the values 0 or 1. 1217 */ 1218 1219 static u32 knl_get_mc_route(int entry, u32 reg) 1220 { 1221 int mc, chan; 1222 1223 WARN_ON(entry >= KNL_MAX_CHANNELS); 1224 1225 mc = GET_BITFIELD(reg, entry*3, (entry*3)+2); 1226 chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1); 1227 1228 return knl_channel_remap(mc, chan); 1229 } 1230 1231 /* 1232 * Render the EDC_ROUTE register in human-readable form. 1233 * Output string s should be at least KNL_MAX_EDCS*2 bytes. 1234 */ 1235 static void knl_show_edc_route(u32 reg, char *s) 1236 { 1237 int i; 1238 1239 for (i = 0; i < KNL_MAX_EDCS; i++) { 1240 s[i*2] = knl_get_edc_route(i, reg) + '0'; 1241 s[i*2+1] = '-'; 1242 } 1243 1244 s[KNL_MAX_EDCS*2 - 1] = '\0'; 1245 } 1246 1247 /* 1248 * Render the MC_ROUTE register in human-readable form. 1249 * Output string s should be at least KNL_MAX_CHANNELS*2 bytes. 1250 */ 1251 static void knl_show_mc_route(u32 reg, char *s) 1252 { 1253 int i; 1254 1255 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 1256 s[i*2] = knl_get_mc_route(i, reg) + '0'; 1257 s[i*2+1] = '-'; 1258 } 1259 1260 s[KNL_MAX_CHANNELS*2 - 1] = '\0'; 1261 } 1262 1263 #define KNL_EDC_ROUTE 0xb8 1264 #define KNL_MC_ROUTE 0xb4 1265 1266 /* Is this dram rule backed by regular DRAM in flat mode? */ 1267 #define KNL_EDRAM(reg) GET_BITFIELD(reg, 29, 29) 1268 1269 /* Is this dram rule cached? */ 1270 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1271 1272 /* Is this rule backed by edc ? */ 1273 #define KNL_EDRAM_ONLY(reg) GET_BITFIELD(reg, 29, 29) 1274 1275 /* Is this rule backed by DRAM, cacheable in EDRAM? */ 1276 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1277 1278 /* Is this rule mod3? */ 1279 #define KNL_MOD3(reg) GET_BITFIELD(reg, 27, 27) 1280 1281 /* 1282 * Figure out how big our RAM modules are. 1283 * 1284 * The DIMMMTR register in KNL doesn't tell us the size of the DIMMs, so we 1285 * have to figure this out from the SAD rules, interleave lists, route tables, 1286 * and TAD rules. 1287 * 1288 * SAD rules can have holes in them (e.g. the 3G-4G hole), so we have to 1289 * inspect the TAD rules to figure out how large the SAD regions really are. 1290 * 1291 * When we know the real size of a SAD region and how many ways it's 1292 * interleaved, we know the individual contribution of each channel to 1293 * TAD is size/ways. 1294 * 1295 * Finally, we have to check whether each channel participates in each SAD 1296 * region. 1297 * 1298 * Fortunately, KNL only supports one DIMM per channel, so once we know how 1299 * much memory the channel uses, we know the DIMM is at least that large. 1300 * (The BIOS might possibly choose not to map all available memory, in which 1301 * case we will underreport the size of the DIMM.) 1302 * 1303 * In theory, we could try to determine the EDC sizes as well, but that would 1304 * only work in flat mode, not in cache mode. 1305 * 1306 * @mc_sizes: Output sizes of channels (must have space for KNL_MAX_CHANNELS 1307 * elements) 1308 */ 1309 static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes) 1310 { 1311 u64 sad_base, sad_size, sad_limit = 0; 1312 u64 tad_base, tad_size, tad_limit, tad_deadspace, tad_livespace; 1313 int sad_rule = 0; 1314 int tad_rule = 0; 1315 int intrlv_ways, tad_ways; 1316 u32 first_pkg, pkg; 1317 int i; 1318 u64 sad_actual_size[2]; /* sad size accounting for holes, per mc */ 1319 u32 dram_rule, interleave_reg; 1320 u32 mc_route_reg[KNL_MAX_CHAS]; 1321 u32 edc_route_reg[KNL_MAX_CHAS]; 1322 int edram_only; 1323 char edc_route_string[KNL_MAX_EDCS*2]; 1324 char mc_route_string[KNL_MAX_CHANNELS*2]; 1325 int cur_reg_start; 1326 int mc; 1327 int channel; 1328 int participants[KNL_MAX_CHANNELS]; 1329 1330 for (i = 0; i < KNL_MAX_CHANNELS; i++) 1331 mc_sizes[i] = 0; 1332 1333 /* Read the EDC route table in each CHA. */ 1334 cur_reg_start = 0; 1335 for (i = 0; i < KNL_MAX_CHAS; i++) { 1336 pci_read_config_dword(pvt->knl.pci_cha[i], 1337 KNL_EDC_ROUTE, &edc_route_reg[i]); 1338 1339 if (i > 0 && edc_route_reg[i] != edc_route_reg[i-1]) { 1340 knl_show_edc_route(edc_route_reg[i-1], 1341 edc_route_string); 1342 if (cur_reg_start == i-1) 1343 edac_dbg(0, "edc route table for CHA %d: %s\n", 1344 cur_reg_start, edc_route_string); 1345 else 1346 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1347 cur_reg_start, i-1, edc_route_string); 1348 cur_reg_start = i; 1349 } 1350 } 1351 knl_show_edc_route(edc_route_reg[i-1], edc_route_string); 1352 if (cur_reg_start == i-1) 1353 edac_dbg(0, "edc route table for CHA %d: %s\n", 1354 cur_reg_start, edc_route_string); 1355 else 1356 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1357 cur_reg_start, i-1, edc_route_string); 1358 1359 /* Read the MC route table in each CHA. */ 1360 cur_reg_start = 0; 1361 for (i = 0; i < KNL_MAX_CHAS; i++) { 1362 pci_read_config_dword(pvt->knl.pci_cha[i], 1363 KNL_MC_ROUTE, &mc_route_reg[i]); 1364 1365 if (i > 0 && mc_route_reg[i] != mc_route_reg[i-1]) { 1366 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1367 if (cur_reg_start == i-1) 1368 edac_dbg(0, "mc route table for CHA %d: %s\n", 1369 cur_reg_start, mc_route_string); 1370 else 1371 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1372 cur_reg_start, i-1, mc_route_string); 1373 cur_reg_start = i; 1374 } 1375 } 1376 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1377 if (cur_reg_start == i-1) 1378 edac_dbg(0, "mc route table for CHA %d: %s\n", 1379 cur_reg_start, mc_route_string); 1380 else 1381 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1382 cur_reg_start, i-1, mc_route_string); 1383 1384 /* Process DRAM rules */ 1385 for (sad_rule = 0; sad_rule < pvt->info.max_sad; sad_rule++) { 1386 /* previous limit becomes the new base */ 1387 sad_base = sad_limit; 1388 1389 pci_read_config_dword(pvt->pci_sad0, 1390 pvt->info.dram_rule[sad_rule], &dram_rule); 1391 1392 if (!DRAM_RULE_ENABLE(dram_rule)) 1393 break; 1394 1395 edram_only = KNL_EDRAM_ONLY(dram_rule); 1396 1397 sad_limit = pvt->info.sad_limit(dram_rule)+1; 1398 sad_size = sad_limit - sad_base; 1399 1400 pci_read_config_dword(pvt->pci_sad0, 1401 pvt->info.interleave_list[sad_rule], &interleave_reg); 1402 1403 /* 1404 * Find out how many ways this dram rule is interleaved. 1405 * We stop when we see the first channel again. 1406 */ 1407 first_pkg = sad_pkg(pvt->info.interleave_pkg, 1408 interleave_reg, 0); 1409 for (intrlv_ways = 1; intrlv_ways < 8; intrlv_ways++) { 1410 pkg = sad_pkg(pvt->info.interleave_pkg, 1411 interleave_reg, intrlv_ways); 1412 1413 if ((pkg & 0x8) == 0) { 1414 /* 1415 * 0 bit means memory is non-local, 1416 * which KNL doesn't support 1417 */ 1418 edac_dbg(0, "Unexpected interleave target %d\n", 1419 pkg); 1420 return -1; 1421 } 1422 1423 if (pkg == first_pkg) 1424 break; 1425 } 1426 if (KNL_MOD3(dram_rule)) 1427 intrlv_ways *= 3; 1428 1429 edac_dbg(3, "dram rule %d (base 0x%llx, limit 0x%llx), %d way interleave%s\n", 1430 sad_rule, 1431 sad_base, 1432 sad_limit, 1433 intrlv_ways, 1434 edram_only ? ", EDRAM" : ""); 1435 1436 /* 1437 * Find out how big the SAD region really is by iterating 1438 * over TAD tables (SAD regions may contain holes). 1439 * Each memory controller might have a different TAD table, so 1440 * we have to look at both. 1441 * 1442 * Livespace is the memory that's mapped in this TAD table, 1443 * deadspace is the holes (this could be the MMIO hole, or it 1444 * could be memory that's mapped by the other TAD table but 1445 * not this one). 1446 */ 1447 for (mc = 0; mc < 2; mc++) { 1448 sad_actual_size[mc] = 0; 1449 tad_livespace = 0; 1450 for (tad_rule = 0; 1451 tad_rule < ARRAY_SIZE( 1452 knl_tad_dram_limit_lo); 1453 tad_rule++) { 1454 if (knl_get_tad(pvt, 1455 tad_rule, 1456 mc, 1457 &tad_deadspace, 1458 &tad_limit, 1459 &tad_ways)) 1460 break; 1461 1462 tad_size = (tad_limit+1) - 1463 (tad_livespace + tad_deadspace); 1464 tad_livespace += tad_size; 1465 tad_base = (tad_limit+1) - tad_size; 1466 1467 if (tad_base < sad_base) { 1468 if (tad_limit > sad_base) 1469 edac_dbg(0, "TAD region overlaps lower SAD boundary -- TAD tables may be configured incorrectly.\n"); 1470 } else if (tad_base < sad_limit) { 1471 if (tad_limit+1 > sad_limit) { 1472 edac_dbg(0, "TAD region overlaps upper SAD boundary -- TAD tables may be configured incorrectly.\n"); 1473 } else { 1474 /* TAD region is completely inside SAD region */ 1475 edac_dbg(3, "TAD region %d 0x%llx - 0x%llx (%lld bytes) table%d\n", 1476 tad_rule, tad_base, 1477 tad_limit, tad_size, 1478 mc); 1479 sad_actual_size[mc] += tad_size; 1480 } 1481 } 1482 tad_base = tad_limit+1; 1483 } 1484 } 1485 1486 for (mc = 0; mc < 2; mc++) { 1487 edac_dbg(3, " total TAD DRAM footprint in table%d : 0x%llx (%lld bytes)\n", 1488 mc, sad_actual_size[mc], sad_actual_size[mc]); 1489 } 1490 1491 /* Ignore EDRAM rule */ 1492 if (edram_only) 1493 continue; 1494 1495 /* Figure out which channels participate in interleave. */ 1496 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) 1497 participants[channel] = 0; 1498 1499 /* For each channel, does at least one CHA have 1500 * this channel mapped to the given target? 1501 */ 1502 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1503 int target; 1504 int cha; 1505 1506 for (target = 0; target < KNL_MAX_CHANNELS; target++) { 1507 for (cha = 0; cha < KNL_MAX_CHAS; cha++) { 1508 if (knl_get_mc_route(target, 1509 mc_route_reg[cha]) == channel 1510 && !participants[channel]) { 1511 participants[channel] = 1; 1512 break; 1513 } 1514 } 1515 } 1516 } 1517 1518 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1519 mc = knl_channel_mc(channel); 1520 if (participants[channel]) { 1521 edac_dbg(4, "mc channel %d contributes %lld bytes via sad entry %d\n", 1522 channel, 1523 sad_actual_size[mc]/intrlv_ways, 1524 sad_rule); 1525 mc_sizes[channel] += 1526 sad_actual_size[mc]/intrlv_ways; 1527 } 1528 } 1529 } 1530 1531 return 0; 1532 } 1533 1534 static void get_source_id(struct mem_ctl_info *mci) 1535 { 1536 struct sbridge_pvt *pvt = mci->pvt_info; 1537 u32 reg; 1538 1539 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL || 1540 pvt->info.type == KNIGHTS_LANDING) 1541 pci_read_config_dword(pvt->pci_sad1, SAD_TARGET, ®); 1542 else 1543 pci_read_config_dword(pvt->pci_br0, SAD_TARGET, ®); 1544 1545 if (pvt->info.type == KNIGHTS_LANDING) 1546 pvt->sbridge_dev->source_id = SOURCE_ID_KNL(reg); 1547 else 1548 pvt->sbridge_dev->source_id = SOURCE_ID(reg); 1549 } 1550 1551 static int __populate_dimms(struct mem_ctl_info *mci, 1552 u64 knl_mc_sizes[KNL_MAX_CHANNELS], 1553 enum edac_type mode) 1554 { 1555 struct sbridge_pvt *pvt = mci->pvt_info; 1556 int channels = pvt->info.type == KNIGHTS_LANDING ? KNL_MAX_CHANNELS 1557 : NUM_CHANNELS; 1558 unsigned int i, j, banks, ranks, rows, cols, npages; 1559 struct dimm_info *dimm; 1560 enum mem_type mtype; 1561 u64 size; 1562 1563 mtype = pvt->info.get_memory_type(pvt); 1564 if (mtype == MEM_RDDR3 || mtype == MEM_RDDR4) 1565 edac_dbg(0, "Memory is registered\n"); 1566 else if (mtype == MEM_UNKNOWN) 1567 edac_dbg(0, "Cannot determine memory type\n"); 1568 else 1569 edac_dbg(0, "Memory is unregistered\n"); 1570 1571 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4) 1572 banks = 16; 1573 else 1574 banks = 8; 1575 1576 for (i = 0; i < channels; i++) { 1577 u32 mtr; 1578 1579 int max_dimms_per_channel; 1580 1581 if (pvt->info.type == KNIGHTS_LANDING) { 1582 max_dimms_per_channel = 1; 1583 if (!pvt->knl.pci_channel[i]) 1584 continue; 1585 } else { 1586 max_dimms_per_channel = ARRAY_SIZE(mtr_regs); 1587 if (!pvt->pci_tad[i]) 1588 continue; 1589 } 1590 1591 for (j = 0; j < max_dimms_per_channel; j++) { 1592 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, i, j, 0); 1593 if (pvt->info.type == KNIGHTS_LANDING) { 1594 pci_read_config_dword(pvt->knl.pci_channel[i], 1595 knl_mtr_reg, &mtr); 1596 } else { 1597 pci_read_config_dword(pvt->pci_tad[i], 1598 mtr_regs[j], &mtr); 1599 } 1600 edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr); 1601 if (IS_DIMM_PRESENT(mtr)) { 1602 if (!IS_ECC_ENABLED(pvt->info.mcmtr)) { 1603 sbridge_printk(KERN_ERR, "CPU SrcID #%d, Ha #%d, Channel #%d has DIMMs, but ECC is disabled\n", 1604 pvt->sbridge_dev->source_id, 1605 pvt->sbridge_dev->dom, i); 1606 return -ENODEV; 1607 } 1608 pvt->channel[i].dimms++; 1609 1610 ranks = numrank(pvt->info.type, mtr); 1611 1612 if (pvt->info.type == KNIGHTS_LANDING) { 1613 /* For DDR4, this is fixed. */ 1614 cols = 1 << 10; 1615 rows = knl_mc_sizes[i] / 1616 ((u64) cols * ranks * banks * 8); 1617 } else { 1618 rows = numrow(mtr); 1619 cols = numcol(mtr); 1620 } 1621 1622 size = ((u64)rows * cols * banks * ranks) >> (20 - 3); 1623 npages = MiB_TO_PAGES(size); 1624 1625 edac_dbg(0, "mc#%d: ha %d channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n", 1626 pvt->sbridge_dev->mc, pvt->sbridge_dev->dom, i, j, 1627 size, npages, 1628 banks, ranks, rows, cols); 1629 1630 dimm->nr_pages = npages; 1631 dimm->grain = 32; 1632 dimm->dtype = pvt->info.get_width(pvt, mtr); 1633 dimm->mtype = mtype; 1634 dimm->edac_mode = mode; 1635 snprintf(dimm->label, sizeof(dimm->label), 1636 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u", 1637 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom, i, j); 1638 } 1639 } 1640 } 1641 1642 return 0; 1643 } 1644 1645 static int get_dimm_config(struct mem_ctl_info *mci) 1646 { 1647 struct sbridge_pvt *pvt = mci->pvt_info; 1648 u64 knl_mc_sizes[KNL_MAX_CHANNELS]; 1649 enum edac_type mode; 1650 u32 reg; 1651 1652 pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt); 1653 edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n", 1654 pvt->sbridge_dev->mc, 1655 pvt->sbridge_dev->node_id, 1656 pvt->sbridge_dev->source_id); 1657 1658 /* KNL doesn't support mirroring or lockstep, 1659 * and is always closed page 1660 */ 1661 if (pvt->info.type == KNIGHTS_LANDING) { 1662 mode = EDAC_S4ECD4ED; 1663 pvt->mirror_mode = NON_MIRRORING; 1664 pvt->is_cur_addr_mirrored = false; 1665 1666 if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0) 1667 return -1; 1668 if (pci_read_config_dword(pvt->pci_ta, KNL_MCMTR, &pvt->info.mcmtr)) { 1669 edac_dbg(0, "Failed to read KNL_MCMTR register\n"); 1670 return -ENODEV; 1671 } 1672 } else { 1673 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 1674 if (pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®)) { 1675 edac_dbg(0, "Failed to read HASWELL_HASYSDEFEATURE2 register\n"); 1676 return -ENODEV; 1677 } 1678 pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21); 1679 if (GET_BITFIELD(reg, 28, 28)) { 1680 pvt->mirror_mode = ADDR_RANGE_MIRRORING; 1681 edac_dbg(0, "Address range partial memory mirroring is enabled\n"); 1682 goto next; 1683 } 1684 } 1685 if (pci_read_config_dword(pvt->pci_ras, RASENABLES, ®)) { 1686 edac_dbg(0, "Failed to read RASENABLES register\n"); 1687 return -ENODEV; 1688 } 1689 if (IS_MIRROR_ENABLED(reg)) { 1690 pvt->mirror_mode = FULL_MIRRORING; 1691 edac_dbg(0, "Full memory mirroring is enabled\n"); 1692 } else { 1693 pvt->mirror_mode = NON_MIRRORING; 1694 edac_dbg(0, "Memory mirroring is disabled\n"); 1695 } 1696 1697 next: 1698 if (pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr)) { 1699 edac_dbg(0, "Failed to read MCMTR register\n"); 1700 return -ENODEV; 1701 } 1702 if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) { 1703 edac_dbg(0, "Lockstep is enabled\n"); 1704 mode = EDAC_S8ECD8ED; 1705 pvt->is_lockstep = true; 1706 } else { 1707 edac_dbg(0, "Lockstep is disabled\n"); 1708 mode = EDAC_S4ECD4ED; 1709 pvt->is_lockstep = false; 1710 } 1711 if (IS_CLOSE_PG(pvt->info.mcmtr)) { 1712 edac_dbg(0, "address map is on closed page mode\n"); 1713 pvt->is_close_pg = true; 1714 } else { 1715 edac_dbg(0, "address map is on open page mode\n"); 1716 pvt->is_close_pg = false; 1717 } 1718 } 1719 1720 return __populate_dimms(mci, knl_mc_sizes, mode); 1721 } 1722 1723 static void get_memory_layout(const struct mem_ctl_info *mci) 1724 { 1725 struct sbridge_pvt *pvt = mci->pvt_info; 1726 int i, j, k, n_sads, n_tads, sad_interl; 1727 u32 reg; 1728 u64 limit, prv = 0; 1729 u64 tmp_mb; 1730 u32 gb, mb; 1731 u32 rir_way; 1732 1733 /* 1734 * Step 1) Get TOLM/TOHM ranges 1735 */ 1736 1737 pvt->tolm = pvt->info.get_tolm(pvt); 1738 tmp_mb = (1 + pvt->tolm) >> 20; 1739 1740 gb = div_u64_rem(tmp_mb, 1024, &mb); 1741 edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n", 1742 gb, (mb*1000)/1024, (u64)pvt->tolm); 1743 1744 /* Address range is already 45:25 */ 1745 pvt->tohm = pvt->info.get_tohm(pvt); 1746 tmp_mb = (1 + pvt->tohm) >> 20; 1747 1748 gb = div_u64_rem(tmp_mb, 1024, &mb); 1749 edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n", 1750 gb, (mb*1000)/1024, (u64)pvt->tohm); 1751 1752 /* 1753 * Step 2) Get SAD range and SAD Interleave list 1754 * TAD registers contain the interleave wayness. However, it 1755 * seems simpler to just discover it indirectly, with the 1756 * algorithm bellow. 1757 */ 1758 prv = 0; 1759 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 1760 /* SAD_LIMIT Address range is 45:26 */ 1761 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 1762 ®); 1763 limit = pvt->info.sad_limit(reg); 1764 1765 if (!DRAM_RULE_ENABLE(reg)) 1766 continue; 1767 1768 if (limit <= prv) 1769 break; 1770 1771 tmp_mb = (limit + 1) >> 20; 1772 gb = div_u64_rem(tmp_mb, 1024, &mb); 1773 edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n", 1774 n_sads, 1775 show_dram_attr(pvt->info.dram_attr(reg)), 1776 gb, (mb*1000)/1024, 1777 ((u64)tmp_mb) << 20L, 1778 get_intlv_mode_str(reg, pvt->info.type), 1779 reg); 1780 prv = limit; 1781 1782 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 1783 ®); 1784 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 1785 for (j = 0; j < 8; j++) { 1786 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, j); 1787 if (j > 0 && sad_interl == pkg) 1788 break; 1789 1790 edac_dbg(0, "SAD#%d, interleave #%d: %d\n", 1791 n_sads, j, pkg); 1792 } 1793 } 1794 1795 if (pvt->info.type == KNIGHTS_LANDING) 1796 return; 1797 1798 /* 1799 * Step 3) Get TAD range 1800 */ 1801 prv = 0; 1802 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 1803 pci_read_config_dword(pvt->pci_ha, tad_dram_rule[n_tads], ®); 1804 limit = TAD_LIMIT(reg); 1805 if (limit <= prv) 1806 break; 1807 tmp_mb = (limit + 1) >> 20; 1808 1809 gb = div_u64_rem(tmp_mb, 1024, &mb); 1810 edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n", 1811 n_tads, gb, (mb*1000)/1024, 1812 ((u64)tmp_mb) << 20L, 1813 (u32)(1 << TAD_SOCK(reg)), 1814 (u32)TAD_CH(reg) + 1, 1815 (u32)TAD_TGT0(reg), 1816 (u32)TAD_TGT1(reg), 1817 (u32)TAD_TGT2(reg), 1818 (u32)TAD_TGT3(reg), 1819 reg); 1820 prv = limit; 1821 } 1822 1823 /* 1824 * Step 4) Get TAD offsets, per each channel 1825 */ 1826 for (i = 0; i < NUM_CHANNELS; i++) { 1827 if (!pvt->channel[i].dimms) 1828 continue; 1829 for (j = 0; j < n_tads; j++) { 1830 pci_read_config_dword(pvt->pci_tad[i], 1831 tad_ch_nilv_offset[j], 1832 ®); 1833 tmp_mb = TAD_OFFSET(reg) >> 20; 1834 gb = div_u64_rem(tmp_mb, 1024, &mb); 1835 edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n", 1836 i, j, 1837 gb, (mb*1000)/1024, 1838 ((u64)tmp_mb) << 20L, 1839 reg); 1840 } 1841 } 1842 1843 /* 1844 * Step 6) Get RIR Wayness/Limit, per each channel 1845 */ 1846 for (i = 0; i < NUM_CHANNELS; i++) { 1847 if (!pvt->channel[i].dimms) 1848 continue; 1849 for (j = 0; j < MAX_RIR_RANGES; j++) { 1850 pci_read_config_dword(pvt->pci_tad[i], 1851 rir_way_limit[j], 1852 ®); 1853 1854 if (!IS_RIR_VALID(reg)) 1855 continue; 1856 1857 tmp_mb = pvt->info.rir_limit(reg) >> 20; 1858 rir_way = 1 << RIR_WAY(reg); 1859 gb = div_u64_rem(tmp_mb, 1024, &mb); 1860 edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n", 1861 i, j, 1862 gb, (mb*1000)/1024, 1863 ((u64)tmp_mb) << 20L, 1864 rir_way, 1865 reg); 1866 1867 for (k = 0; k < rir_way; k++) { 1868 pci_read_config_dword(pvt->pci_tad[i], 1869 rir_offset[j][k], 1870 ®); 1871 tmp_mb = RIR_OFFSET(pvt->info.type, reg) << 6; 1872 1873 gb = div_u64_rem(tmp_mb, 1024, &mb); 1874 edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n", 1875 i, j, k, 1876 gb, (mb*1000)/1024, 1877 ((u64)tmp_mb) << 20L, 1878 (u32)RIR_RNK_TGT(pvt->info.type, reg), 1879 reg); 1880 } 1881 } 1882 } 1883 } 1884 1885 static struct mem_ctl_info *get_mci_for_node_id(u8 node_id, u8 ha) 1886 { 1887 struct sbridge_dev *sbridge_dev; 1888 1889 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 1890 if (sbridge_dev->node_id == node_id && sbridge_dev->dom == ha) 1891 return sbridge_dev->mci; 1892 } 1893 return NULL; 1894 } 1895 1896 static int get_memory_error_data(struct mem_ctl_info *mci, 1897 u64 addr, 1898 u8 *socket, u8 *ha, 1899 long *channel_mask, 1900 u8 *rank, 1901 char **area_type, char *msg) 1902 { 1903 struct mem_ctl_info *new_mci; 1904 struct sbridge_pvt *pvt = mci->pvt_info; 1905 struct pci_dev *pci_ha; 1906 int n_rir, n_sads, n_tads, sad_way, sck_xch; 1907 int sad_interl, idx, base_ch; 1908 int interleave_mode, shiftup = 0; 1909 unsigned int sad_interleave[MAX_INTERLEAVE]; 1910 u32 reg, dram_rule; 1911 u8 ch_way, sck_way, pkg, sad_ha = 0; 1912 u32 tad_offset; 1913 u32 rir_way; 1914 u32 mb, gb; 1915 u64 ch_addr, offset, limit = 0, prv = 0; 1916 1917 1918 /* 1919 * Step 0) Check if the address is at special memory ranges 1920 * The check bellow is probably enough to fill all cases where 1921 * the error is not inside a memory, except for the legacy 1922 * range (e. g. VGA addresses). It is unlikely, however, that the 1923 * memory controller would generate an error on that range. 1924 */ 1925 if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) { 1926 sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr); 1927 return -EINVAL; 1928 } 1929 if (addr >= (u64)pvt->tohm) { 1930 sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr); 1931 return -EINVAL; 1932 } 1933 1934 /* 1935 * Step 1) Get socket 1936 */ 1937 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 1938 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 1939 ®); 1940 1941 if (!DRAM_RULE_ENABLE(reg)) 1942 continue; 1943 1944 limit = pvt->info.sad_limit(reg); 1945 if (limit <= prv) { 1946 sprintf(msg, "Can't discover the memory socket"); 1947 return -EINVAL; 1948 } 1949 if (addr <= limit) 1950 break; 1951 prv = limit; 1952 } 1953 if (n_sads == pvt->info.max_sad) { 1954 sprintf(msg, "Can't discover the memory socket"); 1955 return -EINVAL; 1956 } 1957 dram_rule = reg; 1958 *area_type = show_dram_attr(pvt->info.dram_attr(dram_rule)); 1959 interleave_mode = pvt->info.interleave_mode(dram_rule); 1960 1961 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 1962 ®); 1963 1964 if (pvt->info.type == SANDY_BRIDGE) { 1965 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 1966 for (sad_way = 0; sad_way < 8; sad_way++) { 1967 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, sad_way); 1968 if (sad_way > 0 && sad_interl == pkg) 1969 break; 1970 sad_interleave[sad_way] = pkg; 1971 edac_dbg(0, "SAD interleave #%d: %d\n", 1972 sad_way, sad_interleave[sad_way]); 1973 } 1974 edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n", 1975 pvt->sbridge_dev->mc, 1976 n_sads, 1977 addr, 1978 limit, 1979 sad_way + 7, 1980 !interleave_mode ? "" : "XOR[18:16]"); 1981 if (interleave_mode) 1982 idx = ((addr >> 6) ^ (addr >> 16)) & 7; 1983 else 1984 idx = (addr >> 6) & 7; 1985 switch (sad_way) { 1986 case 1: 1987 idx = 0; 1988 break; 1989 case 2: 1990 idx = idx & 1; 1991 break; 1992 case 4: 1993 idx = idx & 3; 1994 break; 1995 case 8: 1996 break; 1997 default: 1998 sprintf(msg, "Can't discover socket interleave"); 1999 return -EINVAL; 2000 } 2001 *socket = sad_interleave[idx]; 2002 edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n", 2003 idx, sad_way, *socket); 2004 } else if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 2005 int bits, a7mode = A7MODE(dram_rule); 2006 2007 if (a7mode) { 2008 /* A7 mode swaps P9 with P6 */ 2009 bits = GET_BITFIELD(addr, 7, 8) << 1; 2010 bits |= GET_BITFIELD(addr, 9, 9); 2011 } else 2012 bits = GET_BITFIELD(addr, 6, 8); 2013 2014 if (interleave_mode == 0) { 2015 /* interleave mode will XOR {8,7,6} with {18,17,16} */ 2016 idx = GET_BITFIELD(addr, 16, 18); 2017 idx ^= bits; 2018 } else 2019 idx = bits; 2020 2021 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2022 *socket = sad_pkg_socket(pkg); 2023 sad_ha = sad_pkg_ha(pkg); 2024 2025 if (a7mode) { 2026 /* MCChanShiftUpEnable */ 2027 pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®); 2028 shiftup = GET_BITFIELD(reg, 22, 22); 2029 } 2030 2031 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %i, shiftup: %i\n", 2032 idx, *socket, sad_ha, shiftup); 2033 } else { 2034 /* Ivy Bridge's SAD mode doesn't support XOR interleave mode */ 2035 idx = (addr >> 6) & 7; 2036 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2037 *socket = sad_pkg_socket(pkg); 2038 sad_ha = sad_pkg_ha(pkg); 2039 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %d\n", 2040 idx, *socket, sad_ha); 2041 } 2042 2043 *ha = sad_ha; 2044 2045 /* 2046 * Move to the proper node structure, in order to access the 2047 * right PCI registers 2048 */ 2049 new_mci = get_mci_for_node_id(*socket, sad_ha); 2050 if (!new_mci) { 2051 sprintf(msg, "Struct for socket #%u wasn't initialized", 2052 *socket); 2053 return -EINVAL; 2054 } 2055 mci = new_mci; 2056 pvt = mci->pvt_info; 2057 2058 /* 2059 * Step 2) Get memory channel 2060 */ 2061 prv = 0; 2062 pci_ha = pvt->pci_ha; 2063 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 2064 pci_read_config_dword(pci_ha, tad_dram_rule[n_tads], ®); 2065 limit = TAD_LIMIT(reg); 2066 if (limit <= prv) { 2067 sprintf(msg, "Can't discover the memory channel"); 2068 return -EINVAL; 2069 } 2070 if (addr <= limit) 2071 break; 2072 prv = limit; 2073 } 2074 if (n_tads == MAX_TAD) { 2075 sprintf(msg, "Can't discover the memory channel"); 2076 return -EINVAL; 2077 } 2078 2079 ch_way = TAD_CH(reg) + 1; 2080 sck_way = TAD_SOCK(reg); 2081 2082 if (ch_way == 3) 2083 idx = addr >> 6; 2084 else { 2085 idx = (addr >> (6 + sck_way + shiftup)) & 0x3; 2086 if (pvt->is_chan_hash) 2087 idx = haswell_chan_hash(idx, addr); 2088 } 2089 idx = idx % ch_way; 2090 2091 /* 2092 * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ??? 2093 */ 2094 switch (idx) { 2095 case 0: 2096 base_ch = TAD_TGT0(reg); 2097 break; 2098 case 1: 2099 base_ch = TAD_TGT1(reg); 2100 break; 2101 case 2: 2102 base_ch = TAD_TGT2(reg); 2103 break; 2104 case 3: 2105 base_ch = TAD_TGT3(reg); 2106 break; 2107 default: 2108 sprintf(msg, "Can't discover the TAD target"); 2109 return -EINVAL; 2110 } 2111 *channel_mask = 1 << base_ch; 2112 2113 pci_read_config_dword(pvt->pci_tad[base_ch], tad_ch_nilv_offset[n_tads], &tad_offset); 2114 2115 if (pvt->mirror_mode == FULL_MIRRORING || 2116 (pvt->mirror_mode == ADDR_RANGE_MIRRORING && n_tads == 0)) { 2117 *channel_mask |= 1 << ((base_ch + 2) % 4); 2118 switch(ch_way) { 2119 case 2: 2120 case 4: 2121 sck_xch = (1 << sck_way) * (ch_way >> 1); 2122 break; 2123 default: 2124 sprintf(msg, "Invalid mirror set. Can't decode addr"); 2125 return -EINVAL; 2126 } 2127 2128 pvt->is_cur_addr_mirrored = true; 2129 } else { 2130 sck_xch = (1 << sck_way) * ch_way; 2131 pvt->is_cur_addr_mirrored = false; 2132 } 2133 2134 if (pvt->is_lockstep) 2135 *channel_mask |= 1 << ((base_ch + 1) % 4); 2136 2137 offset = TAD_OFFSET(tad_offset); 2138 2139 edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n", 2140 n_tads, 2141 addr, 2142 limit, 2143 sck_way, 2144 ch_way, 2145 offset, 2146 idx, 2147 base_ch, 2148 *channel_mask); 2149 2150 /* Calculate channel address */ 2151 /* Remove the TAD offset */ 2152 2153 if (offset > addr) { 2154 sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!", 2155 offset, addr); 2156 return -EINVAL; 2157 } 2158 2159 ch_addr = addr - offset; 2160 ch_addr >>= (6 + shiftup); 2161 ch_addr /= sck_xch; 2162 ch_addr <<= (6 + shiftup); 2163 ch_addr |= addr & ((1 << (6 + shiftup)) - 1); 2164 2165 /* 2166 * Step 3) Decode rank 2167 */ 2168 for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) { 2169 pci_read_config_dword(pvt->pci_tad[base_ch], rir_way_limit[n_rir], ®); 2170 2171 if (!IS_RIR_VALID(reg)) 2172 continue; 2173 2174 limit = pvt->info.rir_limit(reg); 2175 gb = div_u64_rem(limit >> 20, 1024, &mb); 2176 edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n", 2177 n_rir, 2178 gb, (mb*1000)/1024, 2179 limit, 2180 1 << RIR_WAY(reg)); 2181 if (ch_addr <= limit) 2182 break; 2183 } 2184 if (n_rir == MAX_RIR_RANGES) { 2185 sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx", 2186 ch_addr); 2187 return -EINVAL; 2188 } 2189 rir_way = RIR_WAY(reg); 2190 2191 if (pvt->is_close_pg) 2192 idx = (ch_addr >> 6); 2193 else 2194 idx = (ch_addr >> 13); /* FIXME: Datasheet says to shift by 15 */ 2195 idx %= 1 << rir_way; 2196 2197 pci_read_config_dword(pvt->pci_tad[base_ch], rir_offset[n_rir][idx], ®); 2198 *rank = RIR_RNK_TGT(pvt->info.type, reg); 2199 2200 edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n", 2201 n_rir, 2202 ch_addr, 2203 limit, 2204 rir_way, 2205 idx); 2206 2207 return 0; 2208 } 2209 2210 /**************************************************************************** 2211 Device initialization routines: put/get, init/exit 2212 ****************************************************************************/ 2213 2214 /* 2215 * sbridge_put_all_devices 'put' all the devices that we have 2216 * reserved via 'get' 2217 */ 2218 static void sbridge_put_devices(struct sbridge_dev *sbridge_dev) 2219 { 2220 int i; 2221 2222 edac_dbg(0, "\n"); 2223 for (i = 0; i < sbridge_dev->n_devs; i++) { 2224 struct pci_dev *pdev = sbridge_dev->pdev[i]; 2225 if (!pdev) 2226 continue; 2227 edac_dbg(0, "Removing dev %02x:%02x.%d\n", 2228 pdev->bus->number, 2229 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); 2230 pci_dev_put(pdev); 2231 } 2232 } 2233 2234 static void sbridge_put_all_devices(void) 2235 { 2236 struct sbridge_dev *sbridge_dev, *tmp; 2237 2238 list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) { 2239 sbridge_put_devices(sbridge_dev); 2240 free_sbridge_dev(sbridge_dev); 2241 } 2242 } 2243 2244 static int sbridge_get_onedevice(struct pci_dev **prev, 2245 u8 *num_mc, 2246 const struct pci_id_table *table, 2247 const unsigned devno, 2248 const int multi_bus) 2249 { 2250 struct sbridge_dev *sbridge_dev = NULL; 2251 const struct pci_id_descr *dev_descr = &table->descr[devno]; 2252 struct pci_dev *pdev = NULL; 2253 int seg = 0; 2254 u8 bus = 0; 2255 int i = 0; 2256 2257 sbridge_printk(KERN_DEBUG, 2258 "Seeking for: PCI ID %04x:%04x\n", 2259 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2260 2261 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 2262 dev_descr->dev_id, *prev); 2263 2264 if (!pdev) { 2265 if (*prev) { 2266 *prev = pdev; 2267 return 0; 2268 } 2269 2270 if (dev_descr->optional) 2271 return 0; 2272 2273 /* if the HA wasn't found */ 2274 if (devno == 0) 2275 return -ENODEV; 2276 2277 sbridge_printk(KERN_INFO, 2278 "Device not found: %04x:%04x\n", 2279 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2280 2281 /* End of list, leave */ 2282 return -ENODEV; 2283 } 2284 seg = pci_domain_nr(pdev->bus); 2285 bus = pdev->bus->number; 2286 2287 next_imc: 2288 sbridge_dev = get_sbridge_dev(seg, bus, dev_descr->dom, 2289 multi_bus, sbridge_dev); 2290 if (!sbridge_dev) { 2291 /* If the HA1 wasn't found, don't create EDAC second memory controller */ 2292 if (dev_descr->dom == IMC1 && devno != 1) { 2293 edac_dbg(0, "Skip IMC1: %04x:%04x (since HA1 was absent)\n", 2294 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2295 pci_dev_put(pdev); 2296 return 0; 2297 } 2298 2299 if (dev_descr->dom == SOCK) 2300 goto out_imc; 2301 2302 sbridge_dev = alloc_sbridge_dev(seg, bus, dev_descr->dom, table); 2303 if (!sbridge_dev) { 2304 pci_dev_put(pdev); 2305 return -ENOMEM; 2306 } 2307 (*num_mc)++; 2308 } 2309 2310 if (sbridge_dev->pdev[sbridge_dev->i_devs]) { 2311 sbridge_printk(KERN_ERR, 2312 "Duplicated device for %04x:%04x\n", 2313 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2314 pci_dev_put(pdev); 2315 return -ENODEV; 2316 } 2317 2318 sbridge_dev->pdev[sbridge_dev->i_devs++] = pdev; 2319 2320 /* pdev belongs to more than one IMC, do extra gets */ 2321 if (++i > 1) 2322 pci_dev_get(pdev); 2323 2324 if (dev_descr->dom == SOCK && i < table->n_imcs_per_sock) 2325 goto next_imc; 2326 2327 out_imc: 2328 /* Be sure that the device is enabled */ 2329 if (unlikely(pci_enable_device(pdev) < 0)) { 2330 sbridge_printk(KERN_ERR, 2331 "Couldn't enable %04x:%04x\n", 2332 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2333 return -ENODEV; 2334 } 2335 2336 edac_dbg(0, "Detected %04x:%04x\n", 2337 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2338 2339 /* 2340 * As stated on drivers/pci/search.c, the reference count for 2341 * @from is always decremented if it is not %NULL. So, as we need 2342 * to get all devices up to null, we need to do a get for the device 2343 */ 2344 pci_dev_get(pdev); 2345 2346 *prev = pdev; 2347 2348 return 0; 2349 } 2350 2351 /* 2352 * sbridge_get_all_devices - Find and perform 'get' operation on the MCH's 2353 * devices we want to reference for this driver. 2354 * @num_mc: pointer to the memory controllers count, to be incremented in case 2355 * of success. 2356 * @table: model specific table 2357 * 2358 * returns 0 in case of success or error code 2359 */ 2360 static int sbridge_get_all_devices(u8 *num_mc, 2361 const struct pci_id_table *table) 2362 { 2363 int i, rc; 2364 struct pci_dev *pdev = NULL; 2365 int allow_dups = 0; 2366 int multi_bus = 0; 2367 2368 if (table->type == KNIGHTS_LANDING) 2369 allow_dups = multi_bus = 1; 2370 while (table && table->descr) { 2371 for (i = 0; i < table->n_devs_per_sock; i++) { 2372 if (!allow_dups || i == 0 || 2373 table->descr[i].dev_id != 2374 table->descr[i-1].dev_id) { 2375 pdev = NULL; 2376 } 2377 do { 2378 rc = sbridge_get_onedevice(&pdev, num_mc, 2379 table, i, multi_bus); 2380 if (rc < 0) { 2381 if (i == 0) { 2382 i = table->n_devs_per_sock; 2383 break; 2384 } 2385 sbridge_put_all_devices(); 2386 return -ENODEV; 2387 } 2388 } while (pdev && !allow_dups); 2389 } 2390 table++; 2391 } 2392 2393 return 0; 2394 } 2395 2396 /* 2397 * Device IDs for {SBRIDGE,IBRIDGE,HASWELL,BROADWELL}_IMC_HA0_TAD0 are in 2398 * the format: XXXa. So we can convert from a device to the corresponding 2399 * channel like this 2400 */ 2401 #define TAD_DEV_TO_CHAN(dev) (((dev) & 0xf) - 0xa) 2402 2403 static int sbridge_mci_bind_devs(struct mem_ctl_info *mci, 2404 struct sbridge_dev *sbridge_dev) 2405 { 2406 struct sbridge_pvt *pvt = mci->pvt_info; 2407 struct pci_dev *pdev; 2408 u8 saw_chan_mask = 0; 2409 int i; 2410 2411 for (i = 0; i < sbridge_dev->n_devs; i++) { 2412 pdev = sbridge_dev->pdev[i]; 2413 if (!pdev) 2414 continue; 2415 2416 switch (pdev->device) { 2417 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0: 2418 pvt->pci_sad0 = pdev; 2419 break; 2420 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1: 2421 pvt->pci_sad1 = pdev; 2422 break; 2423 case PCI_DEVICE_ID_INTEL_SBRIDGE_BR: 2424 pvt->pci_br0 = pdev; 2425 break; 2426 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0: 2427 pvt->pci_ha = pdev; 2428 break; 2429 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA: 2430 pvt->pci_ta = pdev; 2431 break; 2432 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS: 2433 pvt->pci_ras = pdev; 2434 break; 2435 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0: 2436 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1: 2437 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2: 2438 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3: 2439 { 2440 int id = TAD_DEV_TO_CHAN(pdev->device); 2441 pvt->pci_tad[id] = pdev; 2442 saw_chan_mask |= 1 << id; 2443 } 2444 break; 2445 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO: 2446 pvt->pci_ddrio = pdev; 2447 break; 2448 default: 2449 goto error; 2450 } 2451 2452 edac_dbg(0, "Associated PCI %02x:%02x, bus %d with dev = %p\n", 2453 pdev->vendor, pdev->device, 2454 sbridge_dev->bus, 2455 pdev); 2456 } 2457 2458 /* Check if everything were registered */ 2459 if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha || 2460 !pvt->pci_ras || !pvt->pci_ta) 2461 goto enodev; 2462 2463 if (saw_chan_mask != 0x0f) 2464 goto enodev; 2465 return 0; 2466 2467 enodev: 2468 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2469 return -ENODEV; 2470 2471 error: 2472 sbridge_printk(KERN_ERR, "Unexpected device %02x:%02x\n", 2473 PCI_VENDOR_ID_INTEL, pdev->device); 2474 return -EINVAL; 2475 } 2476 2477 static int ibridge_mci_bind_devs(struct mem_ctl_info *mci, 2478 struct sbridge_dev *sbridge_dev) 2479 { 2480 struct sbridge_pvt *pvt = mci->pvt_info; 2481 struct pci_dev *pdev; 2482 u8 saw_chan_mask = 0; 2483 int i; 2484 2485 for (i = 0; i < sbridge_dev->n_devs; i++) { 2486 pdev = sbridge_dev->pdev[i]; 2487 if (!pdev) 2488 continue; 2489 2490 switch (pdev->device) { 2491 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0: 2492 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1: 2493 pvt->pci_ha = pdev; 2494 break; 2495 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA: 2496 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA: 2497 pvt->pci_ta = pdev; 2498 break; 2499 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS: 2500 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS: 2501 pvt->pci_ras = pdev; 2502 break; 2503 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0: 2504 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1: 2505 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2: 2506 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3: 2507 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0: 2508 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1: 2509 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2: 2510 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3: 2511 { 2512 int id = TAD_DEV_TO_CHAN(pdev->device); 2513 pvt->pci_tad[id] = pdev; 2514 saw_chan_mask |= 1 << id; 2515 } 2516 break; 2517 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0: 2518 pvt->pci_ddrio = pdev; 2519 break; 2520 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0: 2521 pvt->pci_ddrio = pdev; 2522 break; 2523 case PCI_DEVICE_ID_INTEL_IBRIDGE_SAD: 2524 pvt->pci_sad0 = pdev; 2525 break; 2526 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR0: 2527 pvt->pci_br0 = pdev; 2528 break; 2529 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR1: 2530 pvt->pci_br1 = pdev; 2531 break; 2532 default: 2533 goto error; 2534 } 2535 2536 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2537 sbridge_dev->bus, 2538 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2539 pdev); 2540 } 2541 2542 /* Check if everything were registered */ 2543 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_br0 || 2544 !pvt->pci_br1 || !pvt->pci_ras || !pvt->pci_ta) 2545 goto enodev; 2546 2547 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2548 saw_chan_mask != 0x03) /* -EP */ 2549 goto enodev; 2550 return 0; 2551 2552 enodev: 2553 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2554 return -ENODEV; 2555 2556 error: 2557 sbridge_printk(KERN_ERR, 2558 "Unexpected device %02x:%02x\n", PCI_VENDOR_ID_INTEL, 2559 pdev->device); 2560 return -EINVAL; 2561 } 2562 2563 static int haswell_mci_bind_devs(struct mem_ctl_info *mci, 2564 struct sbridge_dev *sbridge_dev) 2565 { 2566 struct sbridge_pvt *pvt = mci->pvt_info; 2567 struct pci_dev *pdev; 2568 u8 saw_chan_mask = 0; 2569 int i; 2570 2571 /* there's only one device per system; not tied to any bus */ 2572 if (pvt->info.pci_vtd == NULL) 2573 /* result will be checked later */ 2574 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2575 PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC, 2576 NULL); 2577 2578 for (i = 0; i < sbridge_dev->n_devs; i++) { 2579 pdev = sbridge_dev->pdev[i]; 2580 if (!pdev) 2581 continue; 2582 2583 switch (pdev->device) { 2584 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0: 2585 pvt->pci_sad0 = pdev; 2586 break; 2587 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1: 2588 pvt->pci_sad1 = pdev; 2589 break; 2590 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0: 2591 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1: 2592 pvt->pci_ha = pdev; 2593 break; 2594 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA: 2595 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA: 2596 pvt->pci_ta = pdev; 2597 break; 2598 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM: 2599 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM: 2600 pvt->pci_ras = pdev; 2601 break; 2602 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0: 2603 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1: 2604 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2: 2605 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3: 2606 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0: 2607 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1: 2608 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2: 2609 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3: 2610 { 2611 int id = TAD_DEV_TO_CHAN(pdev->device); 2612 pvt->pci_tad[id] = pdev; 2613 saw_chan_mask |= 1 << id; 2614 } 2615 break; 2616 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0: 2617 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1: 2618 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2: 2619 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3: 2620 if (!pvt->pci_ddrio) 2621 pvt->pci_ddrio = pdev; 2622 break; 2623 default: 2624 break; 2625 } 2626 2627 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2628 sbridge_dev->bus, 2629 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2630 pdev); 2631 } 2632 2633 /* Check if everything were registered */ 2634 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2635 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2636 goto enodev; 2637 2638 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2639 saw_chan_mask != 0x03) /* -EP */ 2640 goto enodev; 2641 return 0; 2642 2643 enodev: 2644 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2645 return -ENODEV; 2646 } 2647 2648 static int broadwell_mci_bind_devs(struct mem_ctl_info *mci, 2649 struct sbridge_dev *sbridge_dev) 2650 { 2651 struct sbridge_pvt *pvt = mci->pvt_info; 2652 struct pci_dev *pdev; 2653 u8 saw_chan_mask = 0; 2654 int i; 2655 2656 /* there's only one device per system; not tied to any bus */ 2657 if (pvt->info.pci_vtd == NULL) 2658 /* result will be checked later */ 2659 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2660 PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC, 2661 NULL); 2662 2663 for (i = 0; i < sbridge_dev->n_devs; i++) { 2664 pdev = sbridge_dev->pdev[i]; 2665 if (!pdev) 2666 continue; 2667 2668 switch (pdev->device) { 2669 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0: 2670 pvt->pci_sad0 = pdev; 2671 break; 2672 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1: 2673 pvt->pci_sad1 = pdev; 2674 break; 2675 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0: 2676 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1: 2677 pvt->pci_ha = pdev; 2678 break; 2679 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA: 2680 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA: 2681 pvt->pci_ta = pdev; 2682 break; 2683 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM: 2684 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM: 2685 pvt->pci_ras = pdev; 2686 break; 2687 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0: 2688 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1: 2689 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2: 2690 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3: 2691 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0: 2692 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1: 2693 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2: 2694 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3: 2695 { 2696 int id = TAD_DEV_TO_CHAN(pdev->device); 2697 pvt->pci_tad[id] = pdev; 2698 saw_chan_mask |= 1 << id; 2699 } 2700 break; 2701 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0: 2702 pvt->pci_ddrio = pdev; 2703 break; 2704 default: 2705 break; 2706 } 2707 2708 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2709 sbridge_dev->bus, 2710 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2711 pdev); 2712 } 2713 2714 /* Check if everything were registered */ 2715 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2716 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2717 goto enodev; 2718 2719 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2720 saw_chan_mask != 0x03) /* -EP */ 2721 goto enodev; 2722 return 0; 2723 2724 enodev: 2725 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2726 return -ENODEV; 2727 } 2728 2729 static int knl_mci_bind_devs(struct mem_ctl_info *mci, 2730 struct sbridge_dev *sbridge_dev) 2731 { 2732 struct sbridge_pvt *pvt = mci->pvt_info; 2733 struct pci_dev *pdev; 2734 int dev, func; 2735 2736 int i; 2737 int devidx; 2738 2739 for (i = 0; i < sbridge_dev->n_devs; i++) { 2740 pdev = sbridge_dev->pdev[i]; 2741 if (!pdev) 2742 continue; 2743 2744 /* Extract PCI device and function. */ 2745 dev = (pdev->devfn >> 3) & 0x1f; 2746 func = pdev->devfn & 0x7; 2747 2748 switch (pdev->device) { 2749 case PCI_DEVICE_ID_INTEL_KNL_IMC_MC: 2750 if (dev == 8) 2751 pvt->knl.pci_mc0 = pdev; 2752 else if (dev == 9) 2753 pvt->knl.pci_mc1 = pdev; 2754 else { 2755 sbridge_printk(KERN_ERR, 2756 "Memory controller in unexpected place! (dev %d, fn %d)\n", 2757 dev, func); 2758 continue; 2759 } 2760 break; 2761 2762 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0: 2763 pvt->pci_sad0 = pdev; 2764 break; 2765 2766 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1: 2767 pvt->pci_sad1 = pdev; 2768 break; 2769 2770 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHA: 2771 /* There are one of these per tile, and range from 2772 * 1.14.0 to 1.18.5. 2773 */ 2774 devidx = ((dev-14)*8)+func; 2775 2776 if (devidx < 0 || devidx >= KNL_MAX_CHAS) { 2777 sbridge_printk(KERN_ERR, 2778 "Caching and Home Agent in unexpected place! (dev %d, fn %d)\n", 2779 dev, func); 2780 continue; 2781 } 2782 2783 WARN_ON(pvt->knl.pci_cha[devidx] != NULL); 2784 2785 pvt->knl.pci_cha[devidx] = pdev; 2786 break; 2787 2788 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN: 2789 devidx = -1; 2790 2791 /* 2792 * MC0 channels 0-2 are device 9 function 2-4, 2793 * MC1 channels 3-5 are device 8 function 2-4. 2794 */ 2795 2796 if (dev == 9) 2797 devidx = func-2; 2798 else if (dev == 8) 2799 devidx = 3 + (func-2); 2800 2801 if (devidx < 0 || devidx >= KNL_MAX_CHANNELS) { 2802 sbridge_printk(KERN_ERR, 2803 "DRAM Channel Registers in unexpected place! (dev %d, fn %d)\n", 2804 dev, func); 2805 continue; 2806 } 2807 2808 WARN_ON(pvt->knl.pci_channel[devidx] != NULL); 2809 pvt->knl.pci_channel[devidx] = pdev; 2810 break; 2811 2812 case PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM: 2813 pvt->knl.pci_mc_info = pdev; 2814 break; 2815 2816 case PCI_DEVICE_ID_INTEL_KNL_IMC_TA: 2817 pvt->pci_ta = pdev; 2818 break; 2819 2820 default: 2821 sbridge_printk(KERN_ERR, "Unexpected device %d\n", 2822 pdev->device); 2823 break; 2824 } 2825 } 2826 2827 if (!pvt->knl.pci_mc0 || !pvt->knl.pci_mc1 || 2828 !pvt->pci_sad0 || !pvt->pci_sad1 || 2829 !pvt->pci_ta) { 2830 goto enodev; 2831 } 2832 2833 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 2834 if (!pvt->knl.pci_channel[i]) { 2835 sbridge_printk(KERN_ERR, "Missing channel %d\n", i); 2836 goto enodev; 2837 } 2838 } 2839 2840 for (i = 0; i < KNL_MAX_CHAS; i++) { 2841 if (!pvt->knl.pci_cha[i]) { 2842 sbridge_printk(KERN_ERR, "Missing CHA %d\n", i); 2843 goto enodev; 2844 } 2845 } 2846 2847 return 0; 2848 2849 enodev: 2850 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2851 return -ENODEV; 2852 } 2853 2854 /**************************************************************************** 2855 Error check routines 2856 ****************************************************************************/ 2857 2858 /* 2859 * While Sandy Bridge has error count registers, SMI BIOS read values from 2860 * and resets the counters. So, they are not reliable for the OS to read 2861 * from them. So, we have no option but to just trust on whatever MCE is 2862 * telling us about the errors. 2863 */ 2864 static void sbridge_mce_output_error(struct mem_ctl_info *mci, 2865 const struct mce *m) 2866 { 2867 struct mem_ctl_info *new_mci; 2868 struct sbridge_pvt *pvt = mci->pvt_info; 2869 enum hw_event_mc_err_type tp_event; 2870 char *type, *optype, msg[256]; 2871 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0); 2872 bool overflow = GET_BITFIELD(m->status, 62, 62); 2873 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61); 2874 bool recoverable; 2875 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); 2876 u32 mscod = GET_BITFIELD(m->status, 16, 31); 2877 u32 errcode = GET_BITFIELD(m->status, 0, 15); 2878 u32 channel = GET_BITFIELD(m->status, 0, 3); 2879 u32 optypenum = GET_BITFIELD(m->status, 4, 6); 2880 long channel_mask, first_channel; 2881 u8 rank, socket, ha; 2882 int rc, dimm; 2883 char *area_type = NULL; 2884 2885 if (pvt->info.type != SANDY_BRIDGE) 2886 recoverable = true; 2887 else 2888 recoverable = GET_BITFIELD(m->status, 56, 56); 2889 2890 if (uncorrected_error) { 2891 if (ripv) { 2892 type = "FATAL"; 2893 tp_event = HW_EVENT_ERR_FATAL; 2894 } else { 2895 type = "NON_FATAL"; 2896 tp_event = HW_EVENT_ERR_UNCORRECTED; 2897 } 2898 } else { 2899 type = "CORRECTED"; 2900 tp_event = HW_EVENT_ERR_CORRECTED; 2901 } 2902 2903 /* 2904 * According with Table 15-9 of the Intel Architecture spec vol 3A, 2905 * memory errors should fit in this mask: 2906 * 000f 0000 1mmm cccc (binary) 2907 * where: 2908 * f = Correction Report Filtering Bit. If 1, subsequent errors 2909 * won't be shown 2910 * mmm = error type 2911 * cccc = channel 2912 * If the mask doesn't match, report an error to the parsing logic 2913 */ 2914 if (! ((errcode & 0xef80) == 0x80)) { 2915 optype = "Can't parse: it is not a mem"; 2916 } else { 2917 switch (optypenum) { 2918 case 0: 2919 optype = "generic undef request error"; 2920 break; 2921 case 1: 2922 optype = "memory read error"; 2923 break; 2924 case 2: 2925 optype = "memory write error"; 2926 break; 2927 case 3: 2928 optype = "addr/cmd error"; 2929 break; 2930 case 4: 2931 optype = "memory scrubbing error"; 2932 break; 2933 default: 2934 optype = "reserved"; 2935 break; 2936 } 2937 } 2938 2939 /* Only decode errors with an valid address (ADDRV) */ 2940 if (!GET_BITFIELD(m->status, 58, 58)) 2941 return; 2942 2943 if (pvt->info.type == KNIGHTS_LANDING) { 2944 if (channel == 14) { 2945 edac_dbg(0, "%s%s err_code:%04x:%04x EDRAM bank %d\n", 2946 overflow ? " OVERFLOW" : "", 2947 (uncorrected_error && recoverable) 2948 ? " recoverable" : "", 2949 mscod, errcode, 2950 m->bank); 2951 } else { 2952 char A = *("A"); 2953 2954 /* 2955 * Reported channel is in range 0-2, so we can't map it 2956 * back to mc. To figure out mc we check machine check 2957 * bank register that reported this error. 2958 * bank15 means mc0 and bank16 means mc1. 2959 */ 2960 channel = knl_channel_remap(m->bank == 16, channel); 2961 channel_mask = 1 << channel; 2962 2963 snprintf(msg, sizeof(msg), 2964 "%s%s err_code:%04x:%04x channel:%d (DIMM_%c)", 2965 overflow ? " OVERFLOW" : "", 2966 (uncorrected_error && recoverable) 2967 ? " recoverable" : " ", 2968 mscod, errcode, channel, A + channel); 2969 edac_mc_handle_error(tp_event, mci, core_err_cnt, 2970 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 2971 channel, 0, -1, 2972 optype, msg); 2973 } 2974 return; 2975 } else { 2976 rc = get_memory_error_data(mci, m->addr, &socket, &ha, 2977 &channel_mask, &rank, &area_type, msg); 2978 } 2979 2980 if (rc < 0) 2981 goto err_parsing; 2982 new_mci = get_mci_for_node_id(socket, ha); 2983 if (!new_mci) { 2984 strcpy(msg, "Error: socket got corrupted!"); 2985 goto err_parsing; 2986 } 2987 mci = new_mci; 2988 pvt = mci->pvt_info; 2989 2990 first_channel = find_first_bit(&channel_mask, NUM_CHANNELS); 2991 2992 if (rank < 4) 2993 dimm = 0; 2994 else if (rank < 8) 2995 dimm = 1; 2996 else 2997 dimm = 2; 2998 2999 3000 /* 3001 * FIXME: On some memory configurations (mirror, lockstep), the 3002 * Memory Controller can't point the error to a single DIMM. The 3003 * EDAC core should be handling the channel mask, in order to point 3004 * to the group of dimm's where the error may be happening. 3005 */ 3006 if (!pvt->is_lockstep && !pvt->is_cur_addr_mirrored && !pvt->is_close_pg) 3007 channel = first_channel; 3008 3009 snprintf(msg, sizeof(msg), 3010 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d", 3011 overflow ? " OVERFLOW" : "", 3012 (uncorrected_error && recoverable) ? " recoverable" : "", 3013 area_type, 3014 mscod, errcode, 3015 socket, ha, 3016 channel_mask, 3017 rank); 3018 3019 edac_dbg(0, "%s\n", msg); 3020 3021 /* FIXME: need support for channel mask */ 3022 3023 if (channel == CHANNEL_UNSPECIFIED) 3024 channel = -1; 3025 3026 /* Call the helper to output message */ 3027 edac_mc_handle_error(tp_event, mci, core_err_cnt, 3028 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 3029 channel, dimm, -1, 3030 optype, msg); 3031 return; 3032 err_parsing: 3033 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, 3034 -1, -1, -1, 3035 msg, ""); 3036 3037 } 3038 3039 /* 3040 * Check that logging is enabled and that this is the right type 3041 * of error for us to handle. 3042 */ 3043 static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, 3044 void *data) 3045 { 3046 struct mce *mce = (struct mce *)data; 3047 struct mem_ctl_info *mci; 3048 struct sbridge_pvt *pvt; 3049 char *type; 3050 3051 if (edac_get_report_status() == EDAC_REPORTING_DISABLED) 3052 return NOTIFY_DONE; 3053 3054 mci = get_mci_for_node_id(mce->socketid, IMC0); 3055 if (!mci) 3056 return NOTIFY_DONE; 3057 pvt = mci->pvt_info; 3058 3059 /* 3060 * Just let mcelog handle it if the error is 3061 * outside the memory controller. A memory error 3062 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0. 3063 * bit 12 has an special meaning. 3064 */ 3065 if ((mce->status & 0xefff) >> 7 != 1) 3066 return NOTIFY_DONE; 3067 3068 if (mce->mcgstatus & MCG_STATUS_MCIP) 3069 type = "Exception"; 3070 else 3071 type = "Event"; 3072 3073 sbridge_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n"); 3074 3075 sbridge_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx " 3076 "Bank %d: %016Lx\n", mce->extcpu, type, 3077 mce->mcgstatus, mce->bank, mce->status); 3078 sbridge_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc); 3079 sbridge_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr); 3080 sbridge_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc); 3081 3082 sbridge_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET " 3083 "%u APIC %x\n", mce->cpuvendor, mce->cpuid, 3084 mce->time, mce->socketid, mce->apicid); 3085 3086 sbridge_mce_output_error(mci, mce); 3087 3088 /* Advice mcelog that the error were handled */ 3089 return NOTIFY_STOP; 3090 } 3091 3092 static struct notifier_block sbridge_mce_dec = { 3093 .notifier_call = sbridge_mce_check_error, 3094 .priority = MCE_PRIO_EDAC, 3095 }; 3096 3097 /**************************************************************************** 3098 EDAC register/unregister logic 3099 ****************************************************************************/ 3100 3101 static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev) 3102 { 3103 struct mem_ctl_info *mci = sbridge_dev->mci; 3104 struct sbridge_pvt *pvt; 3105 3106 if (unlikely(!mci || !mci->pvt_info)) { 3107 edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev); 3108 3109 sbridge_printk(KERN_ERR, "Couldn't find mci handler\n"); 3110 return; 3111 } 3112 3113 pvt = mci->pvt_info; 3114 3115 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3116 mci, &sbridge_dev->pdev[0]->dev); 3117 3118 /* Remove MC sysfs nodes */ 3119 edac_mc_del_mc(mci->pdev); 3120 3121 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 3122 kfree(mci->ctl_name); 3123 edac_mc_free(mci); 3124 sbridge_dev->mci = NULL; 3125 } 3126 3127 static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) 3128 { 3129 struct mem_ctl_info *mci; 3130 struct edac_mc_layer layers[2]; 3131 struct sbridge_pvt *pvt; 3132 struct pci_dev *pdev = sbridge_dev->pdev[0]; 3133 int rc; 3134 3135 /* allocate a new MC control structure */ 3136 layers[0].type = EDAC_MC_LAYER_CHANNEL; 3137 layers[0].size = type == KNIGHTS_LANDING ? 3138 KNL_MAX_CHANNELS : NUM_CHANNELS; 3139 layers[0].is_virt_csrow = false; 3140 layers[1].type = EDAC_MC_LAYER_SLOT; 3141 layers[1].size = type == KNIGHTS_LANDING ? 1 : MAX_DIMMS; 3142 layers[1].is_virt_csrow = true; 3143 mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers, 3144 sizeof(*pvt)); 3145 3146 if (unlikely(!mci)) 3147 return -ENOMEM; 3148 3149 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3150 mci, &pdev->dev); 3151 3152 pvt = mci->pvt_info; 3153 memset(pvt, 0, sizeof(*pvt)); 3154 3155 /* Associate sbridge_dev and mci for future usage */ 3156 pvt->sbridge_dev = sbridge_dev; 3157 sbridge_dev->mci = mci; 3158 3159 mci->mtype_cap = type == KNIGHTS_LANDING ? 3160 MEM_FLAG_DDR4 : MEM_FLAG_DDR3; 3161 mci->edac_ctl_cap = EDAC_FLAG_NONE; 3162 mci->edac_cap = EDAC_FLAG_NONE; 3163 mci->mod_name = EDAC_MOD_STR; 3164 mci->dev_name = pci_name(pdev); 3165 mci->ctl_page_to_phys = NULL; 3166 3167 pvt->info.type = type; 3168 switch (type) { 3169 case IVY_BRIDGE: 3170 pvt->info.rankcfgr = IB_RANK_CFG_A; 3171 pvt->info.get_tolm = ibridge_get_tolm; 3172 pvt->info.get_tohm = ibridge_get_tohm; 3173 pvt->info.dram_rule = ibridge_dram_rule; 3174 pvt->info.get_memory_type = get_memory_type; 3175 pvt->info.get_node_id = get_node_id; 3176 pvt->info.rir_limit = rir_limit; 3177 pvt->info.sad_limit = sad_limit; 3178 pvt->info.interleave_mode = interleave_mode; 3179 pvt->info.dram_attr = dram_attr; 3180 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3181 pvt->info.interleave_list = ibridge_interleave_list; 3182 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3183 pvt->info.get_width = ibridge_get_width; 3184 3185 /* Store pci devices at mci for faster access */ 3186 rc = ibridge_mci_bind_devs(mci, sbridge_dev); 3187 if (unlikely(rc < 0)) 3188 goto fail0; 3189 get_source_id(mci); 3190 mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge SrcID#%d_Ha#%d", 3191 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3192 break; 3193 case SANDY_BRIDGE: 3194 pvt->info.rankcfgr = SB_RANK_CFG_A; 3195 pvt->info.get_tolm = sbridge_get_tolm; 3196 pvt->info.get_tohm = sbridge_get_tohm; 3197 pvt->info.dram_rule = sbridge_dram_rule; 3198 pvt->info.get_memory_type = get_memory_type; 3199 pvt->info.get_node_id = get_node_id; 3200 pvt->info.rir_limit = rir_limit; 3201 pvt->info.sad_limit = sad_limit; 3202 pvt->info.interleave_mode = interleave_mode; 3203 pvt->info.dram_attr = dram_attr; 3204 pvt->info.max_sad = ARRAY_SIZE(sbridge_dram_rule); 3205 pvt->info.interleave_list = sbridge_interleave_list; 3206 pvt->info.interleave_pkg = sbridge_interleave_pkg; 3207 pvt->info.get_width = sbridge_get_width; 3208 3209 /* Store pci devices at mci for faster access */ 3210 rc = sbridge_mci_bind_devs(mci, sbridge_dev); 3211 if (unlikely(rc < 0)) 3212 goto fail0; 3213 get_source_id(mci); 3214 mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge SrcID#%d_Ha#%d", 3215 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3216 break; 3217 case HASWELL: 3218 /* rankcfgr isn't used */ 3219 pvt->info.get_tolm = haswell_get_tolm; 3220 pvt->info.get_tohm = haswell_get_tohm; 3221 pvt->info.dram_rule = ibridge_dram_rule; 3222 pvt->info.get_memory_type = haswell_get_memory_type; 3223 pvt->info.get_node_id = haswell_get_node_id; 3224 pvt->info.rir_limit = haswell_rir_limit; 3225 pvt->info.sad_limit = sad_limit; 3226 pvt->info.interleave_mode = interleave_mode; 3227 pvt->info.dram_attr = dram_attr; 3228 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3229 pvt->info.interleave_list = ibridge_interleave_list; 3230 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3231 pvt->info.get_width = ibridge_get_width; 3232 3233 /* Store pci devices at mci for faster access */ 3234 rc = haswell_mci_bind_devs(mci, sbridge_dev); 3235 if (unlikely(rc < 0)) 3236 goto fail0; 3237 get_source_id(mci); 3238 mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell SrcID#%d_Ha#%d", 3239 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3240 break; 3241 case BROADWELL: 3242 /* rankcfgr isn't used */ 3243 pvt->info.get_tolm = haswell_get_tolm; 3244 pvt->info.get_tohm = haswell_get_tohm; 3245 pvt->info.dram_rule = ibridge_dram_rule; 3246 pvt->info.get_memory_type = haswell_get_memory_type; 3247 pvt->info.get_node_id = haswell_get_node_id; 3248 pvt->info.rir_limit = haswell_rir_limit; 3249 pvt->info.sad_limit = sad_limit; 3250 pvt->info.interleave_mode = interleave_mode; 3251 pvt->info.dram_attr = dram_attr; 3252 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3253 pvt->info.interleave_list = ibridge_interleave_list; 3254 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3255 pvt->info.get_width = broadwell_get_width; 3256 3257 /* Store pci devices at mci for faster access */ 3258 rc = broadwell_mci_bind_devs(mci, sbridge_dev); 3259 if (unlikely(rc < 0)) 3260 goto fail0; 3261 get_source_id(mci); 3262 mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell SrcID#%d_Ha#%d", 3263 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3264 break; 3265 case KNIGHTS_LANDING: 3266 /* pvt->info.rankcfgr == ??? */ 3267 pvt->info.get_tolm = knl_get_tolm; 3268 pvt->info.get_tohm = knl_get_tohm; 3269 pvt->info.dram_rule = knl_dram_rule; 3270 pvt->info.get_memory_type = knl_get_memory_type; 3271 pvt->info.get_node_id = knl_get_node_id; 3272 pvt->info.rir_limit = NULL; 3273 pvt->info.sad_limit = knl_sad_limit; 3274 pvt->info.interleave_mode = knl_interleave_mode; 3275 pvt->info.dram_attr = dram_attr_knl; 3276 pvt->info.max_sad = ARRAY_SIZE(knl_dram_rule); 3277 pvt->info.interleave_list = knl_interleave_list; 3278 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3279 pvt->info.get_width = knl_get_width; 3280 3281 rc = knl_mci_bind_devs(mci, sbridge_dev); 3282 if (unlikely(rc < 0)) 3283 goto fail0; 3284 get_source_id(mci); 3285 mci->ctl_name = kasprintf(GFP_KERNEL, "Knights Landing SrcID#%d_Ha#%d", 3286 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3287 break; 3288 } 3289 3290 if (!mci->ctl_name) { 3291 rc = -ENOMEM; 3292 goto fail0; 3293 } 3294 3295 /* Get dimm basic config and the memory layout */ 3296 rc = get_dimm_config(mci); 3297 if (rc < 0) { 3298 edac_dbg(0, "MC: failed to get_dimm_config()\n"); 3299 goto fail; 3300 } 3301 get_memory_layout(mci); 3302 3303 /* record ptr to the generic device */ 3304 mci->pdev = &pdev->dev; 3305 3306 /* add this new MC control structure to EDAC's list of MCs */ 3307 if (unlikely(edac_mc_add_mc(mci))) { 3308 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 3309 rc = -EINVAL; 3310 goto fail; 3311 } 3312 3313 return 0; 3314 3315 fail: 3316 kfree(mci->ctl_name); 3317 fail0: 3318 edac_mc_free(mci); 3319 sbridge_dev->mci = NULL; 3320 return rc; 3321 } 3322 3323 #define ICPU(model, table) \ 3324 { X86_VENDOR_INTEL, 6, model, 0, (unsigned long)&table } 3325 3326 static const struct x86_cpu_id sbridge_cpuids[] = { 3327 ICPU(INTEL_FAM6_SANDYBRIDGE_X, pci_dev_descr_sbridge_table), 3328 ICPU(INTEL_FAM6_IVYBRIDGE_X, pci_dev_descr_ibridge_table), 3329 ICPU(INTEL_FAM6_HASWELL_X, pci_dev_descr_haswell_table), 3330 ICPU(INTEL_FAM6_BROADWELL_X, pci_dev_descr_broadwell_table), 3331 ICPU(INTEL_FAM6_BROADWELL_XEON_D, pci_dev_descr_broadwell_table), 3332 ICPU(INTEL_FAM6_XEON_PHI_KNL, pci_dev_descr_knl_table), 3333 ICPU(INTEL_FAM6_XEON_PHI_KNM, pci_dev_descr_knl_table), 3334 { } 3335 }; 3336 MODULE_DEVICE_TABLE(x86cpu, sbridge_cpuids); 3337 3338 /* 3339 * sbridge_probe Get all devices and register memory controllers 3340 * present. 3341 * return: 3342 * 0 for FOUND a device 3343 * < 0 for error code 3344 */ 3345 3346 static int sbridge_probe(const struct x86_cpu_id *id) 3347 { 3348 int rc = -ENODEV; 3349 u8 mc, num_mc = 0; 3350 struct sbridge_dev *sbridge_dev; 3351 struct pci_id_table *ptable = (struct pci_id_table *)id->driver_data; 3352 3353 /* get the pci devices we want to reserve for our use */ 3354 rc = sbridge_get_all_devices(&num_mc, ptable); 3355 3356 if (unlikely(rc < 0)) { 3357 edac_dbg(0, "couldn't get all devices\n"); 3358 goto fail0; 3359 } 3360 3361 mc = 0; 3362 3363 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 3364 edac_dbg(0, "Registering MC#%d (%d of %d)\n", 3365 mc, mc + 1, num_mc); 3366 3367 sbridge_dev->mc = mc++; 3368 rc = sbridge_register_mci(sbridge_dev, ptable->type); 3369 if (unlikely(rc < 0)) 3370 goto fail1; 3371 } 3372 3373 sbridge_printk(KERN_INFO, "%s\n", SBRIDGE_REVISION); 3374 3375 return 0; 3376 3377 fail1: 3378 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3379 sbridge_unregister_mci(sbridge_dev); 3380 3381 sbridge_put_all_devices(); 3382 fail0: 3383 return rc; 3384 } 3385 3386 /* 3387 * sbridge_remove cleanup 3388 * 3389 */ 3390 static void sbridge_remove(void) 3391 { 3392 struct sbridge_dev *sbridge_dev; 3393 3394 edac_dbg(0, "\n"); 3395 3396 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3397 sbridge_unregister_mci(sbridge_dev); 3398 3399 /* Release PCI resources */ 3400 sbridge_put_all_devices(); 3401 } 3402 3403 /* 3404 * sbridge_init Module entry function 3405 * Try to initialize this module for its devices 3406 */ 3407 static int __init sbridge_init(void) 3408 { 3409 const struct x86_cpu_id *id; 3410 const char *owner; 3411 int rc; 3412 3413 edac_dbg(2, "\n"); 3414 3415 owner = edac_get_owner(); 3416 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 3417 return -EBUSY; 3418 3419 id = x86_match_cpu(sbridge_cpuids); 3420 if (!id) 3421 return -ENODEV; 3422 3423 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 3424 opstate_init(); 3425 3426 rc = sbridge_probe(id); 3427 3428 if (rc >= 0) { 3429 mce_register_decode_chain(&sbridge_mce_dec); 3430 if (edac_get_report_status() == EDAC_REPORTING_DISABLED) 3431 sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n"); 3432 return 0; 3433 } 3434 3435 sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n", 3436 rc); 3437 3438 return rc; 3439 } 3440 3441 /* 3442 * sbridge_exit() Module exit function 3443 * Unregister the driver 3444 */ 3445 static void __exit sbridge_exit(void) 3446 { 3447 edac_dbg(2, "\n"); 3448 sbridge_remove(); 3449 mce_unregister_decode_chain(&sbridge_mce_dec); 3450 } 3451 3452 module_init(sbridge_init); 3453 module_exit(sbridge_exit); 3454 3455 module_param(edac_op_state, int, 0444); 3456 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 3457 3458 MODULE_LICENSE("GPL"); 3459 MODULE_AUTHOR("Mauro Carvalho Chehab"); 3460 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)"); 3461 MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - " 3462 SBRIDGE_REVISION); 3463