1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Intel i7 core/Nehalem Memory Controller kernel module 3 * 4 * This driver supports the memory controllers found on the Intel 5 * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx, 6 * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield 7 * and Westmere-EP. 8 * 9 * Copyright (c) 2009-2010 by: 10 * Mauro Carvalho Chehab 11 * 12 * Red Hat Inc. https://www.redhat.com 13 * 14 * Forked and adapted from the i5400_edac driver 15 * 16 * Based on the following public Intel datasheets: 17 * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor 18 * Datasheet, Volume 2: 19 * http://download.intel.com/design/processor/datashts/320835.pdf 20 * Intel Xeon Processor 5500 Series Datasheet Volume 2 21 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf 22 * also available at: 23 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 24 */ 25 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/pci.h> 29 #include <linux/pci_ids.h> 30 #include <linux/slab.h> 31 #include <linux/delay.h> 32 #include <linux/dmi.h> 33 #include <linux/edac.h> 34 #include <linux/mmzone.h> 35 #include <linux/smp.h> 36 #include <asm/mce.h> 37 #include <asm/processor.h> 38 #include <asm/div64.h> 39 40 #include "edac_module.h" 41 42 /* Static vars */ 43 static LIST_HEAD(i7core_edac_list); 44 static DEFINE_MUTEX(i7core_edac_lock); 45 static int probed; 46 47 static int use_pci_fixup; 48 module_param(use_pci_fixup, int, 0444); 49 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices"); 50 /* 51 * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core 52 * registers start at bus 255, and are not reported by BIOS. 53 * We currently find devices with only 2 sockets. In order to support more QPI 54 * Quick Path Interconnect, just increment this number. 55 */ 56 #define MAX_SOCKET_BUSES 2 57 58 59 /* 60 * Alter this version for the module when modifications are made 61 */ 62 #define I7CORE_REVISION " Ver: 1.0.0" 63 #define EDAC_MOD_STR "i7core_edac" 64 65 /* 66 * Debug macros 67 */ 68 #define i7core_printk(level, fmt, arg...) \ 69 edac_printk(level, "i7core", fmt, ##arg) 70 71 #define i7core_mc_printk(mci, level, fmt, arg...) \ 72 edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg) 73 74 /* 75 * i7core Memory Controller Registers 76 */ 77 78 /* OFFSETS for Device 0 Function 0 */ 79 80 #define MC_CFG_CONTROL 0x90 81 #define MC_CFG_UNLOCK 0x02 82 #define MC_CFG_LOCK 0x00 83 84 /* OFFSETS for Device 3 Function 0 */ 85 86 #define MC_CONTROL 0x48 87 #define MC_STATUS 0x4c 88 #define MC_MAX_DOD 0x64 89 90 /* 91 * OFFSETS for Device 3 Function 4, as indicated on Xeon 5500 datasheet: 92 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 93 */ 94 95 #define MC_TEST_ERR_RCV1 0x60 96 #define DIMM2_COR_ERR(r) ((r) & 0x7fff) 97 98 #define MC_TEST_ERR_RCV0 0x64 99 #define DIMM1_COR_ERR(r) (((r) >> 16) & 0x7fff) 100 #define DIMM0_COR_ERR(r) ((r) & 0x7fff) 101 102 /* OFFSETS for Device 3 Function 2, as indicated on Xeon 5500 datasheet */ 103 #define MC_SSRCONTROL 0x48 104 #define SSR_MODE_DISABLE 0x00 105 #define SSR_MODE_ENABLE 0x01 106 #define SSR_MODE_MASK 0x03 107 108 #define MC_SCRUB_CONTROL 0x4c 109 #define STARTSCRUB (1 << 24) 110 #define SCRUBINTERVAL_MASK 0xffffff 111 112 #define MC_COR_ECC_CNT_0 0x80 113 #define MC_COR_ECC_CNT_1 0x84 114 #define MC_COR_ECC_CNT_2 0x88 115 #define MC_COR_ECC_CNT_3 0x8c 116 #define MC_COR_ECC_CNT_4 0x90 117 #define MC_COR_ECC_CNT_5 0x94 118 119 #define DIMM_TOP_COR_ERR(r) (((r) >> 16) & 0x7fff) 120 #define DIMM_BOT_COR_ERR(r) ((r) & 0x7fff) 121 122 123 /* OFFSETS for Devices 4,5 and 6 Function 0 */ 124 125 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58 126 #define THREE_DIMMS_PRESENT (1 << 24) 127 #define SINGLE_QUAD_RANK_PRESENT (1 << 23) 128 #define QUAD_RANK_PRESENT (1 << 22) 129 #define REGISTERED_DIMM (1 << 15) 130 131 #define MC_CHANNEL_MAPPER 0x60 132 #define RDLCH(r, ch) ((((r) >> (3 + (ch * 6))) & 0x07) - 1) 133 #define WRLCH(r, ch) ((((r) >> (ch * 6)) & 0x07) - 1) 134 135 #define MC_CHANNEL_RANK_PRESENT 0x7c 136 #define RANK_PRESENT_MASK 0xffff 137 138 #define MC_CHANNEL_ADDR_MATCH 0xf0 139 #define MC_CHANNEL_ERROR_MASK 0xf8 140 #define MC_CHANNEL_ERROR_INJECT 0xfc 141 #define INJECT_ADDR_PARITY 0x10 142 #define INJECT_ECC 0x08 143 #define MASK_CACHELINE 0x06 144 #define MASK_FULL_CACHELINE 0x06 145 #define MASK_MSB32_CACHELINE 0x04 146 #define MASK_LSB32_CACHELINE 0x02 147 #define NO_MASK_CACHELINE 0x00 148 #define REPEAT_EN 0x01 149 150 /* OFFSETS for Devices 4,5 and 6 Function 1 */ 151 152 #define MC_DOD_CH_DIMM0 0x48 153 #define MC_DOD_CH_DIMM1 0x4c 154 #define MC_DOD_CH_DIMM2 0x50 155 #define RANKOFFSET_MASK ((1 << 12) | (1 << 11) | (1 << 10)) 156 #define RANKOFFSET(x) ((x & RANKOFFSET_MASK) >> 10) 157 #define DIMM_PRESENT_MASK (1 << 9) 158 #define DIMM_PRESENT(x) (((x) & DIMM_PRESENT_MASK) >> 9) 159 #define MC_DOD_NUMBANK_MASK ((1 << 8) | (1 << 7)) 160 #define MC_DOD_NUMBANK(x) (((x) & MC_DOD_NUMBANK_MASK) >> 7) 161 #define MC_DOD_NUMRANK_MASK ((1 << 6) | (1 << 5)) 162 #define MC_DOD_NUMRANK(x) (((x) & MC_DOD_NUMRANK_MASK) >> 5) 163 #define MC_DOD_NUMROW_MASK ((1 << 4) | (1 << 3) | (1 << 2)) 164 #define MC_DOD_NUMROW(x) (((x) & MC_DOD_NUMROW_MASK) >> 2) 165 #define MC_DOD_NUMCOL_MASK 3 166 #define MC_DOD_NUMCOL(x) ((x) & MC_DOD_NUMCOL_MASK) 167 168 #define MC_RANK_PRESENT 0x7c 169 170 #define MC_SAG_CH_0 0x80 171 #define MC_SAG_CH_1 0x84 172 #define MC_SAG_CH_2 0x88 173 #define MC_SAG_CH_3 0x8c 174 #define MC_SAG_CH_4 0x90 175 #define MC_SAG_CH_5 0x94 176 #define MC_SAG_CH_6 0x98 177 #define MC_SAG_CH_7 0x9c 178 179 #define MC_RIR_LIMIT_CH_0 0x40 180 #define MC_RIR_LIMIT_CH_1 0x44 181 #define MC_RIR_LIMIT_CH_2 0x48 182 #define MC_RIR_LIMIT_CH_3 0x4C 183 #define MC_RIR_LIMIT_CH_4 0x50 184 #define MC_RIR_LIMIT_CH_5 0x54 185 #define MC_RIR_LIMIT_CH_6 0x58 186 #define MC_RIR_LIMIT_CH_7 0x5C 187 #define MC_RIR_LIMIT_MASK ((1 << 10) - 1) 188 189 #define MC_RIR_WAY_CH 0x80 190 #define MC_RIR_WAY_OFFSET_MASK (((1 << 14) - 1) & ~0x7) 191 #define MC_RIR_WAY_RANK_MASK 0x7 192 193 /* 194 * i7core structs 195 */ 196 197 #define NUM_CHANS 3 198 #define MAX_DIMMS 3 /* Max DIMMS per channel */ 199 #define MAX_MCR_FUNC 4 200 #define MAX_CHAN_FUNC 3 201 202 struct i7core_info { 203 u32 mc_control; 204 u32 mc_status; 205 u32 max_dod; 206 u32 ch_map; 207 }; 208 209 210 struct i7core_inject { 211 int enable; 212 213 u32 section; 214 u32 type; 215 u32 eccmask; 216 217 /* Error address mask */ 218 int channel, dimm, rank, bank, page, col; 219 }; 220 221 struct i7core_channel { 222 bool is_3dimms_present; 223 bool is_single_4rank; 224 bool has_4rank; 225 u32 dimms; 226 }; 227 228 struct pci_id_descr { 229 int dev; 230 int func; 231 int dev_id; 232 int optional; 233 }; 234 235 struct pci_id_table { 236 const struct pci_id_descr *descr; 237 int n_devs; 238 }; 239 240 struct i7core_dev { 241 struct list_head list; 242 u8 socket; 243 struct pci_dev **pdev; 244 int n_devs; 245 struct mem_ctl_info *mci; 246 }; 247 248 struct i7core_pvt { 249 struct device *addrmatch_dev, *chancounts_dev; 250 251 struct pci_dev *pci_noncore; 252 struct pci_dev *pci_mcr[MAX_MCR_FUNC + 1]; 253 struct pci_dev *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1]; 254 255 struct i7core_dev *i7core_dev; 256 257 struct i7core_info info; 258 struct i7core_inject inject; 259 struct i7core_channel channel[NUM_CHANS]; 260 261 int ce_count_available; 262 263 /* ECC corrected errors counts per udimm */ 264 unsigned long udimm_ce_count[MAX_DIMMS]; 265 int udimm_last_ce_count[MAX_DIMMS]; 266 /* ECC corrected errors counts per rdimm */ 267 unsigned long rdimm_ce_count[NUM_CHANS][MAX_DIMMS]; 268 int rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS]; 269 270 bool is_registered, enable_scrub; 271 272 /* DCLK Frequency used for computing scrub rate */ 273 int dclk_freq; 274 275 /* Struct to control EDAC polling */ 276 struct edac_pci_ctl_info *i7core_pci; 277 }; 278 279 #define PCI_DESCR(device, function, device_id) \ 280 .dev = (device), \ 281 .func = (function), \ 282 .dev_id = (device_id) 283 284 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = { 285 /* Memory controller */ 286 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR) }, 287 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD) }, 288 /* Exists only for RDIMM */ 289 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1 }, 290 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) }, 291 292 /* Channel 0 */ 293 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) }, 294 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) }, 295 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) }, 296 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC) }, 297 298 /* Channel 1 */ 299 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) }, 300 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) }, 301 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) }, 302 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC) }, 303 304 /* Channel 2 */ 305 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) }, 306 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) }, 307 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) }, 308 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC) }, 309 310 /* Generic Non-core registers */ 311 /* 312 * This is the PCI device on i7core and on Xeon 35xx (8086:2c41) 313 * On Xeon 55xx, however, it has a different id (8086:2c40). So, 314 * the probing code needs to test for the other address in case of 315 * failure of this one 316 */ 317 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE) }, 318 319 }; 320 321 static const struct pci_id_descr pci_dev_descr_lynnfield[] = { 322 { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR) }, 323 { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD) }, 324 { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST) }, 325 326 { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) }, 327 { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) }, 328 { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) }, 329 { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC) }, 330 331 { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) }, 332 { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) }, 333 { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) }, 334 { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC) }, 335 336 /* 337 * This is the PCI device has an alternate address on some 338 * processors like Core i7 860 339 */ 340 { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE) }, 341 }; 342 343 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = { 344 /* Memory controller */ 345 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2) }, 346 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2) }, 347 /* Exists only for RDIMM */ 348 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1 }, 349 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) }, 350 351 /* Channel 0 */ 352 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) }, 353 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) }, 354 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) }, 355 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2) }, 356 357 /* Channel 1 */ 358 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) }, 359 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) }, 360 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) }, 361 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2) }, 362 363 /* Channel 2 */ 364 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) }, 365 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) }, 366 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) }, 367 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2) }, 368 369 /* Generic Non-core registers */ 370 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2) }, 371 372 }; 373 374 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) } 375 static const struct pci_id_table pci_dev_table[] = { 376 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem), 377 PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield), 378 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere), 379 { NULL, } 380 }; 381 382 /* 383 * pci_device_id table for which devices we are looking for 384 */ 385 static const struct pci_device_id i7core_pci_tbl[] = { 386 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)}, 387 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)}, 388 { 0, } 389 }; 390 391 /**************************************************************************** 392 Ancillary status routines 393 ****************************************************************************/ 394 395 /* MC_CONTROL bits */ 396 #define CH_ACTIVE(pvt, ch) ((pvt)->info.mc_control & (1 << (8 + ch))) 397 #define ECCx8(pvt) ((pvt)->info.mc_control & (1 << 1)) 398 399 /* MC_STATUS bits */ 400 #define ECC_ENABLED(pvt) ((pvt)->info.mc_status & (1 << 4)) 401 #define CH_DISABLED(pvt, ch) ((pvt)->info.mc_status & (1 << ch)) 402 403 /* MC_MAX_DOD read functions */ 404 static inline int numdimms(u32 dimms) 405 { 406 return (dimms & 0x3) + 1; 407 } 408 409 static inline int numrank(u32 rank) 410 { 411 static const int ranks[] = { 1, 2, 4, -EINVAL }; 412 413 return ranks[rank & 0x3]; 414 } 415 416 static inline int numbank(u32 bank) 417 { 418 static const int banks[] = { 4, 8, 16, -EINVAL }; 419 420 return banks[bank & 0x3]; 421 } 422 423 static inline int numrow(u32 row) 424 { 425 static const int rows[] = { 426 1 << 12, 1 << 13, 1 << 14, 1 << 15, 427 1 << 16, -EINVAL, -EINVAL, -EINVAL, 428 }; 429 430 return rows[row & 0x7]; 431 } 432 433 static inline int numcol(u32 col) 434 { 435 static const int cols[] = { 436 1 << 10, 1 << 11, 1 << 12, -EINVAL, 437 }; 438 return cols[col & 0x3]; 439 } 440 441 static struct i7core_dev *get_i7core_dev(u8 socket) 442 { 443 struct i7core_dev *i7core_dev; 444 445 list_for_each_entry(i7core_dev, &i7core_edac_list, list) { 446 if (i7core_dev->socket == socket) 447 return i7core_dev; 448 } 449 450 return NULL; 451 } 452 453 static struct i7core_dev *alloc_i7core_dev(u8 socket, 454 const struct pci_id_table *table) 455 { 456 struct i7core_dev *i7core_dev; 457 458 i7core_dev = kzalloc_obj(*i7core_dev); 459 if (!i7core_dev) 460 return NULL; 461 462 i7core_dev->pdev = kzalloc_objs(*i7core_dev->pdev, table->n_devs); 463 if (!i7core_dev->pdev) { 464 kfree(i7core_dev); 465 return NULL; 466 } 467 468 i7core_dev->socket = socket; 469 i7core_dev->n_devs = table->n_devs; 470 list_add_tail(&i7core_dev->list, &i7core_edac_list); 471 472 return i7core_dev; 473 } 474 475 static void free_i7core_dev(struct i7core_dev *i7core_dev) 476 { 477 list_del(&i7core_dev->list); 478 kfree(i7core_dev->pdev); 479 kfree(i7core_dev); 480 } 481 482 /**************************************************************************** 483 Memory check routines 484 ****************************************************************************/ 485 486 static int get_dimm_config(struct mem_ctl_info *mci) 487 { 488 struct i7core_pvt *pvt = mci->pvt_info; 489 struct pci_dev *pdev; 490 int i, j; 491 enum edac_type mode; 492 enum mem_type mtype; 493 struct dimm_info *dimm; 494 495 /* Get data from the MC register, function 0 */ 496 pdev = pvt->pci_mcr[0]; 497 if (!pdev) 498 return -ENODEV; 499 500 /* Device 3 function 0 reads */ 501 pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control); 502 pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status); 503 pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod); 504 pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map); 505 506 edac_dbg(0, "QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n", 507 pvt->i7core_dev->socket, pvt->info.mc_control, 508 pvt->info.mc_status, pvt->info.max_dod, pvt->info.ch_map); 509 510 if (ECC_ENABLED(pvt)) { 511 edac_dbg(0, "ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4); 512 if (ECCx8(pvt)) 513 mode = EDAC_S8ECD8ED; 514 else 515 mode = EDAC_S4ECD4ED; 516 } else { 517 edac_dbg(0, "ECC disabled\n"); 518 mode = EDAC_NONE; 519 } 520 521 /* FIXME: need to handle the error codes */ 522 edac_dbg(0, "DOD Max limits: DIMMS: %d, %d-ranked, %d-banked x%x x 0x%x\n", 523 numdimms(pvt->info.max_dod), 524 numrank(pvt->info.max_dod >> 2), 525 numbank(pvt->info.max_dod >> 4), 526 numrow(pvt->info.max_dod >> 6), 527 numcol(pvt->info.max_dod >> 9)); 528 529 for (i = 0; i < NUM_CHANS; i++) { 530 u32 data, dimm_dod[3], value[8]; 531 532 if (!pvt->pci_ch[i][0]) 533 continue; 534 535 if (!CH_ACTIVE(pvt, i)) { 536 edac_dbg(0, "Channel %i is not active\n", i); 537 continue; 538 } 539 if (CH_DISABLED(pvt, i)) { 540 edac_dbg(0, "Channel %i is disabled\n", i); 541 continue; 542 } 543 544 /* Devices 4-6 function 0 */ 545 pci_read_config_dword(pvt->pci_ch[i][0], 546 MC_CHANNEL_DIMM_INIT_PARAMS, &data); 547 548 549 if (data & THREE_DIMMS_PRESENT) 550 pvt->channel[i].is_3dimms_present = true; 551 552 if (data & SINGLE_QUAD_RANK_PRESENT) 553 pvt->channel[i].is_single_4rank = true; 554 555 if (data & QUAD_RANK_PRESENT) 556 pvt->channel[i].has_4rank = true; 557 558 if (data & REGISTERED_DIMM) 559 mtype = MEM_RDDR3; 560 else 561 mtype = MEM_DDR3; 562 563 /* Devices 4-6 function 1 */ 564 pci_read_config_dword(pvt->pci_ch[i][1], 565 MC_DOD_CH_DIMM0, &dimm_dod[0]); 566 pci_read_config_dword(pvt->pci_ch[i][1], 567 MC_DOD_CH_DIMM1, &dimm_dod[1]); 568 pci_read_config_dword(pvt->pci_ch[i][1], 569 MC_DOD_CH_DIMM2, &dimm_dod[2]); 570 571 edac_dbg(0, "Ch%d phy rd%d, wr%d (0x%08x): %s%s%s%cDIMMs\n", 572 i, 573 RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i), 574 data, 575 pvt->channel[i].is_3dimms_present ? "3DIMMS " : "", 576 pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "", 577 pvt->channel[i].has_4rank ? "HAS_4R " : "", 578 (data & REGISTERED_DIMM) ? 'R' : 'U'); 579 580 for (j = 0; j < 3; j++) { 581 u32 banks, ranks, rows, cols; 582 u32 size, npages; 583 584 if (!DIMM_PRESENT(dimm_dod[j])) 585 continue; 586 587 dimm = edac_get_dimm(mci, i, j, 0); 588 banks = numbank(MC_DOD_NUMBANK(dimm_dod[j])); 589 ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j])); 590 rows = numrow(MC_DOD_NUMROW(dimm_dod[j])); 591 cols = numcol(MC_DOD_NUMCOL(dimm_dod[j])); 592 593 /* DDR3 has 8 I/O banks */ 594 size = (rows * cols * banks * ranks) >> (20 - 3); 595 596 edac_dbg(0, "\tdimm %d %d MiB offset: %x, bank: %d, rank: %d, row: %#x, col: %#x\n", 597 j, size, 598 RANKOFFSET(dimm_dod[j]), 599 banks, ranks, rows, cols); 600 601 npages = MiB_TO_PAGES(size); 602 603 dimm->nr_pages = npages; 604 605 switch (banks) { 606 case 4: 607 dimm->dtype = DEV_X4; 608 break; 609 case 8: 610 dimm->dtype = DEV_X8; 611 break; 612 case 16: 613 dimm->dtype = DEV_X16; 614 break; 615 default: 616 dimm->dtype = DEV_UNKNOWN; 617 } 618 619 snprintf(dimm->label, sizeof(dimm->label), 620 "CPU#%uChannel#%u_DIMM#%u", 621 pvt->i7core_dev->socket, i, j); 622 dimm->grain = 8; 623 dimm->edac_mode = mode; 624 dimm->mtype = mtype; 625 } 626 627 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]); 628 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]); 629 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]); 630 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]); 631 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]); 632 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]); 633 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]); 634 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]); 635 edac_dbg(1, "\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i); 636 for (j = 0; j < 8; j++) 637 edac_dbg(1, "\t\t%#x\t%#x\t%#x\n", 638 (value[j] >> 27) & 0x1, 639 (value[j] >> 24) & 0x7, 640 (value[j] & ((1 << 24) - 1))); 641 } 642 643 return 0; 644 } 645 646 /**************************************************************************** 647 Error insertion routines 648 ****************************************************************************/ 649 650 #define to_mci(k) container_of(k, struct mem_ctl_info, dev) 651 652 /* The i7core has independent error injection features per channel. 653 However, to have a simpler code, we don't allow enabling error injection 654 on more than one channel. 655 Also, since a change at an inject parameter will be applied only at enable, 656 we're disabling error injection on all write calls to the sysfs nodes that 657 controls the error code injection. 658 */ 659 static int disable_inject(const struct mem_ctl_info *mci) 660 { 661 struct i7core_pvt *pvt = mci->pvt_info; 662 663 pvt->inject.enable = 0; 664 665 if (!pvt->pci_ch[pvt->inject.channel][0]) 666 return -ENODEV; 667 668 pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0], 669 MC_CHANNEL_ERROR_INJECT, 0); 670 671 return 0; 672 } 673 674 /* 675 * i7core inject inject.section 676 * 677 * accept and store error injection inject.section value 678 * bit 0 - refers to the lower 32-byte half cacheline 679 * bit 1 - refers to the upper 32-byte half cacheline 680 */ 681 static ssize_t i7core_inject_section_store(struct device *dev, 682 struct device_attribute *mattr, 683 const char *data, size_t count) 684 { 685 struct mem_ctl_info *mci = to_mci(dev); 686 struct i7core_pvt *pvt = mci->pvt_info; 687 unsigned long value; 688 int rc; 689 690 if (pvt->inject.enable) 691 disable_inject(mci); 692 693 rc = kstrtoul(data, 10, &value); 694 if ((rc < 0) || (value > 3)) 695 return -EIO; 696 697 pvt->inject.section = (u32) value; 698 return count; 699 } 700 701 static ssize_t i7core_inject_section_show(struct device *dev, 702 struct device_attribute *mattr, 703 char *data) 704 { 705 struct mem_ctl_info *mci = to_mci(dev); 706 struct i7core_pvt *pvt = mci->pvt_info; 707 return sprintf(data, "0x%08x\n", pvt->inject.section); 708 } 709 710 /* 711 * i7core inject.type 712 * 713 * accept and store error injection inject.section value 714 * bit 0 - repeat enable - Enable error repetition 715 * bit 1 - inject ECC error 716 * bit 2 - inject parity error 717 */ 718 static ssize_t i7core_inject_type_store(struct device *dev, 719 struct device_attribute *mattr, 720 const char *data, size_t count) 721 { 722 struct mem_ctl_info *mci = to_mci(dev); 723 struct i7core_pvt *pvt = mci->pvt_info; 724 unsigned long value; 725 int rc; 726 727 if (pvt->inject.enable) 728 disable_inject(mci); 729 730 rc = kstrtoul(data, 10, &value); 731 if ((rc < 0) || (value > 7)) 732 return -EIO; 733 734 pvt->inject.type = (u32) value; 735 return count; 736 } 737 738 static ssize_t i7core_inject_type_show(struct device *dev, 739 struct device_attribute *mattr, 740 char *data) 741 { 742 struct mem_ctl_info *mci = to_mci(dev); 743 struct i7core_pvt *pvt = mci->pvt_info; 744 745 return sprintf(data, "0x%08x\n", pvt->inject.type); 746 } 747 748 /* 749 * i7core_inject_inject.eccmask_store 750 * 751 * The type of error (UE/CE) will depend on the inject.eccmask value: 752 * Any bits set to a 1 will flip the corresponding ECC bit 753 * Correctable errors can be injected by flipping 1 bit or the bits within 754 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or 755 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an 756 * uncorrectable error to be injected. 757 */ 758 static ssize_t i7core_inject_eccmask_store(struct device *dev, 759 struct device_attribute *mattr, 760 const char *data, size_t count) 761 { 762 struct mem_ctl_info *mci = to_mci(dev); 763 struct i7core_pvt *pvt = mci->pvt_info; 764 unsigned long value; 765 int rc; 766 767 if (pvt->inject.enable) 768 disable_inject(mci); 769 770 rc = kstrtoul(data, 10, &value); 771 if (rc < 0) 772 return -EIO; 773 774 pvt->inject.eccmask = (u32) value; 775 return count; 776 } 777 778 static ssize_t i7core_inject_eccmask_show(struct device *dev, 779 struct device_attribute *mattr, 780 char *data) 781 { 782 struct mem_ctl_info *mci = to_mci(dev); 783 struct i7core_pvt *pvt = mci->pvt_info; 784 785 return sprintf(data, "0x%08x\n", pvt->inject.eccmask); 786 } 787 788 /* 789 * i7core_addrmatch 790 * 791 * The type of error (UE/CE) will depend on the inject.eccmask value: 792 * Any bits set to a 1 will flip the corresponding ECC bit 793 * Correctable errors can be injected by flipping 1 bit or the bits within 794 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or 795 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an 796 * uncorrectable error to be injected. 797 */ 798 799 #define DECLARE_ADDR_MATCH(param, limit) \ 800 static ssize_t i7core_inject_store_##param( \ 801 struct device *dev, \ 802 struct device_attribute *mattr, \ 803 const char *data, size_t count) \ 804 { \ 805 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 806 struct i7core_pvt *pvt; \ 807 long value; \ 808 int rc; \ 809 \ 810 edac_dbg(1, "\n"); \ 811 pvt = mci->pvt_info; \ 812 \ 813 if (pvt->inject.enable) \ 814 disable_inject(mci); \ 815 \ 816 if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\ 817 value = -1; \ 818 else { \ 819 rc = kstrtoul(data, 10, &value); \ 820 if ((rc < 0) || (value >= limit)) \ 821 return -EIO; \ 822 } \ 823 \ 824 pvt->inject.param = value; \ 825 \ 826 return count; \ 827 } \ 828 \ 829 static ssize_t i7core_inject_show_##param( \ 830 struct device *dev, \ 831 struct device_attribute *mattr, \ 832 char *data) \ 833 { \ 834 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 835 struct i7core_pvt *pvt; \ 836 \ 837 pvt = mci->pvt_info; \ 838 edac_dbg(1, "pvt=%p\n", pvt); \ 839 if (pvt->inject.param < 0) \ 840 return sprintf(data, "any\n"); \ 841 else \ 842 return sprintf(data, "%d\n", pvt->inject.param);\ 843 } 844 845 #define ATTR_ADDR_MATCH(param) \ 846 static DEVICE_ATTR(param, S_IRUGO | S_IWUSR, \ 847 i7core_inject_show_##param, \ 848 i7core_inject_store_##param) 849 850 DECLARE_ADDR_MATCH(channel, 3); 851 DECLARE_ADDR_MATCH(dimm, 3); 852 DECLARE_ADDR_MATCH(rank, 4); 853 DECLARE_ADDR_MATCH(bank, 32); 854 DECLARE_ADDR_MATCH(page, 0x10000); 855 DECLARE_ADDR_MATCH(col, 0x4000); 856 857 ATTR_ADDR_MATCH(channel); 858 ATTR_ADDR_MATCH(dimm); 859 ATTR_ADDR_MATCH(rank); 860 ATTR_ADDR_MATCH(bank); 861 ATTR_ADDR_MATCH(page); 862 ATTR_ADDR_MATCH(col); 863 864 static int write_and_test(struct pci_dev *dev, const int where, const u32 val) 865 { 866 u32 read; 867 int count; 868 869 edac_dbg(0, "setting pci %02x:%02x.%x reg=%02x value=%08x\n", 870 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), 871 where, val); 872 873 for (count = 0; count < 10; count++) { 874 if (count) 875 msleep(100); 876 pci_write_config_dword(dev, where, val); 877 pci_read_config_dword(dev, where, &read); 878 879 if (read == val) 880 return 0; 881 } 882 883 i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x " 884 "write=%08x. Read=%08x\n", 885 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), 886 where, val, read); 887 888 return -EINVAL; 889 } 890 891 /* 892 * This routine prepares the Memory Controller for error injection. 893 * The error will be injected when some process tries to write to the 894 * memory that matches the given criteria. 895 * The criteria can be set in terms of a mask where dimm, rank, bank, page 896 * and col can be specified. 897 * A -1 value for any of the mask items will make the MCU to ignore 898 * that matching criteria for error injection. 899 * 900 * It should be noticed that the error will only happen after a write operation 901 * on a memory that matches the condition. if REPEAT_EN is not enabled at 902 * inject mask, then it will produce just one error. Otherwise, it will repeat 903 * until the injectmask would be cleaned. 904 * 905 * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD 906 * is reliable enough to check if the MC is using the 907 * three channels. However, this is not clear at the datasheet. 908 */ 909 static ssize_t i7core_inject_enable_store(struct device *dev, 910 struct device_attribute *mattr, 911 const char *data, size_t count) 912 { 913 struct mem_ctl_info *mci = to_mci(dev); 914 struct i7core_pvt *pvt = mci->pvt_info; 915 u32 injectmask; 916 u64 mask = 0; 917 int rc; 918 long enable; 919 920 if (!pvt->pci_ch[pvt->inject.channel][0]) 921 return 0; 922 923 rc = kstrtoul(data, 10, &enable); 924 if ((rc < 0)) 925 return 0; 926 927 if (enable) { 928 pvt->inject.enable = 1; 929 } else { 930 disable_inject(mci); 931 return count; 932 } 933 934 /* Sets pvt->inject.dimm mask */ 935 if (pvt->inject.dimm < 0) 936 mask |= 1LL << 41; 937 else { 938 if (pvt->channel[pvt->inject.channel].dimms > 2) 939 mask |= (pvt->inject.dimm & 0x3LL) << 35; 940 else 941 mask |= (pvt->inject.dimm & 0x1LL) << 36; 942 } 943 944 /* Sets pvt->inject.rank mask */ 945 if (pvt->inject.rank < 0) 946 mask |= 1LL << 40; 947 else { 948 if (pvt->channel[pvt->inject.channel].dimms > 2) 949 mask |= (pvt->inject.rank & 0x1LL) << 34; 950 else 951 mask |= (pvt->inject.rank & 0x3LL) << 34; 952 } 953 954 /* Sets pvt->inject.bank mask */ 955 if (pvt->inject.bank < 0) 956 mask |= 1LL << 39; 957 else 958 mask |= (pvt->inject.bank & 0x15LL) << 30; 959 960 /* Sets pvt->inject.page mask */ 961 if (pvt->inject.page < 0) 962 mask |= 1LL << 38; 963 else 964 mask |= (pvt->inject.page & 0xffff) << 14; 965 966 /* Sets pvt->inject.column mask */ 967 if (pvt->inject.col < 0) 968 mask |= 1LL << 37; 969 else 970 mask |= (pvt->inject.col & 0x3fff); 971 972 /* 973 * bit 0: REPEAT_EN 974 * bits 1-2: MASK_HALF_CACHELINE 975 * bit 3: INJECT_ECC 976 * bit 4: INJECT_ADDR_PARITY 977 */ 978 979 injectmask = (pvt->inject.type & 1) | 980 (pvt->inject.section & 0x3) << 1 | 981 (pvt->inject.type & 0x6) << (3 - 1); 982 983 /* Unlock writes to registers - this register is write only */ 984 pci_write_config_dword(pvt->pci_noncore, 985 MC_CFG_CONTROL, 0x2); 986 987 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 988 MC_CHANNEL_ADDR_MATCH, mask); 989 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 990 MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L); 991 992 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 993 MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask); 994 995 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 996 MC_CHANNEL_ERROR_INJECT, injectmask); 997 998 /* 999 * This is something undocumented, based on my tests 1000 * Without writing 8 to this register, errors aren't injected. Not sure 1001 * why. 1002 */ 1003 pci_write_config_dword(pvt->pci_noncore, 1004 MC_CFG_CONTROL, 8); 1005 1006 edac_dbg(0, "Error inject addr match 0x%016llx, ecc 0x%08x, inject 0x%08x\n", 1007 mask, pvt->inject.eccmask, injectmask); 1008 1009 1010 return count; 1011 } 1012 1013 static ssize_t i7core_inject_enable_show(struct device *dev, 1014 struct device_attribute *mattr, 1015 char *data) 1016 { 1017 struct mem_ctl_info *mci = to_mci(dev); 1018 struct i7core_pvt *pvt = mci->pvt_info; 1019 u32 injectmask; 1020 1021 if (!pvt->pci_ch[pvt->inject.channel][0]) 1022 return 0; 1023 1024 pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0], 1025 MC_CHANNEL_ERROR_INJECT, &injectmask); 1026 1027 edac_dbg(0, "Inject error read: 0x%018x\n", injectmask); 1028 1029 if (injectmask & 0x0c) 1030 pvt->inject.enable = 1; 1031 1032 return sprintf(data, "%d\n", pvt->inject.enable); 1033 } 1034 1035 #define DECLARE_COUNTER(param) \ 1036 static ssize_t i7core_show_counter_##param( \ 1037 struct device *dev, \ 1038 struct device_attribute *mattr, \ 1039 char *data) \ 1040 { \ 1041 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 1042 struct i7core_pvt *pvt = mci->pvt_info; \ 1043 \ 1044 edac_dbg(1, "\n"); \ 1045 if (!pvt->ce_count_available || (pvt->is_registered)) \ 1046 return sprintf(data, "data unavailable\n"); \ 1047 return sprintf(data, "%lu\n", \ 1048 pvt->udimm_ce_count[param]); \ 1049 } 1050 1051 #define ATTR_COUNTER(param) \ 1052 static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR, \ 1053 i7core_show_counter_##param, \ 1054 NULL) 1055 1056 DECLARE_COUNTER(0); 1057 DECLARE_COUNTER(1); 1058 DECLARE_COUNTER(2); 1059 1060 ATTR_COUNTER(0); 1061 ATTR_COUNTER(1); 1062 ATTR_COUNTER(2); 1063 1064 /* 1065 * inject_addrmatch device sysfs struct 1066 */ 1067 1068 static struct attribute *i7core_addrmatch_attrs[] = { 1069 &dev_attr_channel.attr, 1070 &dev_attr_dimm.attr, 1071 &dev_attr_rank.attr, 1072 &dev_attr_bank.attr, 1073 &dev_attr_page.attr, 1074 &dev_attr_col.attr, 1075 NULL 1076 }; 1077 1078 static const struct attribute_group addrmatch_grp = { 1079 .attrs = i7core_addrmatch_attrs, 1080 }; 1081 1082 static const struct attribute_group *addrmatch_groups[] = { 1083 &addrmatch_grp, 1084 NULL 1085 }; 1086 1087 static void addrmatch_release(struct device *device) 1088 { 1089 edac_dbg(1, "Releasing device %s\n", dev_name(device)); 1090 kfree(device); 1091 } 1092 1093 static const struct device_type addrmatch_type = { 1094 .groups = addrmatch_groups, 1095 .release = addrmatch_release, 1096 }; 1097 1098 /* 1099 * all_channel_counts sysfs struct 1100 */ 1101 1102 static struct attribute *i7core_udimm_counters_attrs[] = { 1103 &dev_attr_udimm0.attr, 1104 &dev_attr_udimm1.attr, 1105 &dev_attr_udimm2.attr, 1106 NULL 1107 }; 1108 1109 static const struct attribute_group all_channel_counts_grp = { 1110 .attrs = i7core_udimm_counters_attrs, 1111 }; 1112 1113 static const struct attribute_group *all_channel_counts_groups[] = { 1114 &all_channel_counts_grp, 1115 NULL 1116 }; 1117 1118 static void all_channel_counts_release(struct device *device) 1119 { 1120 edac_dbg(1, "Releasing device %s\n", dev_name(device)); 1121 kfree(device); 1122 } 1123 1124 static const struct device_type all_channel_counts_type = { 1125 .groups = all_channel_counts_groups, 1126 .release = all_channel_counts_release, 1127 }; 1128 1129 /* 1130 * inject sysfs attributes 1131 */ 1132 1133 static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR, 1134 i7core_inject_section_show, i7core_inject_section_store); 1135 1136 static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR, 1137 i7core_inject_type_show, i7core_inject_type_store); 1138 1139 1140 static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR, 1141 i7core_inject_eccmask_show, i7core_inject_eccmask_store); 1142 1143 static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR, 1144 i7core_inject_enable_show, i7core_inject_enable_store); 1145 1146 static struct attribute *i7core_dev_attrs[] = { 1147 &dev_attr_inject_section.attr, 1148 &dev_attr_inject_type.attr, 1149 &dev_attr_inject_eccmask.attr, 1150 &dev_attr_inject_enable.attr, 1151 NULL 1152 }; 1153 1154 ATTRIBUTE_GROUPS(i7core_dev); 1155 1156 static int i7core_create_sysfs_devices(struct mem_ctl_info *mci) 1157 { 1158 struct i7core_pvt *pvt = mci->pvt_info; 1159 int rc; 1160 1161 pvt->addrmatch_dev = kzalloc_obj(*pvt->addrmatch_dev); 1162 if (!pvt->addrmatch_dev) 1163 return -ENOMEM; 1164 1165 pvt->addrmatch_dev->type = &addrmatch_type; 1166 pvt->addrmatch_dev->bus = mci->dev.bus; 1167 device_initialize(pvt->addrmatch_dev); 1168 pvt->addrmatch_dev->parent = &mci->dev; 1169 dev_set_name(pvt->addrmatch_dev, "inject_addrmatch"); 1170 dev_set_drvdata(pvt->addrmatch_dev, mci); 1171 1172 edac_dbg(1, "creating %s\n", dev_name(pvt->addrmatch_dev)); 1173 1174 rc = device_add(pvt->addrmatch_dev); 1175 if (rc < 0) 1176 goto err_put_addrmatch; 1177 1178 if (!pvt->is_registered) { 1179 pvt->chancounts_dev = kzalloc_obj(*pvt->chancounts_dev); 1180 if (!pvt->chancounts_dev) { 1181 rc = -ENOMEM; 1182 goto err_del_addrmatch; 1183 } 1184 1185 pvt->chancounts_dev->type = &all_channel_counts_type; 1186 pvt->chancounts_dev->bus = mci->dev.bus; 1187 device_initialize(pvt->chancounts_dev); 1188 pvt->chancounts_dev->parent = &mci->dev; 1189 dev_set_name(pvt->chancounts_dev, "all_channel_counts"); 1190 dev_set_drvdata(pvt->chancounts_dev, mci); 1191 1192 edac_dbg(1, "creating %s\n", dev_name(pvt->chancounts_dev)); 1193 1194 rc = device_add(pvt->chancounts_dev); 1195 if (rc < 0) 1196 goto err_put_chancounts; 1197 } 1198 return 0; 1199 1200 err_put_chancounts: 1201 put_device(pvt->chancounts_dev); 1202 err_del_addrmatch: 1203 device_del(pvt->addrmatch_dev); 1204 err_put_addrmatch: 1205 put_device(pvt->addrmatch_dev); 1206 1207 return rc; 1208 } 1209 1210 static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci) 1211 { 1212 struct i7core_pvt *pvt = mci->pvt_info; 1213 1214 edac_dbg(1, "\n"); 1215 1216 if (!pvt->is_registered) { 1217 device_del(pvt->chancounts_dev); 1218 put_device(pvt->chancounts_dev); 1219 } 1220 device_del(pvt->addrmatch_dev); 1221 put_device(pvt->addrmatch_dev); 1222 } 1223 1224 /**************************************************************************** 1225 Device initialization routines: put/get, init/exit 1226 ****************************************************************************/ 1227 1228 /* 1229 * i7core_put_all_devices 'put' all the devices that we have 1230 * reserved via 'get' 1231 */ 1232 static void i7core_put_devices(struct i7core_dev *i7core_dev) 1233 { 1234 int i; 1235 1236 edac_dbg(0, "\n"); 1237 for (i = 0; i < i7core_dev->n_devs; i++) { 1238 struct pci_dev *pdev = i7core_dev->pdev[i]; 1239 if (!pdev) 1240 continue; 1241 edac_dbg(0, "Removing dev %02x:%02x.%d\n", 1242 pdev->bus->number, 1243 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); 1244 pci_dev_put(pdev); 1245 } 1246 } 1247 1248 static void i7core_put_all_devices(void) 1249 { 1250 struct i7core_dev *i7core_dev, *tmp; 1251 1252 list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) { 1253 i7core_put_devices(i7core_dev); 1254 free_i7core_dev(i7core_dev); 1255 } 1256 } 1257 1258 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table) 1259 { 1260 struct pci_dev *pdev = NULL; 1261 int i; 1262 1263 /* 1264 * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses 1265 * aren't announced by acpi. So, we need to use a legacy scan probing 1266 * to detect them 1267 */ 1268 while (table && table->descr) { 1269 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL); 1270 if (unlikely(!pdev)) { 1271 for (i = 0; i < MAX_SOCKET_BUSES; i++) 1272 pcibios_scan_specific_bus(255-i); 1273 } 1274 pci_dev_put(pdev); 1275 table++; 1276 } 1277 } 1278 1279 static unsigned i7core_pci_lastbus(void) 1280 { 1281 int last_bus = 0, bus; 1282 struct pci_bus *b = NULL; 1283 1284 while ((b = pci_find_next_bus(b)) != NULL) { 1285 bus = b->number; 1286 edac_dbg(0, "Found bus %d\n", bus); 1287 if (bus > last_bus) 1288 last_bus = bus; 1289 } 1290 1291 edac_dbg(0, "Last bus %d\n", last_bus); 1292 1293 return last_bus; 1294 } 1295 1296 /* 1297 * i7core_get_all_devices Find and perform 'get' operation on the MCH's 1298 * device/functions we want to reference for this driver 1299 * 1300 * Need to 'get' device 16 func 1 and func 2 1301 */ 1302 static int i7core_get_onedevice(struct pci_dev **prev, 1303 const struct pci_id_table *table, 1304 const unsigned devno, 1305 const unsigned last_bus) 1306 { 1307 struct i7core_dev *i7core_dev; 1308 const struct pci_id_descr *dev_descr = &table->descr[devno]; 1309 1310 struct pci_dev *pdev = NULL; 1311 u8 bus = 0; 1312 u8 socket = 0; 1313 1314 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1315 dev_descr->dev_id, *prev); 1316 1317 /* 1318 * On Xeon 55xx, the Intel QuickPath Arch Generic Non-core regs 1319 * is at addr 8086:2c40, instead of 8086:2c41. So, we need 1320 * to probe for the alternate address in case of failure 1321 */ 1322 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev) { 1323 pci_dev_get(*prev); /* pci_get_device will put it */ 1324 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1325 PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev); 1326 } 1327 1328 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && 1329 !pdev) { 1330 pci_dev_get(*prev); /* pci_get_device will put it */ 1331 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1332 PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT, 1333 *prev); 1334 } 1335 1336 if (!pdev) { 1337 if (*prev) { 1338 *prev = pdev; 1339 return 0; 1340 } 1341 1342 if (dev_descr->optional) 1343 return 0; 1344 1345 if (devno == 0) 1346 return -ENODEV; 1347 1348 i7core_printk(KERN_INFO, 1349 "Device not found: dev %02x.%d PCI ID %04x:%04x\n", 1350 dev_descr->dev, dev_descr->func, 1351 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1352 1353 /* End of list, leave */ 1354 return -ENODEV; 1355 } 1356 bus = pdev->bus->number; 1357 1358 socket = last_bus - bus; 1359 1360 i7core_dev = get_i7core_dev(socket); 1361 if (!i7core_dev) { 1362 i7core_dev = alloc_i7core_dev(socket, table); 1363 if (!i7core_dev) { 1364 pci_dev_put(pdev); 1365 return -ENOMEM; 1366 } 1367 } 1368 1369 if (i7core_dev->pdev[devno]) { 1370 i7core_printk(KERN_ERR, 1371 "Duplicated device for " 1372 "dev %02x:%02x.%d PCI ID %04x:%04x\n", 1373 bus, dev_descr->dev, dev_descr->func, 1374 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1375 pci_dev_put(pdev); 1376 return -ENODEV; 1377 } 1378 1379 i7core_dev->pdev[devno] = pdev; 1380 1381 /* Sanity check */ 1382 if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev || 1383 PCI_FUNC(pdev->devfn) != dev_descr->func)) { 1384 i7core_printk(KERN_ERR, 1385 "Device PCI ID %04x:%04x " 1386 "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n", 1387 PCI_VENDOR_ID_INTEL, dev_descr->dev_id, 1388 bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 1389 bus, dev_descr->dev, dev_descr->func); 1390 return -ENODEV; 1391 } 1392 1393 /* Be sure that the device is enabled */ 1394 if (unlikely(pci_enable_device(pdev) < 0)) { 1395 i7core_printk(KERN_ERR, 1396 "Couldn't enable " 1397 "dev %02x:%02x.%d PCI ID %04x:%04x\n", 1398 bus, dev_descr->dev, dev_descr->func, 1399 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1400 return -ENODEV; 1401 } 1402 1403 edac_dbg(0, "Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n", 1404 socket, bus, dev_descr->dev, 1405 dev_descr->func, 1406 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1407 1408 /* 1409 * As stated on drivers/pci/search.c, the reference count for 1410 * @from is always decremented if it is not %NULL. So, as we need 1411 * to get all devices up to null, we need to do a get for the device 1412 */ 1413 pci_dev_get(pdev); 1414 1415 *prev = pdev; 1416 1417 return 0; 1418 } 1419 1420 static int i7core_get_all_devices(void) 1421 { 1422 int i, rc, last_bus; 1423 struct pci_dev *pdev = NULL; 1424 const struct pci_id_table *table = pci_dev_table; 1425 1426 last_bus = i7core_pci_lastbus(); 1427 1428 while (table && table->descr) { 1429 for (i = 0; i < table->n_devs; i++) { 1430 pdev = NULL; 1431 do { 1432 rc = i7core_get_onedevice(&pdev, table, i, 1433 last_bus); 1434 if (rc < 0) { 1435 if (i == 0) { 1436 i = table->n_devs; 1437 break; 1438 } 1439 i7core_put_all_devices(); 1440 return -ENODEV; 1441 } 1442 } while (pdev); 1443 } 1444 table++; 1445 } 1446 1447 return 0; 1448 } 1449 1450 static int mci_bind_devs(struct mem_ctl_info *mci, 1451 struct i7core_dev *i7core_dev) 1452 { 1453 struct i7core_pvt *pvt = mci->pvt_info; 1454 struct pci_dev *pdev; 1455 int i, func, slot; 1456 char *family; 1457 1458 pvt->is_registered = false; 1459 pvt->enable_scrub = false; 1460 for (i = 0; i < i7core_dev->n_devs; i++) { 1461 pdev = i7core_dev->pdev[i]; 1462 if (!pdev) 1463 continue; 1464 1465 func = PCI_FUNC(pdev->devfn); 1466 slot = PCI_SLOT(pdev->devfn); 1467 if (slot == 3) { 1468 if (unlikely(func > MAX_MCR_FUNC)) 1469 goto error; 1470 pvt->pci_mcr[func] = pdev; 1471 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) { 1472 if (unlikely(func > MAX_CHAN_FUNC)) 1473 goto error; 1474 pvt->pci_ch[slot - 4][func] = pdev; 1475 } else if (!slot && !func) { 1476 pvt->pci_noncore = pdev; 1477 1478 /* Detect the processor family */ 1479 switch (pdev->device) { 1480 case PCI_DEVICE_ID_INTEL_I7_NONCORE: 1481 family = "Xeon 35xx/ i7core"; 1482 pvt->enable_scrub = false; 1483 break; 1484 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT: 1485 family = "i7-800/i5-700"; 1486 pvt->enable_scrub = false; 1487 break; 1488 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE: 1489 family = "Xeon 34xx"; 1490 pvt->enable_scrub = false; 1491 break; 1492 case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT: 1493 family = "Xeon 55xx"; 1494 pvt->enable_scrub = true; 1495 break; 1496 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2: 1497 family = "Xeon 56xx / i7-900"; 1498 pvt->enable_scrub = true; 1499 break; 1500 default: 1501 family = "unknown"; 1502 pvt->enable_scrub = false; 1503 } 1504 edac_dbg(0, "Detected a processor type %s\n", family); 1505 } else 1506 goto error; 1507 1508 edac_dbg(0, "Associated fn %d.%d, dev = %p, socket %d\n", 1509 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 1510 pdev, i7core_dev->socket); 1511 1512 if (PCI_SLOT(pdev->devfn) == 3 && 1513 PCI_FUNC(pdev->devfn) == 2) 1514 pvt->is_registered = true; 1515 } 1516 1517 return 0; 1518 1519 error: 1520 i7core_printk(KERN_ERR, "Device %d, function %d " 1521 "is out of the expected range\n", 1522 slot, func); 1523 return -EINVAL; 1524 } 1525 1526 /**************************************************************************** 1527 Error check routines 1528 ****************************************************************************/ 1529 1530 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci, 1531 const int chan, 1532 const int new0, 1533 const int new1, 1534 const int new2) 1535 { 1536 struct i7core_pvt *pvt = mci->pvt_info; 1537 int add0 = 0, add1 = 0, add2 = 0; 1538 /* Updates CE counters if it is not the first time here */ 1539 if (pvt->ce_count_available) { 1540 /* Updates CE counters */ 1541 1542 add2 = new2 - pvt->rdimm_last_ce_count[chan][2]; 1543 add1 = new1 - pvt->rdimm_last_ce_count[chan][1]; 1544 add0 = new0 - pvt->rdimm_last_ce_count[chan][0]; 1545 1546 if (add2 < 0) 1547 add2 += 0x7fff; 1548 pvt->rdimm_ce_count[chan][2] += add2; 1549 1550 if (add1 < 0) 1551 add1 += 0x7fff; 1552 pvt->rdimm_ce_count[chan][1] += add1; 1553 1554 if (add0 < 0) 1555 add0 += 0x7fff; 1556 pvt->rdimm_ce_count[chan][0] += add0; 1557 } else 1558 pvt->ce_count_available = 1; 1559 1560 /* Store the new values */ 1561 pvt->rdimm_last_ce_count[chan][2] = new2; 1562 pvt->rdimm_last_ce_count[chan][1] = new1; 1563 pvt->rdimm_last_ce_count[chan][0] = new0; 1564 1565 /*updated the edac core */ 1566 if (add0 != 0) 1567 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add0, 1568 0, 0, 0, 1569 chan, 0, -1, "error", ""); 1570 if (add1 != 0) 1571 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add1, 1572 0, 0, 0, 1573 chan, 1, -1, "error", ""); 1574 if (add2 != 0) 1575 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add2, 1576 0, 0, 0, 1577 chan, 2, -1, "error", ""); 1578 } 1579 1580 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci) 1581 { 1582 struct i7core_pvt *pvt = mci->pvt_info; 1583 u32 rcv[3][2]; 1584 int i, new0, new1, new2; 1585 1586 /*Read DEV 3: FUN 2: MC_COR_ECC_CNT regs directly*/ 1587 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0, 1588 &rcv[0][0]); 1589 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1, 1590 &rcv[0][1]); 1591 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2, 1592 &rcv[1][0]); 1593 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3, 1594 &rcv[1][1]); 1595 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4, 1596 &rcv[2][0]); 1597 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5, 1598 &rcv[2][1]); 1599 for (i = 0 ; i < 3; i++) { 1600 edac_dbg(3, "MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n", 1601 (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]); 1602 /*if the channel has 3 dimms*/ 1603 if (pvt->channel[i].dimms > 2) { 1604 new0 = DIMM_BOT_COR_ERR(rcv[i][0]); 1605 new1 = DIMM_TOP_COR_ERR(rcv[i][0]); 1606 new2 = DIMM_BOT_COR_ERR(rcv[i][1]); 1607 } else { 1608 new0 = DIMM_TOP_COR_ERR(rcv[i][0]) + 1609 DIMM_BOT_COR_ERR(rcv[i][0]); 1610 new1 = DIMM_TOP_COR_ERR(rcv[i][1]) + 1611 DIMM_BOT_COR_ERR(rcv[i][1]); 1612 new2 = 0; 1613 } 1614 1615 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2); 1616 } 1617 } 1618 1619 /* This function is based on the device 3 function 4 registers as described on: 1620 * Intel Xeon Processor 5500 Series Datasheet Volume 2 1621 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf 1622 * also available at: 1623 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 1624 */ 1625 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci) 1626 { 1627 struct i7core_pvt *pvt = mci->pvt_info; 1628 u32 rcv1, rcv0; 1629 int new0, new1, new2; 1630 1631 if (!pvt->pci_mcr[4]) { 1632 edac_dbg(0, "MCR registers not found\n"); 1633 return; 1634 } 1635 1636 /* Corrected test errors */ 1637 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1); 1638 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0); 1639 1640 /* Store the new values */ 1641 new2 = DIMM2_COR_ERR(rcv1); 1642 new1 = DIMM1_COR_ERR(rcv0); 1643 new0 = DIMM0_COR_ERR(rcv0); 1644 1645 /* Updates CE counters if it is not the first time here */ 1646 if (pvt->ce_count_available) { 1647 /* Updates CE counters */ 1648 int add0, add1, add2; 1649 1650 add2 = new2 - pvt->udimm_last_ce_count[2]; 1651 add1 = new1 - pvt->udimm_last_ce_count[1]; 1652 add0 = new0 - pvt->udimm_last_ce_count[0]; 1653 1654 if (add2 < 0) 1655 add2 += 0x7fff; 1656 pvt->udimm_ce_count[2] += add2; 1657 1658 if (add1 < 0) 1659 add1 += 0x7fff; 1660 pvt->udimm_ce_count[1] += add1; 1661 1662 if (add0 < 0) 1663 add0 += 0x7fff; 1664 pvt->udimm_ce_count[0] += add0; 1665 1666 if (add0 | add1 | add2) 1667 i7core_printk(KERN_ERR, "New Corrected error(s): " 1668 "dimm0: +%d, dimm1: +%d, dimm2 +%d\n", 1669 add0, add1, add2); 1670 } else 1671 pvt->ce_count_available = 1; 1672 1673 /* Store the new values */ 1674 pvt->udimm_last_ce_count[2] = new2; 1675 pvt->udimm_last_ce_count[1] = new1; 1676 pvt->udimm_last_ce_count[0] = new0; 1677 } 1678 1679 /* 1680 * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32 1681 * Architectures Software Developer’s Manual Volume 3B. 1682 * Nehalem are defined as family 0x06, model 0x1a 1683 * 1684 * The MCA registers used here are the following ones: 1685 * struct mce field MCA Register 1686 * m->status MSR_IA32_MC8_STATUS 1687 * m->addr MSR_IA32_MC8_ADDR 1688 * m->misc MSR_IA32_MC8_MISC 1689 * In the case of Nehalem, the error information is masked at .status and .misc 1690 * fields 1691 */ 1692 static void i7core_mce_output_error(struct mem_ctl_info *mci, 1693 const struct mce *m) 1694 { 1695 struct i7core_pvt *pvt = mci->pvt_info; 1696 char *optype, *err; 1697 enum hw_event_mc_err_type tp_event; 1698 unsigned long error = m->status & 0x1ff0000l; 1699 bool uncorrected_error = m->mcgstatus & 1ll << 61; 1700 bool ripv = m->mcgstatus & 1; 1701 u32 optypenum = (m->status >> 4) & 0x07; 1702 u32 core_err_cnt = (m->status >> 38) & 0x7fff; 1703 u32 dimm = (m->misc >> 16) & 0x3; 1704 u32 channel = (m->misc >> 18) & 0x3; 1705 u32 syndrome = m->misc >> 32; 1706 u32 errnum = find_first_bit(&error, 32); 1707 1708 if (uncorrected_error) { 1709 core_err_cnt = 1; 1710 if (ripv) 1711 tp_event = HW_EVENT_ERR_UNCORRECTED; 1712 else 1713 tp_event = HW_EVENT_ERR_FATAL; 1714 } else { 1715 tp_event = HW_EVENT_ERR_CORRECTED; 1716 } 1717 1718 switch (optypenum) { 1719 case 0: 1720 optype = "generic undef request"; 1721 break; 1722 case 1: 1723 optype = "read error"; 1724 break; 1725 case 2: 1726 optype = "write error"; 1727 break; 1728 case 3: 1729 optype = "addr/cmd error"; 1730 break; 1731 case 4: 1732 optype = "scrubbing error"; 1733 break; 1734 default: 1735 optype = "reserved"; 1736 break; 1737 } 1738 1739 switch (errnum) { 1740 case 16: 1741 err = "read ECC error"; 1742 break; 1743 case 17: 1744 err = "RAS ECC error"; 1745 break; 1746 case 18: 1747 err = "write parity error"; 1748 break; 1749 case 19: 1750 err = "redundancy loss"; 1751 break; 1752 case 20: 1753 err = "reserved"; 1754 break; 1755 case 21: 1756 err = "memory range error"; 1757 break; 1758 case 22: 1759 err = "RTID out of range"; 1760 break; 1761 case 23: 1762 err = "address parity error"; 1763 break; 1764 case 24: 1765 err = "byte enable parity error"; 1766 break; 1767 default: 1768 err = "unknown"; 1769 } 1770 1771 /* 1772 * Call the helper to output message 1773 * FIXME: what to do if core_err_cnt > 1? Currently, it generates 1774 * only one event 1775 */ 1776 if (uncorrected_error || !pvt->is_registered) 1777 edac_mc_handle_error(tp_event, mci, core_err_cnt, 1778 m->addr >> PAGE_SHIFT, 1779 m->addr & ~PAGE_MASK, 1780 syndrome, 1781 channel, dimm, -1, 1782 err, optype); 1783 } 1784 1785 /* 1786 * i7core_check_error Retrieve and process errors reported by the 1787 * hardware. Called by the Core module. 1788 */ 1789 static void i7core_check_error(struct mem_ctl_info *mci, struct mce *m) 1790 { 1791 struct i7core_pvt *pvt = mci->pvt_info; 1792 1793 i7core_mce_output_error(mci, m); 1794 1795 /* 1796 * Now, let's increment CE error counts 1797 */ 1798 if (!pvt->is_registered) 1799 i7core_udimm_check_mc_ecc_err(mci); 1800 else 1801 i7core_rdimm_check_mc_ecc_err(mci); 1802 } 1803 1804 /* 1805 * Check that logging is enabled and that this is the right type 1806 * of error for us to handle. 1807 */ 1808 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val, 1809 void *data) 1810 { 1811 struct mce *mce = (struct mce *)data; 1812 struct i7core_dev *i7_dev; 1813 struct mem_ctl_info *mci; 1814 1815 i7_dev = get_i7core_dev(mce->socketid); 1816 if (!i7_dev || (mce->kflags & MCE_HANDLED_CEC)) 1817 return NOTIFY_DONE; 1818 1819 mci = i7_dev->mci; 1820 1821 /* 1822 * Just let mcelog handle it if the error is 1823 * outside the memory controller 1824 */ 1825 if (((mce->status & 0xffff) >> 7) != 1) 1826 return NOTIFY_DONE; 1827 1828 /* Bank 8 registers are the only ones that we know how to handle */ 1829 if (mce->bank != 8) 1830 return NOTIFY_DONE; 1831 1832 i7core_check_error(mci, mce); 1833 1834 /* Advise mcelog that the errors were handled */ 1835 mce->kflags |= MCE_HANDLED_EDAC; 1836 return NOTIFY_OK; 1837 } 1838 1839 static struct notifier_block i7_mce_dec = { 1840 .notifier_call = i7core_mce_check_error, 1841 .priority = MCE_PRIO_EDAC, 1842 }; 1843 1844 struct memdev_dmi_entry { 1845 u8 type; 1846 u8 length; 1847 u16 handle; 1848 u16 phys_mem_array_handle; 1849 u16 mem_err_info_handle; 1850 u16 total_width; 1851 u16 data_width; 1852 u16 size; 1853 u8 form; 1854 u8 device_set; 1855 u8 device_locator; 1856 u8 bank_locator; 1857 u8 memory_type; 1858 u16 type_detail; 1859 u16 speed; 1860 u8 manufacturer; 1861 u8 serial_number; 1862 u8 asset_tag; 1863 u8 part_number; 1864 u8 attributes; 1865 u32 extended_size; 1866 u16 conf_mem_clk_speed; 1867 } __attribute__((__packed__)); 1868 1869 1870 /* 1871 * Decode the DRAM Clock Frequency, be paranoid, make sure that all 1872 * memory devices show the same speed, and if they don't then consider 1873 * all speeds to be invalid. 1874 */ 1875 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq) 1876 { 1877 int *dclk_freq = _dclk_freq; 1878 u16 dmi_mem_clk_speed; 1879 1880 if (*dclk_freq == -1) 1881 return; 1882 1883 if (dh->type == DMI_ENTRY_MEM_DEVICE) { 1884 struct memdev_dmi_entry *memdev_dmi_entry = 1885 (struct memdev_dmi_entry *)dh; 1886 unsigned long conf_mem_clk_speed_offset = 1887 (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed - 1888 (unsigned long)&memdev_dmi_entry->type; 1889 unsigned long speed_offset = 1890 (unsigned long)&memdev_dmi_entry->speed - 1891 (unsigned long)&memdev_dmi_entry->type; 1892 1893 /* Check that a DIMM is present */ 1894 if (memdev_dmi_entry->size == 0) 1895 return; 1896 1897 /* 1898 * Pick the configured speed if it's available, otherwise 1899 * pick the DIMM speed, or we don't have a speed. 1900 */ 1901 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) { 1902 dmi_mem_clk_speed = 1903 memdev_dmi_entry->conf_mem_clk_speed; 1904 } else if (memdev_dmi_entry->length > speed_offset) { 1905 dmi_mem_clk_speed = memdev_dmi_entry->speed; 1906 } else { 1907 *dclk_freq = -1; 1908 return; 1909 } 1910 1911 if (*dclk_freq == 0) { 1912 /* First pass, speed was 0 */ 1913 if (dmi_mem_clk_speed > 0) { 1914 /* Set speed if a valid speed is read */ 1915 *dclk_freq = dmi_mem_clk_speed; 1916 } else { 1917 /* Otherwise we don't have a valid speed */ 1918 *dclk_freq = -1; 1919 } 1920 } else if (*dclk_freq > 0 && 1921 *dclk_freq != dmi_mem_clk_speed) { 1922 /* 1923 * If we have a speed, check that all DIMMS are the same 1924 * speed, otherwise set the speed as invalid. 1925 */ 1926 *dclk_freq = -1; 1927 } 1928 } 1929 } 1930 1931 /* 1932 * The default DCLK frequency is used as a fallback if we 1933 * fail to find anything reliable in the DMI. The value 1934 * is taken straight from the datasheet. 1935 */ 1936 #define DEFAULT_DCLK_FREQ 800 1937 1938 static int get_dclk_freq(void) 1939 { 1940 int dclk_freq = 0; 1941 1942 dmi_walk(decode_dclk, (void *)&dclk_freq); 1943 1944 if (dclk_freq < 1) 1945 return DEFAULT_DCLK_FREQ; 1946 1947 return dclk_freq; 1948 } 1949 1950 /* 1951 * set_sdram_scrub_rate This routine sets byte/sec bandwidth scrub rate 1952 * to hardware according to SCRUBINTERVAL formula 1953 * found in datasheet. 1954 */ 1955 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw) 1956 { 1957 struct i7core_pvt *pvt = mci->pvt_info; 1958 struct pci_dev *pdev; 1959 u32 dw_scrub; 1960 u32 dw_ssr; 1961 1962 /* Get data from the MC register, function 2 */ 1963 pdev = pvt->pci_mcr[2]; 1964 if (!pdev) 1965 return -ENODEV; 1966 1967 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub); 1968 1969 if (new_bw == 0) { 1970 /* Prepare to disable petrol scrub */ 1971 dw_scrub &= ~STARTSCRUB; 1972 /* Stop the patrol scrub engine */ 1973 write_and_test(pdev, MC_SCRUB_CONTROL, 1974 dw_scrub & ~SCRUBINTERVAL_MASK); 1975 1976 /* Get current status of scrub rate and set bit to disable */ 1977 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr); 1978 dw_ssr &= ~SSR_MODE_MASK; 1979 dw_ssr |= SSR_MODE_DISABLE; 1980 } else { 1981 const int cache_line_size = 64; 1982 const u32 freq_dclk_mhz = pvt->dclk_freq; 1983 unsigned long long scrub_interval; 1984 /* 1985 * Translate the desired scrub rate to a register value and 1986 * program the corresponding register value. 1987 */ 1988 scrub_interval = (unsigned long long)freq_dclk_mhz * 1989 cache_line_size * 1000000; 1990 do_div(scrub_interval, new_bw); 1991 1992 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK) 1993 return -EINVAL; 1994 1995 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval; 1996 1997 /* Start the patrol scrub engine */ 1998 pci_write_config_dword(pdev, MC_SCRUB_CONTROL, 1999 STARTSCRUB | dw_scrub); 2000 2001 /* Get current status of scrub rate and set bit to enable */ 2002 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr); 2003 dw_ssr &= ~SSR_MODE_MASK; 2004 dw_ssr |= SSR_MODE_ENABLE; 2005 } 2006 /* Disable or enable scrubbing */ 2007 pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr); 2008 2009 return new_bw; 2010 } 2011 2012 /* 2013 * get_sdram_scrub_rate This routine convert current scrub rate value 2014 * into byte/sec bandwidth according to 2015 * SCRUBINTERVAL formula found in datasheet. 2016 */ 2017 static int get_sdram_scrub_rate(struct mem_ctl_info *mci) 2018 { 2019 struct i7core_pvt *pvt = mci->pvt_info; 2020 struct pci_dev *pdev; 2021 const u32 cache_line_size = 64; 2022 const u32 freq_dclk_mhz = pvt->dclk_freq; 2023 unsigned long long scrub_rate; 2024 u32 scrubval; 2025 2026 /* Get data from the MC register, function 2 */ 2027 pdev = pvt->pci_mcr[2]; 2028 if (!pdev) 2029 return -ENODEV; 2030 2031 /* Get current scrub control data */ 2032 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval); 2033 2034 /* Mask highest 8-bits to 0 */ 2035 scrubval &= SCRUBINTERVAL_MASK; 2036 if (!scrubval) 2037 return 0; 2038 2039 /* Calculate scrub rate value into byte/sec bandwidth */ 2040 scrub_rate = (unsigned long long)freq_dclk_mhz * 2041 1000000 * cache_line_size; 2042 do_div(scrub_rate, scrubval); 2043 return (int)scrub_rate; 2044 } 2045 2046 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci) 2047 { 2048 struct i7core_pvt *pvt = mci->pvt_info; 2049 u32 pci_lock; 2050 2051 /* Unlock writes to pci registers */ 2052 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock); 2053 pci_lock &= ~0x3; 2054 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, 2055 pci_lock | MC_CFG_UNLOCK); 2056 2057 mci->set_sdram_scrub_rate = set_sdram_scrub_rate; 2058 mci->get_sdram_scrub_rate = get_sdram_scrub_rate; 2059 } 2060 2061 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci) 2062 { 2063 struct i7core_pvt *pvt = mci->pvt_info; 2064 u32 pci_lock; 2065 2066 /* Lock writes to pci registers */ 2067 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock); 2068 pci_lock &= ~0x3; 2069 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, 2070 pci_lock | MC_CFG_LOCK); 2071 } 2072 2073 static void i7core_pci_ctl_create(struct i7core_pvt *pvt) 2074 { 2075 pvt->i7core_pci = edac_pci_create_generic_ctl( 2076 &pvt->i7core_dev->pdev[0]->dev, 2077 EDAC_MOD_STR); 2078 if (unlikely(!pvt->i7core_pci)) 2079 i7core_printk(KERN_WARNING, 2080 "Unable to setup PCI error report via EDAC\n"); 2081 } 2082 2083 static void i7core_pci_ctl_release(struct i7core_pvt *pvt) 2084 { 2085 if (likely(pvt->i7core_pci)) 2086 edac_pci_release_generic_ctl(pvt->i7core_pci); 2087 else 2088 i7core_printk(KERN_ERR, 2089 "Couldn't find mem_ctl_info for socket %d\n", 2090 pvt->i7core_dev->socket); 2091 pvt->i7core_pci = NULL; 2092 } 2093 2094 static void i7core_unregister_mci(struct i7core_dev *i7core_dev) 2095 { 2096 struct mem_ctl_info *mci = i7core_dev->mci; 2097 struct i7core_pvt *pvt; 2098 2099 if (unlikely(!mci || !mci->pvt_info)) { 2100 edac_dbg(0, "MC: dev = %p\n", &i7core_dev->pdev[0]->dev); 2101 2102 i7core_printk(KERN_ERR, "Couldn't find mci handler\n"); 2103 return; 2104 } 2105 2106 pvt = mci->pvt_info; 2107 2108 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev); 2109 2110 /* Disable scrubrate setting */ 2111 if (pvt->enable_scrub) 2112 disable_sdram_scrub_setting(mci); 2113 2114 /* Disable EDAC polling */ 2115 i7core_pci_ctl_release(pvt); 2116 2117 /* Remove MC sysfs nodes */ 2118 i7core_delete_sysfs_devices(mci); 2119 edac_mc_del_mc(mci->pdev); 2120 2121 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 2122 kfree(mci->ctl_name); 2123 edac_mc_free(mci); 2124 i7core_dev->mci = NULL; 2125 } 2126 2127 static int i7core_register_mci(struct i7core_dev *i7core_dev) 2128 { 2129 struct mem_ctl_info *mci; 2130 struct i7core_pvt *pvt; 2131 int rc; 2132 struct edac_mc_layer layers[2]; 2133 2134 /* allocate a new MC control structure */ 2135 2136 layers[0].type = EDAC_MC_LAYER_CHANNEL; 2137 layers[0].size = NUM_CHANS; 2138 layers[0].is_virt_csrow = false; 2139 layers[1].type = EDAC_MC_LAYER_SLOT; 2140 layers[1].size = MAX_DIMMS; 2141 layers[1].is_virt_csrow = true; 2142 mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers, 2143 sizeof(*pvt)); 2144 if (unlikely(!mci)) 2145 return -ENOMEM; 2146 2147 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev); 2148 2149 pvt = mci->pvt_info; 2150 memset(pvt, 0, sizeof(*pvt)); 2151 2152 /* Associates i7core_dev and mci for future usage */ 2153 pvt->i7core_dev = i7core_dev; 2154 i7core_dev->mci = mci; 2155 2156 /* 2157 * FIXME: how to handle RDDR3 at MCI level? It is possible to have 2158 * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different 2159 * memory channels 2160 */ 2161 mci->mtype_cap = MEM_FLAG_DDR3; 2162 mci->edac_ctl_cap = EDAC_FLAG_NONE; 2163 mci->edac_cap = EDAC_FLAG_NONE; 2164 mci->mod_name = "i7core_edac.c"; 2165 2166 mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d", i7core_dev->socket); 2167 if (!mci->ctl_name) { 2168 rc = -ENOMEM; 2169 goto fail1; 2170 } 2171 2172 mci->dev_name = pci_name(i7core_dev->pdev[0]); 2173 mci->ctl_page_to_phys = NULL; 2174 2175 /* Store pci devices at mci for faster access */ 2176 rc = mci_bind_devs(mci, i7core_dev); 2177 if (unlikely(rc < 0)) 2178 goto fail0; 2179 2180 2181 /* Get dimm basic config */ 2182 get_dimm_config(mci); 2183 /* record ptr to the generic device */ 2184 mci->pdev = &i7core_dev->pdev[0]->dev; 2185 2186 /* Enable scrubrate setting */ 2187 if (pvt->enable_scrub) 2188 enable_sdram_scrub_setting(mci); 2189 2190 /* add this new MC control structure to EDAC's list of MCs */ 2191 if (unlikely(edac_mc_add_mc_with_groups(mci, i7core_dev_groups))) { 2192 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 2193 /* FIXME: perhaps some code should go here that disables error 2194 * reporting if we just enabled it 2195 */ 2196 2197 rc = -EINVAL; 2198 goto fail0; 2199 } 2200 if (i7core_create_sysfs_devices(mci)) { 2201 edac_dbg(0, "MC: failed to create sysfs nodes\n"); 2202 edac_mc_del_mc(mci->pdev); 2203 rc = -EINVAL; 2204 goto fail0; 2205 } 2206 2207 /* Default error mask is any memory */ 2208 pvt->inject.channel = 0; 2209 pvt->inject.dimm = -1; 2210 pvt->inject.rank = -1; 2211 pvt->inject.bank = -1; 2212 pvt->inject.page = -1; 2213 pvt->inject.col = -1; 2214 2215 /* allocating generic PCI control info */ 2216 i7core_pci_ctl_create(pvt); 2217 2218 /* DCLK for scrub rate setting */ 2219 pvt->dclk_freq = get_dclk_freq(); 2220 2221 return 0; 2222 2223 fail0: 2224 kfree(mci->ctl_name); 2225 2226 fail1: 2227 edac_mc_free(mci); 2228 i7core_dev->mci = NULL; 2229 return rc; 2230 } 2231 2232 /* 2233 * i7core_probe Probe for ONE instance of device to see if it is 2234 * present. 2235 * return: 2236 * 0 for FOUND a device 2237 * < 0 for error code 2238 */ 2239 2240 static int i7core_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2241 { 2242 int rc, count = 0; 2243 struct i7core_dev *i7core_dev; 2244 2245 /* get the pci devices we want to reserve for our use */ 2246 mutex_lock(&i7core_edac_lock); 2247 2248 /* 2249 * All memory controllers are allocated at the first pass. 2250 */ 2251 if (unlikely(probed >= 1)) { 2252 mutex_unlock(&i7core_edac_lock); 2253 return -ENODEV; 2254 } 2255 probed++; 2256 2257 rc = i7core_get_all_devices(); 2258 if (unlikely(rc < 0)) 2259 goto fail0; 2260 2261 list_for_each_entry(i7core_dev, &i7core_edac_list, list) { 2262 count++; 2263 rc = i7core_register_mci(i7core_dev); 2264 if (unlikely(rc < 0)) 2265 goto fail1; 2266 } 2267 2268 /* 2269 * Nehalem-EX uses a different memory controller. However, as the 2270 * memory controller is not visible on some Nehalem/Nehalem-EP, we 2271 * need to indirectly probe via a X58 PCI device. The same devices 2272 * are found on (some) Nehalem-EX. So, on those machines, the 2273 * probe routine needs to return -ENODEV, as the actual Memory 2274 * Controller registers won't be detected. 2275 */ 2276 if (!count) { 2277 rc = -ENODEV; 2278 goto fail1; 2279 } 2280 2281 i7core_printk(KERN_INFO, 2282 "Driver loaded, %d memory controller(s) found.\n", 2283 count); 2284 2285 mutex_unlock(&i7core_edac_lock); 2286 return 0; 2287 2288 fail1: 2289 list_for_each_entry(i7core_dev, &i7core_edac_list, list) 2290 i7core_unregister_mci(i7core_dev); 2291 2292 i7core_put_all_devices(); 2293 fail0: 2294 mutex_unlock(&i7core_edac_lock); 2295 return rc; 2296 } 2297 2298 /* 2299 * i7core_remove destructor for one instance of device 2300 * 2301 */ 2302 static void i7core_remove(struct pci_dev *pdev) 2303 { 2304 struct i7core_dev *i7core_dev; 2305 2306 edac_dbg(0, "\n"); 2307 2308 /* 2309 * we have a trouble here: pdev value for removal will be wrong, since 2310 * it will point to the X58 register used to detect that the machine 2311 * is a Nehalem or upper design. However, due to the way several PCI 2312 * devices are grouped together to provide MC functionality, we need 2313 * to use a different method for releasing the devices 2314 */ 2315 2316 mutex_lock(&i7core_edac_lock); 2317 2318 if (unlikely(!probed)) { 2319 mutex_unlock(&i7core_edac_lock); 2320 return; 2321 } 2322 2323 list_for_each_entry(i7core_dev, &i7core_edac_list, list) 2324 i7core_unregister_mci(i7core_dev); 2325 2326 /* Release PCI resources */ 2327 i7core_put_all_devices(); 2328 2329 probed--; 2330 2331 mutex_unlock(&i7core_edac_lock); 2332 } 2333 2334 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl); 2335 2336 /* 2337 * i7core_driver pci_driver structure for this module 2338 * 2339 */ 2340 static struct pci_driver i7core_driver = { 2341 .name = "i7core_edac", 2342 .probe = i7core_probe, 2343 .remove = i7core_remove, 2344 .id_table = i7core_pci_tbl, 2345 }; 2346 2347 /* 2348 * i7core_init Module entry function 2349 * Try to initialize this module for its devices 2350 */ 2351 static int __init i7core_init(void) 2352 { 2353 int pci_rc; 2354 2355 edac_dbg(2, "\n"); 2356 2357 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 2358 opstate_init(); 2359 2360 if (use_pci_fixup) 2361 i7core_xeon_pci_fixup(pci_dev_table); 2362 2363 pci_rc = pci_register_driver(&i7core_driver); 2364 2365 if (pci_rc >= 0) { 2366 mce_register_decode_chain(&i7_mce_dec); 2367 return 0; 2368 } 2369 2370 i7core_printk(KERN_ERR, "Failed to register device with error %d.\n", 2371 pci_rc); 2372 2373 return pci_rc; 2374 } 2375 2376 /* 2377 * i7core_exit() Module exit function 2378 * Unregister the driver 2379 */ 2380 static void __exit i7core_exit(void) 2381 { 2382 edac_dbg(2, "\n"); 2383 pci_unregister_driver(&i7core_driver); 2384 mce_unregister_decode_chain(&i7_mce_dec); 2385 } 2386 2387 module_init(i7core_init); 2388 module_exit(i7core_exit); 2389 2390 MODULE_LICENSE("GPL"); 2391 MODULE_AUTHOR("Mauro Carvalho Chehab"); 2392 MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)"); 2393 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - " 2394 I7CORE_REVISION); 2395 2396 module_param(edac_op_state, int, 0444); 2397 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2398