1 /* Intel i7 core/Nehalem Memory Controller kernel module 2 * 3 * This driver supports the memory controllers found on the Intel 4 * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx, 5 * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield 6 * and Westmere-EP. 7 * 8 * This file may be distributed under the terms of the 9 * GNU General Public License version 2 only. 10 * 11 * Copyright (c) 2009-2010 by: 12 * Mauro Carvalho Chehab 13 * 14 * Red Hat Inc. http://www.redhat.com 15 * 16 * Forked and adapted from the i5400_edac driver 17 * 18 * Based on the following public Intel datasheets: 19 * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor 20 * Datasheet, Volume 2: 21 * http://download.intel.com/design/processor/datashts/320835.pdf 22 * Intel Xeon Processor 5500 Series Datasheet Volume 2 23 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf 24 * also available at: 25 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 26 */ 27 28 #include <linux/module.h> 29 #include <linux/init.h> 30 #include <linux/pci.h> 31 #include <linux/pci_ids.h> 32 #include <linux/slab.h> 33 #include <linux/delay.h> 34 #include <linux/dmi.h> 35 #include <linux/edac.h> 36 #include <linux/mmzone.h> 37 #include <linux/smp.h> 38 #include <asm/mce.h> 39 #include <asm/processor.h> 40 #include <asm/div64.h> 41 42 #include "edac_core.h" 43 44 /* Static vars */ 45 static LIST_HEAD(i7core_edac_list); 46 static DEFINE_MUTEX(i7core_edac_lock); 47 static int probed; 48 49 static int use_pci_fixup; 50 module_param(use_pci_fixup, int, 0444); 51 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices"); 52 /* 53 * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core 54 * registers start at bus 255, and are not reported by BIOS. 55 * We currently find devices with only 2 sockets. In order to support more QPI 56 * Quick Path Interconnect, just increment this number. 57 */ 58 #define MAX_SOCKET_BUSES 2 59 60 61 /* 62 * Alter this version for the module when modifications are made 63 */ 64 #define I7CORE_REVISION " Ver: 1.0.0" 65 #define EDAC_MOD_STR "i7core_edac" 66 67 /* 68 * Debug macros 69 */ 70 #define i7core_printk(level, fmt, arg...) \ 71 edac_printk(level, "i7core", fmt, ##arg) 72 73 #define i7core_mc_printk(mci, level, fmt, arg...) \ 74 edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg) 75 76 /* 77 * i7core Memory Controller Registers 78 */ 79 80 /* OFFSETS for Device 0 Function 0 */ 81 82 #define MC_CFG_CONTROL 0x90 83 #define MC_CFG_UNLOCK 0x02 84 #define MC_CFG_LOCK 0x00 85 86 /* OFFSETS for Device 3 Function 0 */ 87 88 #define MC_CONTROL 0x48 89 #define MC_STATUS 0x4c 90 #define MC_MAX_DOD 0x64 91 92 /* 93 * OFFSETS for Device 3 Function 4, as indicated on Xeon 5500 datasheet: 94 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 95 */ 96 97 #define MC_TEST_ERR_RCV1 0x60 98 #define DIMM2_COR_ERR(r) ((r) & 0x7fff) 99 100 #define MC_TEST_ERR_RCV0 0x64 101 #define DIMM1_COR_ERR(r) (((r) >> 16) & 0x7fff) 102 #define DIMM0_COR_ERR(r) ((r) & 0x7fff) 103 104 /* OFFSETS for Device 3 Function 2, as indicated on Xeon 5500 datasheet */ 105 #define MC_SSRCONTROL 0x48 106 #define SSR_MODE_DISABLE 0x00 107 #define SSR_MODE_ENABLE 0x01 108 #define SSR_MODE_MASK 0x03 109 110 #define MC_SCRUB_CONTROL 0x4c 111 #define STARTSCRUB (1 << 24) 112 #define SCRUBINTERVAL_MASK 0xffffff 113 114 #define MC_COR_ECC_CNT_0 0x80 115 #define MC_COR_ECC_CNT_1 0x84 116 #define MC_COR_ECC_CNT_2 0x88 117 #define MC_COR_ECC_CNT_3 0x8c 118 #define MC_COR_ECC_CNT_4 0x90 119 #define MC_COR_ECC_CNT_5 0x94 120 121 #define DIMM_TOP_COR_ERR(r) (((r) >> 16) & 0x7fff) 122 #define DIMM_BOT_COR_ERR(r) ((r) & 0x7fff) 123 124 125 /* OFFSETS for Devices 4,5 and 6 Function 0 */ 126 127 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58 128 #define THREE_DIMMS_PRESENT (1 << 24) 129 #define SINGLE_QUAD_RANK_PRESENT (1 << 23) 130 #define QUAD_RANK_PRESENT (1 << 22) 131 #define REGISTERED_DIMM (1 << 15) 132 133 #define MC_CHANNEL_MAPPER 0x60 134 #define RDLCH(r, ch) ((((r) >> (3 + (ch * 6))) & 0x07) - 1) 135 #define WRLCH(r, ch) ((((r) >> (ch * 6)) & 0x07) - 1) 136 137 #define MC_CHANNEL_RANK_PRESENT 0x7c 138 #define RANK_PRESENT_MASK 0xffff 139 140 #define MC_CHANNEL_ADDR_MATCH 0xf0 141 #define MC_CHANNEL_ERROR_MASK 0xf8 142 #define MC_CHANNEL_ERROR_INJECT 0xfc 143 #define INJECT_ADDR_PARITY 0x10 144 #define INJECT_ECC 0x08 145 #define MASK_CACHELINE 0x06 146 #define MASK_FULL_CACHELINE 0x06 147 #define MASK_MSB32_CACHELINE 0x04 148 #define MASK_LSB32_CACHELINE 0x02 149 #define NO_MASK_CACHELINE 0x00 150 #define REPEAT_EN 0x01 151 152 /* OFFSETS for Devices 4,5 and 6 Function 1 */ 153 154 #define MC_DOD_CH_DIMM0 0x48 155 #define MC_DOD_CH_DIMM1 0x4c 156 #define MC_DOD_CH_DIMM2 0x50 157 #define RANKOFFSET_MASK ((1 << 12) | (1 << 11) | (1 << 10)) 158 #define RANKOFFSET(x) ((x & RANKOFFSET_MASK) >> 10) 159 #define DIMM_PRESENT_MASK (1 << 9) 160 #define DIMM_PRESENT(x) (((x) & DIMM_PRESENT_MASK) >> 9) 161 #define MC_DOD_NUMBANK_MASK ((1 << 8) | (1 << 7)) 162 #define MC_DOD_NUMBANK(x) (((x) & MC_DOD_NUMBANK_MASK) >> 7) 163 #define MC_DOD_NUMRANK_MASK ((1 << 6) | (1 << 5)) 164 #define MC_DOD_NUMRANK(x) (((x) & MC_DOD_NUMRANK_MASK) >> 5) 165 #define MC_DOD_NUMROW_MASK ((1 << 4) | (1 << 3) | (1 << 2)) 166 #define MC_DOD_NUMROW(x) (((x) & MC_DOD_NUMROW_MASK) >> 2) 167 #define MC_DOD_NUMCOL_MASK 3 168 #define MC_DOD_NUMCOL(x) ((x) & MC_DOD_NUMCOL_MASK) 169 170 #define MC_RANK_PRESENT 0x7c 171 172 #define MC_SAG_CH_0 0x80 173 #define MC_SAG_CH_1 0x84 174 #define MC_SAG_CH_2 0x88 175 #define MC_SAG_CH_3 0x8c 176 #define MC_SAG_CH_4 0x90 177 #define MC_SAG_CH_5 0x94 178 #define MC_SAG_CH_6 0x98 179 #define MC_SAG_CH_7 0x9c 180 181 #define MC_RIR_LIMIT_CH_0 0x40 182 #define MC_RIR_LIMIT_CH_1 0x44 183 #define MC_RIR_LIMIT_CH_2 0x48 184 #define MC_RIR_LIMIT_CH_3 0x4C 185 #define MC_RIR_LIMIT_CH_4 0x50 186 #define MC_RIR_LIMIT_CH_5 0x54 187 #define MC_RIR_LIMIT_CH_6 0x58 188 #define MC_RIR_LIMIT_CH_7 0x5C 189 #define MC_RIR_LIMIT_MASK ((1 << 10) - 1) 190 191 #define MC_RIR_WAY_CH 0x80 192 #define MC_RIR_WAY_OFFSET_MASK (((1 << 14) - 1) & ~0x7) 193 #define MC_RIR_WAY_RANK_MASK 0x7 194 195 /* 196 * i7core structs 197 */ 198 199 #define NUM_CHANS 3 200 #define MAX_DIMMS 3 /* Max DIMMS per channel */ 201 #define MAX_MCR_FUNC 4 202 #define MAX_CHAN_FUNC 3 203 204 struct i7core_info { 205 u32 mc_control; 206 u32 mc_status; 207 u32 max_dod; 208 u32 ch_map; 209 }; 210 211 212 struct i7core_inject { 213 int enable; 214 215 u32 section; 216 u32 type; 217 u32 eccmask; 218 219 /* Error address mask */ 220 int channel, dimm, rank, bank, page, col; 221 }; 222 223 struct i7core_channel { 224 bool is_3dimms_present; 225 bool is_single_4rank; 226 bool has_4rank; 227 u32 dimms; 228 }; 229 230 struct pci_id_descr { 231 int dev; 232 int func; 233 int dev_id; 234 int optional; 235 }; 236 237 struct pci_id_table { 238 const struct pci_id_descr *descr; 239 int n_devs; 240 }; 241 242 struct i7core_dev { 243 struct list_head list; 244 u8 socket; 245 struct pci_dev **pdev; 246 int n_devs; 247 struct mem_ctl_info *mci; 248 }; 249 250 struct i7core_pvt { 251 struct device *addrmatch_dev, *chancounts_dev; 252 253 struct pci_dev *pci_noncore; 254 struct pci_dev *pci_mcr[MAX_MCR_FUNC + 1]; 255 struct pci_dev *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1]; 256 257 struct i7core_dev *i7core_dev; 258 259 struct i7core_info info; 260 struct i7core_inject inject; 261 struct i7core_channel channel[NUM_CHANS]; 262 263 int ce_count_available; 264 265 /* ECC corrected errors counts per udimm */ 266 unsigned long udimm_ce_count[MAX_DIMMS]; 267 int udimm_last_ce_count[MAX_DIMMS]; 268 /* ECC corrected errors counts per rdimm */ 269 unsigned long rdimm_ce_count[NUM_CHANS][MAX_DIMMS]; 270 int rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS]; 271 272 bool is_registered, enable_scrub; 273 274 /* Fifo double buffers */ 275 struct mce mce_entry[MCE_LOG_LEN]; 276 struct mce mce_outentry[MCE_LOG_LEN]; 277 278 /* Fifo in/out counters */ 279 unsigned mce_in, mce_out; 280 281 /* Count indicator to show errors not got */ 282 unsigned mce_overrun; 283 284 /* DCLK Frequency used for computing scrub rate */ 285 int dclk_freq; 286 287 /* Struct to control EDAC polling */ 288 struct edac_pci_ctl_info *i7core_pci; 289 }; 290 291 #define PCI_DESCR(device, function, device_id) \ 292 .dev = (device), \ 293 .func = (function), \ 294 .dev_id = (device_id) 295 296 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = { 297 /* Memory controller */ 298 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR) }, 299 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD) }, 300 /* Exists only for RDIMM */ 301 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1 }, 302 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) }, 303 304 /* Channel 0 */ 305 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) }, 306 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) }, 307 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) }, 308 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC) }, 309 310 /* Channel 1 */ 311 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) }, 312 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) }, 313 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) }, 314 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC) }, 315 316 /* Channel 2 */ 317 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) }, 318 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) }, 319 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) }, 320 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC) }, 321 322 /* Generic Non-core registers */ 323 /* 324 * This is the PCI device on i7core and on Xeon 35xx (8086:2c41) 325 * On Xeon 55xx, however, it has a different id (8086:2c40). So, 326 * the probing code needs to test for the other address in case of 327 * failure of this one 328 */ 329 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE) }, 330 331 }; 332 333 static const struct pci_id_descr pci_dev_descr_lynnfield[] = { 334 { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR) }, 335 { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD) }, 336 { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST) }, 337 338 { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) }, 339 { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) }, 340 { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) }, 341 { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC) }, 342 343 { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) }, 344 { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) }, 345 { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) }, 346 { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC) }, 347 348 /* 349 * This is the PCI device has an alternate address on some 350 * processors like Core i7 860 351 */ 352 { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE) }, 353 }; 354 355 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = { 356 /* Memory controller */ 357 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2) }, 358 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2) }, 359 /* Exists only for RDIMM */ 360 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1 }, 361 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) }, 362 363 /* Channel 0 */ 364 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) }, 365 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) }, 366 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) }, 367 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2) }, 368 369 /* Channel 1 */ 370 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) }, 371 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) }, 372 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) }, 373 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2) }, 374 375 /* Channel 2 */ 376 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) }, 377 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) }, 378 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) }, 379 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2) }, 380 381 /* Generic Non-core registers */ 382 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2) }, 383 384 }; 385 386 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) } 387 static const struct pci_id_table pci_dev_table[] = { 388 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem), 389 PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield), 390 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere), 391 {0,} /* 0 terminated list. */ 392 }; 393 394 /* 395 * pci_device_id table for which devices we are looking for 396 */ 397 static const struct pci_device_id i7core_pci_tbl[] = { 398 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)}, 399 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)}, 400 {0,} /* 0 terminated list. */ 401 }; 402 403 /**************************************************************************** 404 Ancillary status routines 405 ****************************************************************************/ 406 407 /* MC_CONTROL bits */ 408 #define CH_ACTIVE(pvt, ch) ((pvt)->info.mc_control & (1 << (8 + ch))) 409 #define ECCx8(pvt) ((pvt)->info.mc_control & (1 << 1)) 410 411 /* MC_STATUS bits */ 412 #define ECC_ENABLED(pvt) ((pvt)->info.mc_status & (1 << 4)) 413 #define CH_DISABLED(pvt, ch) ((pvt)->info.mc_status & (1 << ch)) 414 415 /* MC_MAX_DOD read functions */ 416 static inline int numdimms(u32 dimms) 417 { 418 return (dimms & 0x3) + 1; 419 } 420 421 static inline int numrank(u32 rank) 422 { 423 static const int ranks[] = { 1, 2, 4, -EINVAL }; 424 425 return ranks[rank & 0x3]; 426 } 427 428 static inline int numbank(u32 bank) 429 { 430 static const int banks[] = { 4, 8, 16, -EINVAL }; 431 432 return banks[bank & 0x3]; 433 } 434 435 static inline int numrow(u32 row) 436 { 437 static const int rows[] = { 438 1 << 12, 1 << 13, 1 << 14, 1 << 15, 439 1 << 16, -EINVAL, -EINVAL, -EINVAL, 440 }; 441 442 return rows[row & 0x7]; 443 } 444 445 static inline int numcol(u32 col) 446 { 447 static const int cols[] = { 448 1 << 10, 1 << 11, 1 << 12, -EINVAL, 449 }; 450 return cols[col & 0x3]; 451 } 452 453 static struct i7core_dev *get_i7core_dev(u8 socket) 454 { 455 struct i7core_dev *i7core_dev; 456 457 list_for_each_entry(i7core_dev, &i7core_edac_list, list) { 458 if (i7core_dev->socket == socket) 459 return i7core_dev; 460 } 461 462 return NULL; 463 } 464 465 static struct i7core_dev *alloc_i7core_dev(u8 socket, 466 const struct pci_id_table *table) 467 { 468 struct i7core_dev *i7core_dev; 469 470 i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL); 471 if (!i7core_dev) 472 return NULL; 473 474 i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs, 475 GFP_KERNEL); 476 if (!i7core_dev->pdev) { 477 kfree(i7core_dev); 478 return NULL; 479 } 480 481 i7core_dev->socket = socket; 482 i7core_dev->n_devs = table->n_devs; 483 list_add_tail(&i7core_dev->list, &i7core_edac_list); 484 485 return i7core_dev; 486 } 487 488 static void free_i7core_dev(struct i7core_dev *i7core_dev) 489 { 490 list_del(&i7core_dev->list); 491 kfree(i7core_dev->pdev); 492 kfree(i7core_dev); 493 } 494 495 /**************************************************************************** 496 Memory check routines 497 ****************************************************************************/ 498 499 static int get_dimm_config(struct mem_ctl_info *mci) 500 { 501 struct i7core_pvt *pvt = mci->pvt_info; 502 struct pci_dev *pdev; 503 int i, j; 504 enum edac_type mode; 505 enum mem_type mtype; 506 struct dimm_info *dimm; 507 508 /* Get data from the MC register, function 0 */ 509 pdev = pvt->pci_mcr[0]; 510 if (!pdev) 511 return -ENODEV; 512 513 /* Device 3 function 0 reads */ 514 pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control); 515 pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status); 516 pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod); 517 pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map); 518 519 edac_dbg(0, "QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n", 520 pvt->i7core_dev->socket, pvt->info.mc_control, 521 pvt->info.mc_status, pvt->info.max_dod, pvt->info.ch_map); 522 523 if (ECC_ENABLED(pvt)) { 524 edac_dbg(0, "ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4); 525 if (ECCx8(pvt)) 526 mode = EDAC_S8ECD8ED; 527 else 528 mode = EDAC_S4ECD4ED; 529 } else { 530 edac_dbg(0, "ECC disabled\n"); 531 mode = EDAC_NONE; 532 } 533 534 /* FIXME: need to handle the error codes */ 535 edac_dbg(0, "DOD Max limits: DIMMS: %d, %d-ranked, %d-banked x%x x 0x%x\n", 536 numdimms(pvt->info.max_dod), 537 numrank(pvt->info.max_dod >> 2), 538 numbank(pvt->info.max_dod >> 4), 539 numrow(pvt->info.max_dod >> 6), 540 numcol(pvt->info.max_dod >> 9)); 541 542 for (i = 0; i < NUM_CHANS; i++) { 543 u32 data, dimm_dod[3], value[8]; 544 545 if (!pvt->pci_ch[i][0]) 546 continue; 547 548 if (!CH_ACTIVE(pvt, i)) { 549 edac_dbg(0, "Channel %i is not active\n", i); 550 continue; 551 } 552 if (CH_DISABLED(pvt, i)) { 553 edac_dbg(0, "Channel %i is disabled\n", i); 554 continue; 555 } 556 557 /* Devices 4-6 function 0 */ 558 pci_read_config_dword(pvt->pci_ch[i][0], 559 MC_CHANNEL_DIMM_INIT_PARAMS, &data); 560 561 562 if (data & THREE_DIMMS_PRESENT) 563 pvt->channel[i].is_3dimms_present = true; 564 565 if (data & SINGLE_QUAD_RANK_PRESENT) 566 pvt->channel[i].is_single_4rank = true; 567 568 if (data & QUAD_RANK_PRESENT) 569 pvt->channel[i].has_4rank = true; 570 571 if (data & REGISTERED_DIMM) 572 mtype = MEM_RDDR3; 573 else 574 mtype = MEM_DDR3; 575 576 /* Devices 4-6 function 1 */ 577 pci_read_config_dword(pvt->pci_ch[i][1], 578 MC_DOD_CH_DIMM0, &dimm_dod[0]); 579 pci_read_config_dword(pvt->pci_ch[i][1], 580 MC_DOD_CH_DIMM1, &dimm_dod[1]); 581 pci_read_config_dword(pvt->pci_ch[i][1], 582 MC_DOD_CH_DIMM2, &dimm_dod[2]); 583 584 edac_dbg(0, "Ch%d phy rd%d, wr%d (0x%08x): %s%s%s%cDIMMs\n", 585 i, 586 RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i), 587 data, 588 pvt->channel[i].is_3dimms_present ? "3DIMMS " : "", 589 pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "", 590 pvt->channel[i].has_4rank ? "HAS_4R " : "", 591 (data & REGISTERED_DIMM) ? 'R' : 'U'); 592 593 for (j = 0; j < 3; j++) { 594 u32 banks, ranks, rows, cols; 595 u32 size, npages; 596 597 if (!DIMM_PRESENT(dimm_dod[j])) 598 continue; 599 600 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, 601 i, j, 0); 602 banks = numbank(MC_DOD_NUMBANK(dimm_dod[j])); 603 ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j])); 604 rows = numrow(MC_DOD_NUMROW(dimm_dod[j])); 605 cols = numcol(MC_DOD_NUMCOL(dimm_dod[j])); 606 607 /* DDR3 has 8 I/O banks */ 608 size = (rows * cols * banks * ranks) >> (20 - 3); 609 610 edac_dbg(0, "\tdimm %d %d Mb offset: %x, bank: %d, rank: %d, row: %#x, col: %#x\n", 611 j, size, 612 RANKOFFSET(dimm_dod[j]), 613 banks, ranks, rows, cols); 614 615 npages = MiB_TO_PAGES(size); 616 617 dimm->nr_pages = npages; 618 619 switch (banks) { 620 case 4: 621 dimm->dtype = DEV_X4; 622 break; 623 case 8: 624 dimm->dtype = DEV_X8; 625 break; 626 case 16: 627 dimm->dtype = DEV_X16; 628 break; 629 default: 630 dimm->dtype = DEV_UNKNOWN; 631 } 632 633 snprintf(dimm->label, sizeof(dimm->label), 634 "CPU#%uChannel#%u_DIMM#%u", 635 pvt->i7core_dev->socket, i, j); 636 dimm->grain = 8; 637 dimm->edac_mode = mode; 638 dimm->mtype = mtype; 639 } 640 641 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]); 642 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]); 643 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]); 644 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]); 645 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]); 646 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]); 647 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]); 648 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]); 649 edac_dbg(1, "\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i); 650 for (j = 0; j < 8; j++) 651 edac_dbg(1, "\t\t%#x\t%#x\t%#x\n", 652 (value[j] >> 27) & 0x1, 653 (value[j] >> 24) & 0x7, 654 (value[j] & ((1 << 24) - 1))); 655 } 656 657 return 0; 658 } 659 660 /**************************************************************************** 661 Error insertion routines 662 ****************************************************************************/ 663 664 #define to_mci(k) container_of(k, struct mem_ctl_info, dev) 665 666 /* The i7core has independent error injection features per channel. 667 However, to have a simpler code, we don't allow enabling error injection 668 on more than one channel. 669 Also, since a change at an inject parameter will be applied only at enable, 670 we're disabling error injection on all write calls to the sysfs nodes that 671 controls the error code injection. 672 */ 673 static int disable_inject(const struct mem_ctl_info *mci) 674 { 675 struct i7core_pvt *pvt = mci->pvt_info; 676 677 pvt->inject.enable = 0; 678 679 if (!pvt->pci_ch[pvt->inject.channel][0]) 680 return -ENODEV; 681 682 pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0], 683 MC_CHANNEL_ERROR_INJECT, 0); 684 685 return 0; 686 } 687 688 /* 689 * i7core inject inject.section 690 * 691 * accept and store error injection inject.section value 692 * bit 0 - refers to the lower 32-byte half cacheline 693 * bit 1 - refers to the upper 32-byte half cacheline 694 */ 695 static ssize_t i7core_inject_section_store(struct device *dev, 696 struct device_attribute *mattr, 697 const char *data, size_t count) 698 { 699 struct mem_ctl_info *mci = to_mci(dev); 700 struct i7core_pvt *pvt = mci->pvt_info; 701 unsigned long value; 702 int rc; 703 704 if (pvt->inject.enable) 705 disable_inject(mci); 706 707 rc = kstrtoul(data, 10, &value); 708 if ((rc < 0) || (value > 3)) 709 return -EIO; 710 711 pvt->inject.section = (u32) value; 712 return count; 713 } 714 715 static ssize_t i7core_inject_section_show(struct device *dev, 716 struct device_attribute *mattr, 717 char *data) 718 { 719 struct mem_ctl_info *mci = to_mci(dev); 720 struct i7core_pvt *pvt = mci->pvt_info; 721 return sprintf(data, "0x%08x\n", pvt->inject.section); 722 } 723 724 /* 725 * i7core inject.type 726 * 727 * accept and store error injection inject.section value 728 * bit 0 - repeat enable - Enable error repetition 729 * bit 1 - inject ECC error 730 * bit 2 - inject parity error 731 */ 732 static ssize_t i7core_inject_type_store(struct device *dev, 733 struct device_attribute *mattr, 734 const char *data, size_t count) 735 { 736 struct mem_ctl_info *mci = to_mci(dev); 737 struct i7core_pvt *pvt = mci->pvt_info; 738 unsigned long value; 739 int rc; 740 741 if (pvt->inject.enable) 742 disable_inject(mci); 743 744 rc = kstrtoul(data, 10, &value); 745 if ((rc < 0) || (value > 7)) 746 return -EIO; 747 748 pvt->inject.type = (u32) value; 749 return count; 750 } 751 752 static ssize_t i7core_inject_type_show(struct device *dev, 753 struct device_attribute *mattr, 754 char *data) 755 { 756 struct mem_ctl_info *mci = to_mci(dev); 757 struct i7core_pvt *pvt = mci->pvt_info; 758 759 return sprintf(data, "0x%08x\n", pvt->inject.type); 760 } 761 762 /* 763 * i7core_inject_inject.eccmask_store 764 * 765 * The type of error (UE/CE) will depend on the inject.eccmask value: 766 * Any bits set to a 1 will flip the corresponding ECC bit 767 * Correctable errors can be injected by flipping 1 bit or the bits within 768 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or 769 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an 770 * uncorrectable error to be injected. 771 */ 772 static ssize_t i7core_inject_eccmask_store(struct device *dev, 773 struct device_attribute *mattr, 774 const char *data, size_t count) 775 { 776 struct mem_ctl_info *mci = to_mci(dev); 777 struct i7core_pvt *pvt = mci->pvt_info; 778 unsigned long value; 779 int rc; 780 781 if (pvt->inject.enable) 782 disable_inject(mci); 783 784 rc = kstrtoul(data, 10, &value); 785 if (rc < 0) 786 return -EIO; 787 788 pvt->inject.eccmask = (u32) value; 789 return count; 790 } 791 792 static ssize_t i7core_inject_eccmask_show(struct device *dev, 793 struct device_attribute *mattr, 794 char *data) 795 { 796 struct mem_ctl_info *mci = to_mci(dev); 797 struct i7core_pvt *pvt = mci->pvt_info; 798 799 return sprintf(data, "0x%08x\n", pvt->inject.eccmask); 800 } 801 802 /* 803 * i7core_addrmatch 804 * 805 * The type of error (UE/CE) will depend on the inject.eccmask value: 806 * Any bits set to a 1 will flip the corresponding ECC bit 807 * Correctable errors can be injected by flipping 1 bit or the bits within 808 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or 809 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an 810 * uncorrectable error to be injected. 811 */ 812 813 #define DECLARE_ADDR_MATCH(param, limit) \ 814 static ssize_t i7core_inject_store_##param( \ 815 struct device *dev, \ 816 struct device_attribute *mattr, \ 817 const char *data, size_t count) \ 818 { \ 819 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 820 struct i7core_pvt *pvt; \ 821 long value; \ 822 int rc; \ 823 \ 824 edac_dbg(1, "\n"); \ 825 pvt = mci->pvt_info; \ 826 \ 827 if (pvt->inject.enable) \ 828 disable_inject(mci); \ 829 \ 830 if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\ 831 value = -1; \ 832 else { \ 833 rc = kstrtoul(data, 10, &value); \ 834 if ((rc < 0) || (value >= limit)) \ 835 return -EIO; \ 836 } \ 837 \ 838 pvt->inject.param = value; \ 839 \ 840 return count; \ 841 } \ 842 \ 843 static ssize_t i7core_inject_show_##param( \ 844 struct device *dev, \ 845 struct device_attribute *mattr, \ 846 char *data) \ 847 { \ 848 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 849 struct i7core_pvt *pvt; \ 850 \ 851 pvt = mci->pvt_info; \ 852 edac_dbg(1, "pvt=%p\n", pvt); \ 853 if (pvt->inject.param < 0) \ 854 return sprintf(data, "any\n"); \ 855 else \ 856 return sprintf(data, "%d\n", pvt->inject.param);\ 857 } 858 859 #define ATTR_ADDR_MATCH(param) \ 860 static DEVICE_ATTR(param, S_IRUGO | S_IWUSR, \ 861 i7core_inject_show_##param, \ 862 i7core_inject_store_##param) 863 864 DECLARE_ADDR_MATCH(channel, 3); 865 DECLARE_ADDR_MATCH(dimm, 3); 866 DECLARE_ADDR_MATCH(rank, 4); 867 DECLARE_ADDR_MATCH(bank, 32); 868 DECLARE_ADDR_MATCH(page, 0x10000); 869 DECLARE_ADDR_MATCH(col, 0x4000); 870 871 ATTR_ADDR_MATCH(channel); 872 ATTR_ADDR_MATCH(dimm); 873 ATTR_ADDR_MATCH(rank); 874 ATTR_ADDR_MATCH(bank); 875 ATTR_ADDR_MATCH(page); 876 ATTR_ADDR_MATCH(col); 877 878 static int write_and_test(struct pci_dev *dev, const int where, const u32 val) 879 { 880 u32 read; 881 int count; 882 883 edac_dbg(0, "setting pci %02x:%02x.%x reg=%02x value=%08x\n", 884 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), 885 where, val); 886 887 for (count = 0; count < 10; count++) { 888 if (count) 889 msleep(100); 890 pci_write_config_dword(dev, where, val); 891 pci_read_config_dword(dev, where, &read); 892 893 if (read == val) 894 return 0; 895 } 896 897 i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x " 898 "write=%08x. Read=%08x\n", 899 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), 900 where, val, read); 901 902 return -EINVAL; 903 } 904 905 /* 906 * This routine prepares the Memory Controller for error injection. 907 * The error will be injected when some process tries to write to the 908 * memory that matches the given criteria. 909 * The criteria can be set in terms of a mask where dimm, rank, bank, page 910 * and col can be specified. 911 * A -1 value for any of the mask items will make the MCU to ignore 912 * that matching criteria for error injection. 913 * 914 * It should be noticed that the error will only happen after a write operation 915 * on a memory that matches the condition. if REPEAT_EN is not enabled at 916 * inject mask, then it will produce just one error. Otherwise, it will repeat 917 * until the injectmask would be cleaned. 918 * 919 * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD 920 * is reliable enough to check if the MC is using the 921 * three channels. However, this is not clear at the datasheet. 922 */ 923 static ssize_t i7core_inject_enable_store(struct device *dev, 924 struct device_attribute *mattr, 925 const char *data, size_t count) 926 { 927 struct mem_ctl_info *mci = to_mci(dev); 928 struct i7core_pvt *pvt = mci->pvt_info; 929 u32 injectmask; 930 u64 mask = 0; 931 int rc; 932 long enable; 933 934 if (!pvt->pci_ch[pvt->inject.channel][0]) 935 return 0; 936 937 rc = kstrtoul(data, 10, &enable); 938 if ((rc < 0)) 939 return 0; 940 941 if (enable) { 942 pvt->inject.enable = 1; 943 } else { 944 disable_inject(mci); 945 return count; 946 } 947 948 /* Sets pvt->inject.dimm mask */ 949 if (pvt->inject.dimm < 0) 950 mask |= 1LL << 41; 951 else { 952 if (pvt->channel[pvt->inject.channel].dimms > 2) 953 mask |= (pvt->inject.dimm & 0x3LL) << 35; 954 else 955 mask |= (pvt->inject.dimm & 0x1LL) << 36; 956 } 957 958 /* Sets pvt->inject.rank mask */ 959 if (pvt->inject.rank < 0) 960 mask |= 1LL << 40; 961 else { 962 if (pvt->channel[pvt->inject.channel].dimms > 2) 963 mask |= (pvt->inject.rank & 0x1LL) << 34; 964 else 965 mask |= (pvt->inject.rank & 0x3LL) << 34; 966 } 967 968 /* Sets pvt->inject.bank mask */ 969 if (pvt->inject.bank < 0) 970 mask |= 1LL << 39; 971 else 972 mask |= (pvt->inject.bank & 0x15LL) << 30; 973 974 /* Sets pvt->inject.page mask */ 975 if (pvt->inject.page < 0) 976 mask |= 1LL << 38; 977 else 978 mask |= (pvt->inject.page & 0xffff) << 14; 979 980 /* Sets pvt->inject.column mask */ 981 if (pvt->inject.col < 0) 982 mask |= 1LL << 37; 983 else 984 mask |= (pvt->inject.col & 0x3fff); 985 986 /* 987 * bit 0: REPEAT_EN 988 * bits 1-2: MASK_HALF_CACHELINE 989 * bit 3: INJECT_ECC 990 * bit 4: INJECT_ADDR_PARITY 991 */ 992 993 injectmask = (pvt->inject.type & 1) | 994 (pvt->inject.section & 0x3) << 1 | 995 (pvt->inject.type & 0x6) << (3 - 1); 996 997 /* Unlock writes to registers - this register is write only */ 998 pci_write_config_dword(pvt->pci_noncore, 999 MC_CFG_CONTROL, 0x2); 1000 1001 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 1002 MC_CHANNEL_ADDR_MATCH, mask); 1003 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 1004 MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L); 1005 1006 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 1007 MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask); 1008 1009 write_and_test(pvt->pci_ch[pvt->inject.channel][0], 1010 MC_CHANNEL_ERROR_INJECT, injectmask); 1011 1012 /* 1013 * This is something undocumented, based on my tests 1014 * Without writing 8 to this register, errors aren't injected. Not sure 1015 * why. 1016 */ 1017 pci_write_config_dword(pvt->pci_noncore, 1018 MC_CFG_CONTROL, 8); 1019 1020 edac_dbg(0, "Error inject addr match 0x%016llx, ecc 0x%08x, inject 0x%08x\n", 1021 mask, pvt->inject.eccmask, injectmask); 1022 1023 1024 return count; 1025 } 1026 1027 static ssize_t i7core_inject_enable_show(struct device *dev, 1028 struct device_attribute *mattr, 1029 char *data) 1030 { 1031 struct mem_ctl_info *mci = to_mci(dev); 1032 struct i7core_pvt *pvt = mci->pvt_info; 1033 u32 injectmask; 1034 1035 if (!pvt->pci_ch[pvt->inject.channel][0]) 1036 return 0; 1037 1038 pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0], 1039 MC_CHANNEL_ERROR_INJECT, &injectmask); 1040 1041 edac_dbg(0, "Inject error read: 0x%018x\n", injectmask); 1042 1043 if (injectmask & 0x0c) 1044 pvt->inject.enable = 1; 1045 1046 return sprintf(data, "%d\n", pvt->inject.enable); 1047 } 1048 1049 #define DECLARE_COUNTER(param) \ 1050 static ssize_t i7core_show_counter_##param( \ 1051 struct device *dev, \ 1052 struct device_attribute *mattr, \ 1053 char *data) \ 1054 { \ 1055 struct mem_ctl_info *mci = dev_get_drvdata(dev); \ 1056 struct i7core_pvt *pvt = mci->pvt_info; \ 1057 \ 1058 edac_dbg(1, "\n"); \ 1059 if (!pvt->ce_count_available || (pvt->is_registered)) \ 1060 return sprintf(data, "data unavailable\n"); \ 1061 return sprintf(data, "%lu\n", \ 1062 pvt->udimm_ce_count[param]); \ 1063 } 1064 1065 #define ATTR_COUNTER(param) \ 1066 static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR, \ 1067 i7core_show_counter_##param, \ 1068 NULL) 1069 1070 DECLARE_COUNTER(0); 1071 DECLARE_COUNTER(1); 1072 DECLARE_COUNTER(2); 1073 1074 ATTR_COUNTER(0); 1075 ATTR_COUNTER(1); 1076 ATTR_COUNTER(2); 1077 1078 /* 1079 * inject_addrmatch device sysfs struct 1080 */ 1081 1082 static struct attribute *i7core_addrmatch_attrs[] = { 1083 &dev_attr_channel.attr, 1084 &dev_attr_dimm.attr, 1085 &dev_attr_rank.attr, 1086 &dev_attr_bank.attr, 1087 &dev_attr_page.attr, 1088 &dev_attr_col.attr, 1089 NULL 1090 }; 1091 1092 static struct attribute_group addrmatch_grp = { 1093 .attrs = i7core_addrmatch_attrs, 1094 }; 1095 1096 static const struct attribute_group *addrmatch_groups[] = { 1097 &addrmatch_grp, 1098 NULL 1099 }; 1100 1101 static void addrmatch_release(struct device *device) 1102 { 1103 edac_dbg(1, "Releasing device %s\n", dev_name(device)); 1104 kfree(device); 1105 } 1106 1107 static struct device_type addrmatch_type = { 1108 .groups = addrmatch_groups, 1109 .release = addrmatch_release, 1110 }; 1111 1112 /* 1113 * all_channel_counts sysfs struct 1114 */ 1115 1116 static struct attribute *i7core_udimm_counters_attrs[] = { 1117 &dev_attr_udimm0.attr, 1118 &dev_attr_udimm1.attr, 1119 &dev_attr_udimm2.attr, 1120 NULL 1121 }; 1122 1123 static struct attribute_group all_channel_counts_grp = { 1124 .attrs = i7core_udimm_counters_attrs, 1125 }; 1126 1127 static const struct attribute_group *all_channel_counts_groups[] = { 1128 &all_channel_counts_grp, 1129 NULL 1130 }; 1131 1132 static void all_channel_counts_release(struct device *device) 1133 { 1134 edac_dbg(1, "Releasing device %s\n", dev_name(device)); 1135 kfree(device); 1136 } 1137 1138 static struct device_type all_channel_counts_type = { 1139 .groups = all_channel_counts_groups, 1140 .release = all_channel_counts_release, 1141 }; 1142 1143 /* 1144 * inject sysfs attributes 1145 */ 1146 1147 static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR, 1148 i7core_inject_section_show, i7core_inject_section_store); 1149 1150 static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR, 1151 i7core_inject_type_show, i7core_inject_type_store); 1152 1153 1154 static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR, 1155 i7core_inject_eccmask_show, i7core_inject_eccmask_store); 1156 1157 static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR, 1158 i7core_inject_enable_show, i7core_inject_enable_store); 1159 1160 static struct attribute *i7core_dev_attrs[] = { 1161 &dev_attr_inject_section.attr, 1162 &dev_attr_inject_type.attr, 1163 &dev_attr_inject_eccmask.attr, 1164 &dev_attr_inject_enable.attr, 1165 NULL 1166 }; 1167 1168 ATTRIBUTE_GROUPS(i7core_dev); 1169 1170 static int i7core_create_sysfs_devices(struct mem_ctl_info *mci) 1171 { 1172 struct i7core_pvt *pvt = mci->pvt_info; 1173 int rc; 1174 1175 pvt->addrmatch_dev = kzalloc(sizeof(*pvt->addrmatch_dev), GFP_KERNEL); 1176 if (!pvt->addrmatch_dev) 1177 return -ENOMEM; 1178 1179 pvt->addrmatch_dev->type = &addrmatch_type; 1180 pvt->addrmatch_dev->bus = mci->dev.bus; 1181 device_initialize(pvt->addrmatch_dev); 1182 pvt->addrmatch_dev->parent = &mci->dev; 1183 dev_set_name(pvt->addrmatch_dev, "inject_addrmatch"); 1184 dev_set_drvdata(pvt->addrmatch_dev, mci); 1185 1186 edac_dbg(1, "creating %s\n", dev_name(pvt->addrmatch_dev)); 1187 1188 rc = device_add(pvt->addrmatch_dev); 1189 if (rc < 0) 1190 return rc; 1191 1192 if (!pvt->is_registered) { 1193 pvt->chancounts_dev = kzalloc(sizeof(*pvt->chancounts_dev), 1194 GFP_KERNEL); 1195 if (!pvt->chancounts_dev) { 1196 put_device(pvt->addrmatch_dev); 1197 device_del(pvt->addrmatch_dev); 1198 return -ENOMEM; 1199 } 1200 1201 pvt->chancounts_dev->type = &all_channel_counts_type; 1202 pvt->chancounts_dev->bus = mci->dev.bus; 1203 device_initialize(pvt->chancounts_dev); 1204 pvt->chancounts_dev->parent = &mci->dev; 1205 dev_set_name(pvt->chancounts_dev, "all_channel_counts"); 1206 dev_set_drvdata(pvt->chancounts_dev, mci); 1207 1208 edac_dbg(1, "creating %s\n", dev_name(pvt->chancounts_dev)); 1209 1210 rc = device_add(pvt->chancounts_dev); 1211 if (rc < 0) 1212 return rc; 1213 } 1214 return 0; 1215 } 1216 1217 static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci) 1218 { 1219 struct i7core_pvt *pvt = mci->pvt_info; 1220 1221 edac_dbg(1, "\n"); 1222 1223 if (!pvt->is_registered) { 1224 put_device(pvt->chancounts_dev); 1225 device_del(pvt->chancounts_dev); 1226 } 1227 put_device(pvt->addrmatch_dev); 1228 device_del(pvt->addrmatch_dev); 1229 } 1230 1231 /**************************************************************************** 1232 Device initialization routines: put/get, init/exit 1233 ****************************************************************************/ 1234 1235 /* 1236 * i7core_put_all_devices 'put' all the devices that we have 1237 * reserved via 'get' 1238 */ 1239 static void i7core_put_devices(struct i7core_dev *i7core_dev) 1240 { 1241 int i; 1242 1243 edac_dbg(0, "\n"); 1244 for (i = 0; i < i7core_dev->n_devs; i++) { 1245 struct pci_dev *pdev = i7core_dev->pdev[i]; 1246 if (!pdev) 1247 continue; 1248 edac_dbg(0, "Removing dev %02x:%02x.%d\n", 1249 pdev->bus->number, 1250 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); 1251 pci_dev_put(pdev); 1252 } 1253 } 1254 1255 static void i7core_put_all_devices(void) 1256 { 1257 struct i7core_dev *i7core_dev, *tmp; 1258 1259 list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) { 1260 i7core_put_devices(i7core_dev); 1261 free_i7core_dev(i7core_dev); 1262 } 1263 } 1264 1265 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table) 1266 { 1267 struct pci_dev *pdev = NULL; 1268 int i; 1269 1270 /* 1271 * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses 1272 * aren't announced by acpi. So, we need to use a legacy scan probing 1273 * to detect them 1274 */ 1275 while (table && table->descr) { 1276 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL); 1277 if (unlikely(!pdev)) { 1278 for (i = 0; i < MAX_SOCKET_BUSES; i++) 1279 pcibios_scan_specific_bus(255-i); 1280 } 1281 pci_dev_put(pdev); 1282 table++; 1283 } 1284 } 1285 1286 static unsigned i7core_pci_lastbus(void) 1287 { 1288 int last_bus = 0, bus; 1289 struct pci_bus *b = NULL; 1290 1291 while ((b = pci_find_next_bus(b)) != NULL) { 1292 bus = b->number; 1293 edac_dbg(0, "Found bus %d\n", bus); 1294 if (bus > last_bus) 1295 last_bus = bus; 1296 } 1297 1298 edac_dbg(0, "Last bus %d\n", last_bus); 1299 1300 return last_bus; 1301 } 1302 1303 /* 1304 * i7core_get_all_devices Find and perform 'get' operation on the MCH's 1305 * device/functions we want to reference for this driver 1306 * 1307 * Need to 'get' device 16 func 1 and func 2 1308 */ 1309 static int i7core_get_onedevice(struct pci_dev **prev, 1310 const struct pci_id_table *table, 1311 const unsigned devno, 1312 const unsigned last_bus) 1313 { 1314 struct i7core_dev *i7core_dev; 1315 const struct pci_id_descr *dev_descr = &table->descr[devno]; 1316 1317 struct pci_dev *pdev = NULL; 1318 u8 bus = 0; 1319 u8 socket = 0; 1320 1321 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1322 dev_descr->dev_id, *prev); 1323 1324 /* 1325 * On Xeon 55xx, the Intel QuickPath Arch Generic Non-core regs 1326 * is at addr 8086:2c40, instead of 8086:2c41. So, we need 1327 * to probe for the alternate address in case of failure 1328 */ 1329 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev) { 1330 pci_dev_get(*prev); /* pci_get_device will put it */ 1331 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1332 PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev); 1333 } 1334 1335 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && 1336 !pdev) { 1337 pci_dev_get(*prev); /* pci_get_device will put it */ 1338 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 1339 PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT, 1340 *prev); 1341 } 1342 1343 if (!pdev) { 1344 if (*prev) { 1345 *prev = pdev; 1346 return 0; 1347 } 1348 1349 if (dev_descr->optional) 1350 return 0; 1351 1352 if (devno == 0) 1353 return -ENODEV; 1354 1355 i7core_printk(KERN_INFO, 1356 "Device not found: dev %02x.%d PCI ID %04x:%04x\n", 1357 dev_descr->dev, dev_descr->func, 1358 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1359 1360 /* End of list, leave */ 1361 return -ENODEV; 1362 } 1363 bus = pdev->bus->number; 1364 1365 socket = last_bus - bus; 1366 1367 i7core_dev = get_i7core_dev(socket); 1368 if (!i7core_dev) { 1369 i7core_dev = alloc_i7core_dev(socket, table); 1370 if (!i7core_dev) { 1371 pci_dev_put(pdev); 1372 return -ENOMEM; 1373 } 1374 } 1375 1376 if (i7core_dev->pdev[devno]) { 1377 i7core_printk(KERN_ERR, 1378 "Duplicated device for " 1379 "dev %02x:%02x.%d PCI ID %04x:%04x\n", 1380 bus, dev_descr->dev, dev_descr->func, 1381 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1382 pci_dev_put(pdev); 1383 return -ENODEV; 1384 } 1385 1386 i7core_dev->pdev[devno] = pdev; 1387 1388 /* Sanity check */ 1389 if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev || 1390 PCI_FUNC(pdev->devfn) != dev_descr->func)) { 1391 i7core_printk(KERN_ERR, 1392 "Device PCI ID %04x:%04x " 1393 "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n", 1394 PCI_VENDOR_ID_INTEL, dev_descr->dev_id, 1395 bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 1396 bus, dev_descr->dev, dev_descr->func); 1397 return -ENODEV; 1398 } 1399 1400 /* Be sure that the device is enabled */ 1401 if (unlikely(pci_enable_device(pdev) < 0)) { 1402 i7core_printk(KERN_ERR, 1403 "Couldn't enable " 1404 "dev %02x:%02x.%d PCI ID %04x:%04x\n", 1405 bus, dev_descr->dev, dev_descr->func, 1406 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1407 return -ENODEV; 1408 } 1409 1410 edac_dbg(0, "Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n", 1411 socket, bus, dev_descr->dev, 1412 dev_descr->func, 1413 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 1414 1415 /* 1416 * As stated on drivers/pci/search.c, the reference count for 1417 * @from is always decremented if it is not %NULL. So, as we need 1418 * to get all devices up to null, we need to do a get for the device 1419 */ 1420 pci_dev_get(pdev); 1421 1422 *prev = pdev; 1423 1424 return 0; 1425 } 1426 1427 static int i7core_get_all_devices(void) 1428 { 1429 int i, rc, last_bus; 1430 struct pci_dev *pdev = NULL; 1431 const struct pci_id_table *table = pci_dev_table; 1432 1433 last_bus = i7core_pci_lastbus(); 1434 1435 while (table && table->descr) { 1436 for (i = 0; i < table->n_devs; i++) { 1437 pdev = NULL; 1438 do { 1439 rc = i7core_get_onedevice(&pdev, table, i, 1440 last_bus); 1441 if (rc < 0) { 1442 if (i == 0) { 1443 i = table->n_devs; 1444 break; 1445 } 1446 i7core_put_all_devices(); 1447 return -ENODEV; 1448 } 1449 } while (pdev); 1450 } 1451 table++; 1452 } 1453 1454 return 0; 1455 } 1456 1457 static int mci_bind_devs(struct mem_ctl_info *mci, 1458 struct i7core_dev *i7core_dev) 1459 { 1460 struct i7core_pvt *pvt = mci->pvt_info; 1461 struct pci_dev *pdev; 1462 int i, func, slot; 1463 char *family; 1464 1465 pvt->is_registered = false; 1466 pvt->enable_scrub = false; 1467 for (i = 0; i < i7core_dev->n_devs; i++) { 1468 pdev = i7core_dev->pdev[i]; 1469 if (!pdev) 1470 continue; 1471 1472 func = PCI_FUNC(pdev->devfn); 1473 slot = PCI_SLOT(pdev->devfn); 1474 if (slot == 3) { 1475 if (unlikely(func > MAX_MCR_FUNC)) 1476 goto error; 1477 pvt->pci_mcr[func] = pdev; 1478 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) { 1479 if (unlikely(func > MAX_CHAN_FUNC)) 1480 goto error; 1481 pvt->pci_ch[slot - 4][func] = pdev; 1482 } else if (!slot && !func) { 1483 pvt->pci_noncore = pdev; 1484 1485 /* Detect the processor family */ 1486 switch (pdev->device) { 1487 case PCI_DEVICE_ID_INTEL_I7_NONCORE: 1488 family = "Xeon 35xx/ i7core"; 1489 pvt->enable_scrub = false; 1490 break; 1491 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT: 1492 family = "i7-800/i5-700"; 1493 pvt->enable_scrub = false; 1494 break; 1495 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE: 1496 family = "Xeon 34xx"; 1497 pvt->enable_scrub = false; 1498 break; 1499 case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT: 1500 family = "Xeon 55xx"; 1501 pvt->enable_scrub = true; 1502 break; 1503 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2: 1504 family = "Xeon 56xx / i7-900"; 1505 pvt->enable_scrub = true; 1506 break; 1507 default: 1508 family = "unknown"; 1509 pvt->enable_scrub = false; 1510 } 1511 edac_dbg(0, "Detected a processor type %s\n", family); 1512 } else 1513 goto error; 1514 1515 edac_dbg(0, "Associated fn %d.%d, dev = %p, socket %d\n", 1516 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 1517 pdev, i7core_dev->socket); 1518 1519 if (PCI_SLOT(pdev->devfn) == 3 && 1520 PCI_FUNC(pdev->devfn) == 2) 1521 pvt->is_registered = true; 1522 } 1523 1524 return 0; 1525 1526 error: 1527 i7core_printk(KERN_ERR, "Device %d, function %d " 1528 "is out of the expected range\n", 1529 slot, func); 1530 return -EINVAL; 1531 } 1532 1533 /**************************************************************************** 1534 Error check routines 1535 ****************************************************************************/ 1536 1537 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci, 1538 const int chan, 1539 const int new0, 1540 const int new1, 1541 const int new2) 1542 { 1543 struct i7core_pvt *pvt = mci->pvt_info; 1544 int add0 = 0, add1 = 0, add2 = 0; 1545 /* Updates CE counters if it is not the first time here */ 1546 if (pvt->ce_count_available) { 1547 /* Updates CE counters */ 1548 1549 add2 = new2 - pvt->rdimm_last_ce_count[chan][2]; 1550 add1 = new1 - pvt->rdimm_last_ce_count[chan][1]; 1551 add0 = new0 - pvt->rdimm_last_ce_count[chan][0]; 1552 1553 if (add2 < 0) 1554 add2 += 0x7fff; 1555 pvt->rdimm_ce_count[chan][2] += add2; 1556 1557 if (add1 < 0) 1558 add1 += 0x7fff; 1559 pvt->rdimm_ce_count[chan][1] += add1; 1560 1561 if (add0 < 0) 1562 add0 += 0x7fff; 1563 pvt->rdimm_ce_count[chan][0] += add0; 1564 } else 1565 pvt->ce_count_available = 1; 1566 1567 /* Store the new values */ 1568 pvt->rdimm_last_ce_count[chan][2] = new2; 1569 pvt->rdimm_last_ce_count[chan][1] = new1; 1570 pvt->rdimm_last_ce_count[chan][0] = new0; 1571 1572 /*updated the edac core */ 1573 if (add0 != 0) 1574 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add0, 1575 0, 0, 0, 1576 chan, 0, -1, "error", ""); 1577 if (add1 != 0) 1578 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add1, 1579 0, 0, 0, 1580 chan, 1, -1, "error", ""); 1581 if (add2 != 0) 1582 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add2, 1583 0, 0, 0, 1584 chan, 2, -1, "error", ""); 1585 } 1586 1587 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci) 1588 { 1589 struct i7core_pvt *pvt = mci->pvt_info; 1590 u32 rcv[3][2]; 1591 int i, new0, new1, new2; 1592 1593 /*Read DEV 3: FUN 2: MC_COR_ECC_CNT regs directly*/ 1594 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0, 1595 &rcv[0][0]); 1596 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1, 1597 &rcv[0][1]); 1598 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2, 1599 &rcv[1][0]); 1600 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3, 1601 &rcv[1][1]); 1602 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4, 1603 &rcv[2][0]); 1604 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5, 1605 &rcv[2][1]); 1606 for (i = 0 ; i < 3; i++) { 1607 edac_dbg(3, "MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n", 1608 (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]); 1609 /*if the channel has 3 dimms*/ 1610 if (pvt->channel[i].dimms > 2) { 1611 new0 = DIMM_BOT_COR_ERR(rcv[i][0]); 1612 new1 = DIMM_TOP_COR_ERR(rcv[i][0]); 1613 new2 = DIMM_BOT_COR_ERR(rcv[i][1]); 1614 } else { 1615 new0 = DIMM_TOP_COR_ERR(rcv[i][0]) + 1616 DIMM_BOT_COR_ERR(rcv[i][0]); 1617 new1 = DIMM_TOP_COR_ERR(rcv[i][1]) + 1618 DIMM_BOT_COR_ERR(rcv[i][1]); 1619 new2 = 0; 1620 } 1621 1622 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2); 1623 } 1624 } 1625 1626 /* This function is based on the device 3 function 4 registers as described on: 1627 * Intel Xeon Processor 5500 Series Datasheet Volume 2 1628 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf 1629 * also available at: 1630 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf 1631 */ 1632 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci) 1633 { 1634 struct i7core_pvt *pvt = mci->pvt_info; 1635 u32 rcv1, rcv0; 1636 int new0, new1, new2; 1637 1638 if (!pvt->pci_mcr[4]) { 1639 edac_dbg(0, "MCR registers not found\n"); 1640 return; 1641 } 1642 1643 /* Corrected test errors */ 1644 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1); 1645 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0); 1646 1647 /* Store the new values */ 1648 new2 = DIMM2_COR_ERR(rcv1); 1649 new1 = DIMM1_COR_ERR(rcv0); 1650 new0 = DIMM0_COR_ERR(rcv0); 1651 1652 /* Updates CE counters if it is not the first time here */ 1653 if (pvt->ce_count_available) { 1654 /* Updates CE counters */ 1655 int add0, add1, add2; 1656 1657 add2 = new2 - pvt->udimm_last_ce_count[2]; 1658 add1 = new1 - pvt->udimm_last_ce_count[1]; 1659 add0 = new0 - pvt->udimm_last_ce_count[0]; 1660 1661 if (add2 < 0) 1662 add2 += 0x7fff; 1663 pvt->udimm_ce_count[2] += add2; 1664 1665 if (add1 < 0) 1666 add1 += 0x7fff; 1667 pvt->udimm_ce_count[1] += add1; 1668 1669 if (add0 < 0) 1670 add0 += 0x7fff; 1671 pvt->udimm_ce_count[0] += add0; 1672 1673 if (add0 | add1 | add2) 1674 i7core_printk(KERN_ERR, "New Corrected error(s): " 1675 "dimm0: +%d, dimm1: +%d, dimm2 +%d\n", 1676 add0, add1, add2); 1677 } else 1678 pvt->ce_count_available = 1; 1679 1680 /* Store the new values */ 1681 pvt->udimm_last_ce_count[2] = new2; 1682 pvt->udimm_last_ce_count[1] = new1; 1683 pvt->udimm_last_ce_count[0] = new0; 1684 } 1685 1686 /* 1687 * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32 1688 * Architectures Software Developer’s Manual Volume 3B. 1689 * Nehalem are defined as family 0x06, model 0x1a 1690 * 1691 * The MCA registers used here are the following ones: 1692 * struct mce field MCA Register 1693 * m->status MSR_IA32_MC8_STATUS 1694 * m->addr MSR_IA32_MC8_ADDR 1695 * m->misc MSR_IA32_MC8_MISC 1696 * In the case of Nehalem, the error information is masked at .status and .misc 1697 * fields 1698 */ 1699 static void i7core_mce_output_error(struct mem_ctl_info *mci, 1700 const struct mce *m) 1701 { 1702 struct i7core_pvt *pvt = mci->pvt_info; 1703 char *optype, *err; 1704 enum hw_event_mc_err_type tp_event; 1705 unsigned long error = m->status & 0x1ff0000l; 1706 bool uncorrected_error = m->mcgstatus & 1ll << 61; 1707 bool ripv = m->mcgstatus & 1; 1708 u32 optypenum = (m->status >> 4) & 0x07; 1709 u32 core_err_cnt = (m->status >> 38) & 0x7fff; 1710 u32 dimm = (m->misc >> 16) & 0x3; 1711 u32 channel = (m->misc >> 18) & 0x3; 1712 u32 syndrome = m->misc >> 32; 1713 u32 errnum = find_first_bit(&error, 32); 1714 1715 if (uncorrected_error) { 1716 if (ripv) 1717 tp_event = HW_EVENT_ERR_FATAL; 1718 else 1719 tp_event = HW_EVENT_ERR_UNCORRECTED; 1720 } else { 1721 tp_event = HW_EVENT_ERR_CORRECTED; 1722 } 1723 1724 switch (optypenum) { 1725 case 0: 1726 optype = "generic undef request"; 1727 break; 1728 case 1: 1729 optype = "read error"; 1730 break; 1731 case 2: 1732 optype = "write error"; 1733 break; 1734 case 3: 1735 optype = "addr/cmd error"; 1736 break; 1737 case 4: 1738 optype = "scrubbing error"; 1739 break; 1740 default: 1741 optype = "reserved"; 1742 break; 1743 } 1744 1745 switch (errnum) { 1746 case 16: 1747 err = "read ECC error"; 1748 break; 1749 case 17: 1750 err = "RAS ECC error"; 1751 break; 1752 case 18: 1753 err = "write parity error"; 1754 break; 1755 case 19: 1756 err = "redundacy loss"; 1757 break; 1758 case 20: 1759 err = "reserved"; 1760 break; 1761 case 21: 1762 err = "memory range error"; 1763 break; 1764 case 22: 1765 err = "RTID out of range"; 1766 break; 1767 case 23: 1768 err = "address parity error"; 1769 break; 1770 case 24: 1771 err = "byte enable parity error"; 1772 break; 1773 default: 1774 err = "unknown"; 1775 } 1776 1777 /* 1778 * Call the helper to output message 1779 * FIXME: what to do if core_err_cnt > 1? Currently, it generates 1780 * only one event 1781 */ 1782 if (uncorrected_error || !pvt->is_registered) 1783 edac_mc_handle_error(tp_event, mci, core_err_cnt, 1784 m->addr >> PAGE_SHIFT, 1785 m->addr & ~PAGE_MASK, 1786 syndrome, 1787 channel, dimm, -1, 1788 err, optype); 1789 } 1790 1791 /* 1792 * i7core_check_error Retrieve and process errors reported by the 1793 * hardware. Called by the Core module. 1794 */ 1795 static void i7core_check_error(struct mem_ctl_info *mci) 1796 { 1797 struct i7core_pvt *pvt = mci->pvt_info; 1798 int i; 1799 unsigned count = 0; 1800 struct mce *m; 1801 1802 /* 1803 * MCE first step: Copy all mce errors into a temporary buffer 1804 * We use a double buffering here, to reduce the risk of 1805 * losing an error. 1806 */ 1807 smp_rmb(); 1808 count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in) 1809 % MCE_LOG_LEN; 1810 if (!count) 1811 goto check_ce_error; 1812 1813 m = pvt->mce_outentry; 1814 if (pvt->mce_in + count > MCE_LOG_LEN) { 1815 unsigned l = MCE_LOG_LEN - pvt->mce_in; 1816 1817 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l); 1818 smp_wmb(); 1819 pvt->mce_in = 0; 1820 count -= l; 1821 m += l; 1822 } 1823 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count); 1824 smp_wmb(); 1825 pvt->mce_in += count; 1826 1827 smp_rmb(); 1828 if (pvt->mce_overrun) { 1829 i7core_printk(KERN_ERR, "Lost %d memory errors\n", 1830 pvt->mce_overrun); 1831 smp_wmb(); 1832 pvt->mce_overrun = 0; 1833 } 1834 1835 /* 1836 * MCE second step: parse errors and display 1837 */ 1838 for (i = 0; i < count; i++) 1839 i7core_mce_output_error(mci, &pvt->mce_outentry[i]); 1840 1841 /* 1842 * Now, let's increment CE error counts 1843 */ 1844 check_ce_error: 1845 if (!pvt->is_registered) 1846 i7core_udimm_check_mc_ecc_err(mci); 1847 else 1848 i7core_rdimm_check_mc_ecc_err(mci); 1849 } 1850 1851 /* 1852 * i7core_mce_check_error Replicates mcelog routine to get errors 1853 * This routine simply queues mcelog errors, and 1854 * return. The error itself should be handled later 1855 * by i7core_check_error. 1856 * WARNING: As this routine should be called at NMI time, extra care should 1857 * be taken to avoid deadlocks, and to be as fast as possible. 1858 */ 1859 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val, 1860 void *data) 1861 { 1862 struct mce *mce = (struct mce *)data; 1863 struct i7core_dev *i7_dev; 1864 struct mem_ctl_info *mci; 1865 struct i7core_pvt *pvt; 1866 1867 i7_dev = get_i7core_dev(mce->socketid); 1868 if (!i7_dev) 1869 return NOTIFY_BAD; 1870 1871 mci = i7_dev->mci; 1872 pvt = mci->pvt_info; 1873 1874 /* 1875 * Just let mcelog handle it if the error is 1876 * outside the memory controller 1877 */ 1878 if (((mce->status & 0xffff) >> 7) != 1) 1879 return NOTIFY_DONE; 1880 1881 /* Bank 8 registers are the only ones that we know how to handle */ 1882 if (mce->bank != 8) 1883 return NOTIFY_DONE; 1884 1885 smp_rmb(); 1886 if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) { 1887 smp_wmb(); 1888 pvt->mce_overrun++; 1889 return NOTIFY_DONE; 1890 } 1891 1892 /* Copy memory error at the ringbuffer */ 1893 memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce)); 1894 smp_wmb(); 1895 pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN; 1896 1897 /* Handle fatal errors immediately */ 1898 if (mce->mcgstatus & 1) 1899 i7core_check_error(mci); 1900 1901 /* Advise mcelog that the errors were handled */ 1902 return NOTIFY_STOP; 1903 } 1904 1905 static struct notifier_block i7_mce_dec = { 1906 .notifier_call = i7core_mce_check_error, 1907 }; 1908 1909 struct memdev_dmi_entry { 1910 u8 type; 1911 u8 length; 1912 u16 handle; 1913 u16 phys_mem_array_handle; 1914 u16 mem_err_info_handle; 1915 u16 total_width; 1916 u16 data_width; 1917 u16 size; 1918 u8 form; 1919 u8 device_set; 1920 u8 device_locator; 1921 u8 bank_locator; 1922 u8 memory_type; 1923 u16 type_detail; 1924 u16 speed; 1925 u8 manufacturer; 1926 u8 serial_number; 1927 u8 asset_tag; 1928 u8 part_number; 1929 u8 attributes; 1930 u32 extended_size; 1931 u16 conf_mem_clk_speed; 1932 } __attribute__((__packed__)); 1933 1934 1935 /* 1936 * Decode the DRAM Clock Frequency, be paranoid, make sure that all 1937 * memory devices show the same speed, and if they don't then consider 1938 * all speeds to be invalid. 1939 */ 1940 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq) 1941 { 1942 int *dclk_freq = _dclk_freq; 1943 u16 dmi_mem_clk_speed; 1944 1945 if (*dclk_freq == -1) 1946 return; 1947 1948 if (dh->type == DMI_ENTRY_MEM_DEVICE) { 1949 struct memdev_dmi_entry *memdev_dmi_entry = 1950 (struct memdev_dmi_entry *)dh; 1951 unsigned long conf_mem_clk_speed_offset = 1952 (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed - 1953 (unsigned long)&memdev_dmi_entry->type; 1954 unsigned long speed_offset = 1955 (unsigned long)&memdev_dmi_entry->speed - 1956 (unsigned long)&memdev_dmi_entry->type; 1957 1958 /* Check that a DIMM is present */ 1959 if (memdev_dmi_entry->size == 0) 1960 return; 1961 1962 /* 1963 * Pick the configured speed if it's available, otherwise 1964 * pick the DIMM speed, or we don't have a speed. 1965 */ 1966 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) { 1967 dmi_mem_clk_speed = 1968 memdev_dmi_entry->conf_mem_clk_speed; 1969 } else if (memdev_dmi_entry->length > speed_offset) { 1970 dmi_mem_clk_speed = memdev_dmi_entry->speed; 1971 } else { 1972 *dclk_freq = -1; 1973 return; 1974 } 1975 1976 if (*dclk_freq == 0) { 1977 /* First pass, speed was 0 */ 1978 if (dmi_mem_clk_speed > 0) { 1979 /* Set speed if a valid speed is read */ 1980 *dclk_freq = dmi_mem_clk_speed; 1981 } else { 1982 /* Otherwise we don't have a valid speed */ 1983 *dclk_freq = -1; 1984 } 1985 } else if (*dclk_freq > 0 && 1986 *dclk_freq != dmi_mem_clk_speed) { 1987 /* 1988 * If we have a speed, check that all DIMMS are the same 1989 * speed, otherwise set the speed as invalid. 1990 */ 1991 *dclk_freq = -1; 1992 } 1993 } 1994 } 1995 1996 /* 1997 * The default DCLK frequency is used as a fallback if we 1998 * fail to find anything reliable in the DMI. The value 1999 * is taken straight from the datasheet. 2000 */ 2001 #define DEFAULT_DCLK_FREQ 800 2002 2003 static int get_dclk_freq(void) 2004 { 2005 int dclk_freq = 0; 2006 2007 dmi_walk(decode_dclk, (void *)&dclk_freq); 2008 2009 if (dclk_freq < 1) 2010 return DEFAULT_DCLK_FREQ; 2011 2012 return dclk_freq; 2013 } 2014 2015 /* 2016 * set_sdram_scrub_rate This routine sets byte/sec bandwidth scrub rate 2017 * to hardware according to SCRUBINTERVAL formula 2018 * found in datasheet. 2019 */ 2020 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw) 2021 { 2022 struct i7core_pvt *pvt = mci->pvt_info; 2023 struct pci_dev *pdev; 2024 u32 dw_scrub; 2025 u32 dw_ssr; 2026 2027 /* Get data from the MC register, function 2 */ 2028 pdev = pvt->pci_mcr[2]; 2029 if (!pdev) 2030 return -ENODEV; 2031 2032 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub); 2033 2034 if (new_bw == 0) { 2035 /* Prepare to disable petrol scrub */ 2036 dw_scrub &= ~STARTSCRUB; 2037 /* Stop the patrol scrub engine */ 2038 write_and_test(pdev, MC_SCRUB_CONTROL, 2039 dw_scrub & ~SCRUBINTERVAL_MASK); 2040 2041 /* Get current status of scrub rate and set bit to disable */ 2042 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr); 2043 dw_ssr &= ~SSR_MODE_MASK; 2044 dw_ssr |= SSR_MODE_DISABLE; 2045 } else { 2046 const int cache_line_size = 64; 2047 const u32 freq_dclk_mhz = pvt->dclk_freq; 2048 unsigned long long scrub_interval; 2049 /* 2050 * Translate the desired scrub rate to a register value and 2051 * program the corresponding register value. 2052 */ 2053 scrub_interval = (unsigned long long)freq_dclk_mhz * 2054 cache_line_size * 1000000; 2055 do_div(scrub_interval, new_bw); 2056 2057 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK) 2058 return -EINVAL; 2059 2060 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval; 2061 2062 /* Start the patrol scrub engine */ 2063 pci_write_config_dword(pdev, MC_SCRUB_CONTROL, 2064 STARTSCRUB | dw_scrub); 2065 2066 /* Get current status of scrub rate and set bit to enable */ 2067 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr); 2068 dw_ssr &= ~SSR_MODE_MASK; 2069 dw_ssr |= SSR_MODE_ENABLE; 2070 } 2071 /* Disable or enable scrubbing */ 2072 pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr); 2073 2074 return new_bw; 2075 } 2076 2077 /* 2078 * get_sdram_scrub_rate This routine convert current scrub rate value 2079 * into byte/sec bandwidth according to 2080 * SCRUBINTERVAL formula found in datasheet. 2081 */ 2082 static int get_sdram_scrub_rate(struct mem_ctl_info *mci) 2083 { 2084 struct i7core_pvt *pvt = mci->pvt_info; 2085 struct pci_dev *pdev; 2086 const u32 cache_line_size = 64; 2087 const u32 freq_dclk_mhz = pvt->dclk_freq; 2088 unsigned long long scrub_rate; 2089 u32 scrubval; 2090 2091 /* Get data from the MC register, function 2 */ 2092 pdev = pvt->pci_mcr[2]; 2093 if (!pdev) 2094 return -ENODEV; 2095 2096 /* Get current scrub control data */ 2097 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval); 2098 2099 /* Mask highest 8-bits to 0 */ 2100 scrubval &= SCRUBINTERVAL_MASK; 2101 if (!scrubval) 2102 return 0; 2103 2104 /* Calculate scrub rate value into byte/sec bandwidth */ 2105 scrub_rate = (unsigned long long)freq_dclk_mhz * 2106 1000000 * cache_line_size; 2107 do_div(scrub_rate, scrubval); 2108 return (int)scrub_rate; 2109 } 2110 2111 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci) 2112 { 2113 struct i7core_pvt *pvt = mci->pvt_info; 2114 u32 pci_lock; 2115 2116 /* Unlock writes to pci registers */ 2117 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock); 2118 pci_lock &= ~0x3; 2119 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, 2120 pci_lock | MC_CFG_UNLOCK); 2121 2122 mci->set_sdram_scrub_rate = set_sdram_scrub_rate; 2123 mci->get_sdram_scrub_rate = get_sdram_scrub_rate; 2124 } 2125 2126 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci) 2127 { 2128 struct i7core_pvt *pvt = mci->pvt_info; 2129 u32 pci_lock; 2130 2131 /* Lock writes to pci registers */ 2132 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock); 2133 pci_lock &= ~0x3; 2134 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, 2135 pci_lock | MC_CFG_LOCK); 2136 } 2137 2138 static void i7core_pci_ctl_create(struct i7core_pvt *pvt) 2139 { 2140 pvt->i7core_pci = edac_pci_create_generic_ctl( 2141 &pvt->i7core_dev->pdev[0]->dev, 2142 EDAC_MOD_STR); 2143 if (unlikely(!pvt->i7core_pci)) 2144 i7core_printk(KERN_WARNING, 2145 "Unable to setup PCI error report via EDAC\n"); 2146 } 2147 2148 static void i7core_pci_ctl_release(struct i7core_pvt *pvt) 2149 { 2150 if (likely(pvt->i7core_pci)) 2151 edac_pci_release_generic_ctl(pvt->i7core_pci); 2152 else 2153 i7core_printk(KERN_ERR, 2154 "Couldn't find mem_ctl_info for socket %d\n", 2155 pvt->i7core_dev->socket); 2156 pvt->i7core_pci = NULL; 2157 } 2158 2159 static void i7core_unregister_mci(struct i7core_dev *i7core_dev) 2160 { 2161 struct mem_ctl_info *mci = i7core_dev->mci; 2162 struct i7core_pvt *pvt; 2163 2164 if (unlikely(!mci || !mci->pvt_info)) { 2165 edac_dbg(0, "MC: dev = %p\n", &i7core_dev->pdev[0]->dev); 2166 2167 i7core_printk(KERN_ERR, "Couldn't find mci handler\n"); 2168 return; 2169 } 2170 2171 pvt = mci->pvt_info; 2172 2173 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev); 2174 2175 /* Disable scrubrate setting */ 2176 if (pvt->enable_scrub) 2177 disable_sdram_scrub_setting(mci); 2178 2179 /* Disable EDAC polling */ 2180 i7core_pci_ctl_release(pvt); 2181 2182 /* Remove MC sysfs nodes */ 2183 i7core_delete_sysfs_devices(mci); 2184 edac_mc_del_mc(mci->pdev); 2185 2186 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 2187 kfree(mci->ctl_name); 2188 edac_mc_free(mci); 2189 i7core_dev->mci = NULL; 2190 } 2191 2192 static int i7core_register_mci(struct i7core_dev *i7core_dev) 2193 { 2194 struct mem_ctl_info *mci; 2195 struct i7core_pvt *pvt; 2196 int rc; 2197 struct edac_mc_layer layers[2]; 2198 2199 /* allocate a new MC control structure */ 2200 2201 layers[0].type = EDAC_MC_LAYER_CHANNEL; 2202 layers[0].size = NUM_CHANS; 2203 layers[0].is_virt_csrow = false; 2204 layers[1].type = EDAC_MC_LAYER_SLOT; 2205 layers[1].size = MAX_DIMMS; 2206 layers[1].is_virt_csrow = true; 2207 mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers, 2208 sizeof(*pvt)); 2209 if (unlikely(!mci)) 2210 return -ENOMEM; 2211 2212 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev); 2213 2214 pvt = mci->pvt_info; 2215 memset(pvt, 0, sizeof(*pvt)); 2216 2217 /* Associates i7core_dev and mci for future usage */ 2218 pvt->i7core_dev = i7core_dev; 2219 i7core_dev->mci = mci; 2220 2221 /* 2222 * FIXME: how to handle RDDR3 at MCI level? It is possible to have 2223 * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different 2224 * memory channels 2225 */ 2226 mci->mtype_cap = MEM_FLAG_DDR3; 2227 mci->edac_ctl_cap = EDAC_FLAG_NONE; 2228 mci->edac_cap = EDAC_FLAG_NONE; 2229 mci->mod_name = "i7core_edac.c"; 2230 mci->mod_ver = I7CORE_REVISION; 2231 mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d", 2232 i7core_dev->socket); 2233 mci->dev_name = pci_name(i7core_dev->pdev[0]); 2234 mci->ctl_page_to_phys = NULL; 2235 2236 /* Store pci devices at mci for faster access */ 2237 rc = mci_bind_devs(mci, i7core_dev); 2238 if (unlikely(rc < 0)) 2239 goto fail0; 2240 2241 2242 /* Get dimm basic config */ 2243 get_dimm_config(mci); 2244 /* record ptr to the generic device */ 2245 mci->pdev = &i7core_dev->pdev[0]->dev; 2246 /* Set the function pointer to an actual operation function */ 2247 mci->edac_check = i7core_check_error; 2248 2249 /* Enable scrubrate setting */ 2250 if (pvt->enable_scrub) 2251 enable_sdram_scrub_setting(mci); 2252 2253 /* add this new MC control structure to EDAC's list of MCs */ 2254 if (unlikely(edac_mc_add_mc_with_groups(mci, i7core_dev_groups))) { 2255 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 2256 /* FIXME: perhaps some code should go here that disables error 2257 * reporting if we just enabled it 2258 */ 2259 2260 rc = -EINVAL; 2261 goto fail0; 2262 } 2263 if (i7core_create_sysfs_devices(mci)) { 2264 edac_dbg(0, "MC: failed to create sysfs nodes\n"); 2265 edac_mc_del_mc(mci->pdev); 2266 rc = -EINVAL; 2267 goto fail0; 2268 } 2269 2270 /* Default error mask is any memory */ 2271 pvt->inject.channel = 0; 2272 pvt->inject.dimm = -1; 2273 pvt->inject.rank = -1; 2274 pvt->inject.bank = -1; 2275 pvt->inject.page = -1; 2276 pvt->inject.col = -1; 2277 2278 /* allocating generic PCI control info */ 2279 i7core_pci_ctl_create(pvt); 2280 2281 /* DCLK for scrub rate setting */ 2282 pvt->dclk_freq = get_dclk_freq(); 2283 2284 return 0; 2285 2286 fail0: 2287 kfree(mci->ctl_name); 2288 edac_mc_free(mci); 2289 i7core_dev->mci = NULL; 2290 return rc; 2291 } 2292 2293 /* 2294 * i7core_probe Probe for ONE instance of device to see if it is 2295 * present. 2296 * return: 2297 * 0 for FOUND a device 2298 * < 0 for error code 2299 */ 2300 2301 static int i7core_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2302 { 2303 int rc, count = 0; 2304 struct i7core_dev *i7core_dev; 2305 2306 /* get the pci devices we want to reserve for our use */ 2307 mutex_lock(&i7core_edac_lock); 2308 2309 /* 2310 * All memory controllers are allocated at the first pass. 2311 */ 2312 if (unlikely(probed >= 1)) { 2313 mutex_unlock(&i7core_edac_lock); 2314 return -ENODEV; 2315 } 2316 probed++; 2317 2318 rc = i7core_get_all_devices(); 2319 if (unlikely(rc < 0)) 2320 goto fail0; 2321 2322 list_for_each_entry(i7core_dev, &i7core_edac_list, list) { 2323 count++; 2324 rc = i7core_register_mci(i7core_dev); 2325 if (unlikely(rc < 0)) 2326 goto fail1; 2327 } 2328 2329 /* 2330 * Nehalem-EX uses a different memory controller. However, as the 2331 * memory controller is not visible on some Nehalem/Nehalem-EP, we 2332 * need to indirectly probe via a X58 PCI device. The same devices 2333 * are found on (some) Nehalem-EX. So, on those machines, the 2334 * probe routine needs to return -ENODEV, as the actual Memory 2335 * Controller registers won't be detected. 2336 */ 2337 if (!count) { 2338 rc = -ENODEV; 2339 goto fail1; 2340 } 2341 2342 i7core_printk(KERN_INFO, 2343 "Driver loaded, %d memory controller(s) found.\n", 2344 count); 2345 2346 mutex_unlock(&i7core_edac_lock); 2347 return 0; 2348 2349 fail1: 2350 list_for_each_entry(i7core_dev, &i7core_edac_list, list) 2351 i7core_unregister_mci(i7core_dev); 2352 2353 i7core_put_all_devices(); 2354 fail0: 2355 mutex_unlock(&i7core_edac_lock); 2356 return rc; 2357 } 2358 2359 /* 2360 * i7core_remove destructor for one instance of device 2361 * 2362 */ 2363 static void i7core_remove(struct pci_dev *pdev) 2364 { 2365 struct i7core_dev *i7core_dev; 2366 2367 edac_dbg(0, "\n"); 2368 2369 /* 2370 * we have a trouble here: pdev value for removal will be wrong, since 2371 * it will point to the X58 register used to detect that the machine 2372 * is a Nehalem or upper design. However, due to the way several PCI 2373 * devices are grouped together to provide MC functionality, we need 2374 * to use a different method for releasing the devices 2375 */ 2376 2377 mutex_lock(&i7core_edac_lock); 2378 2379 if (unlikely(!probed)) { 2380 mutex_unlock(&i7core_edac_lock); 2381 return; 2382 } 2383 2384 list_for_each_entry(i7core_dev, &i7core_edac_list, list) 2385 i7core_unregister_mci(i7core_dev); 2386 2387 /* Release PCI resources */ 2388 i7core_put_all_devices(); 2389 2390 probed--; 2391 2392 mutex_unlock(&i7core_edac_lock); 2393 } 2394 2395 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl); 2396 2397 /* 2398 * i7core_driver pci_driver structure for this module 2399 * 2400 */ 2401 static struct pci_driver i7core_driver = { 2402 .name = "i7core_edac", 2403 .probe = i7core_probe, 2404 .remove = i7core_remove, 2405 .id_table = i7core_pci_tbl, 2406 }; 2407 2408 /* 2409 * i7core_init Module entry function 2410 * Try to initialize this module for its devices 2411 */ 2412 static int __init i7core_init(void) 2413 { 2414 int pci_rc; 2415 2416 edac_dbg(2, "\n"); 2417 2418 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 2419 opstate_init(); 2420 2421 if (use_pci_fixup) 2422 i7core_xeon_pci_fixup(pci_dev_table); 2423 2424 pci_rc = pci_register_driver(&i7core_driver); 2425 2426 if (pci_rc >= 0) { 2427 mce_register_decode_chain(&i7_mce_dec); 2428 return 0; 2429 } 2430 2431 i7core_printk(KERN_ERR, "Failed to register device with error %d.\n", 2432 pci_rc); 2433 2434 return pci_rc; 2435 } 2436 2437 /* 2438 * i7core_exit() Module exit function 2439 * Unregister the driver 2440 */ 2441 static void __exit i7core_exit(void) 2442 { 2443 edac_dbg(2, "\n"); 2444 pci_unregister_driver(&i7core_driver); 2445 mce_unregister_decode_chain(&i7_mce_dec); 2446 } 2447 2448 module_init(i7core_init); 2449 module_exit(i7core_exit); 2450 2451 MODULE_LICENSE("GPL"); 2452 MODULE_AUTHOR("Mauro Carvalho Chehab"); 2453 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)"); 2454 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - " 2455 I7CORE_REVISION); 2456 2457 module_param(edac_op_state, int, 0444); 2458 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2459