1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Intel i7 core/Nehalem Memory Controller kernel module
3 *
4 * This driver supports the memory controllers found on the Intel
5 * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
6 * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
7 * and Westmere-EP.
8 *
9 * Copyright (c) 2009-2010 by:
10 * Mauro Carvalho Chehab
11 *
12 * Red Hat Inc. https://www.redhat.com
13 *
14 * Forked and adapted from the i5400_edac driver
15 *
16 * Based on the following public Intel datasheets:
17 * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
18 * Datasheet, Volume 2:
19 * http://download.intel.com/design/processor/datashts/320835.pdf
20 * Intel Xeon Processor 5500 Series Datasheet Volume 2
21 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf
22 * also available at:
23 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
24 */
25
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/pci.h>
29 #include <linux/pci_ids.h>
30 #include <linux/slab.h>
31 #include <linux/delay.h>
32 #include <linux/dmi.h>
33 #include <linux/edac.h>
34 #include <linux/mmzone.h>
35 #include <linux/smp.h>
36 #include <asm/mce.h>
37 #include <asm/processor.h>
38 #include <asm/div64.h>
39
40 #include "edac_module.h"
41
42 /* Static vars */
43 static LIST_HEAD(i7core_edac_list);
44 static DEFINE_MUTEX(i7core_edac_lock);
45 static int probed;
46
47 static int use_pci_fixup;
48 module_param(use_pci_fixup, int, 0444);
49 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
50 /*
51 * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
52 * registers start at bus 255, and are not reported by BIOS.
53 * We currently find devices with only 2 sockets. In order to support more QPI
54 * Quick Path Interconnect, just increment this number.
55 */
56 #define MAX_SOCKET_BUSES 2
57
58
59 /*
60 * Alter this version for the module when modifications are made
61 */
62 #define I7CORE_REVISION " Ver: 1.0.0"
63 #define EDAC_MOD_STR "i7core_edac"
64
65 /*
66 * Debug macros
67 */
68 #define i7core_printk(level, fmt, arg...) \
69 edac_printk(level, "i7core", fmt, ##arg)
70
71 #define i7core_mc_printk(mci, level, fmt, arg...) \
72 edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
73
74 /*
75 * i7core Memory Controller Registers
76 */
77
78 /* OFFSETS for Device 0 Function 0 */
79
80 #define MC_CFG_CONTROL 0x90
81 #define MC_CFG_UNLOCK 0x02
82 #define MC_CFG_LOCK 0x00
83
84 /* OFFSETS for Device 3 Function 0 */
85
86 #define MC_CONTROL 0x48
87 #define MC_STATUS 0x4c
88 #define MC_MAX_DOD 0x64
89
90 /*
91 * OFFSETS for Device 3 Function 4, as indicated on Xeon 5500 datasheet:
92 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
93 */
94
95 #define MC_TEST_ERR_RCV1 0x60
96 #define DIMM2_COR_ERR(r) ((r) & 0x7fff)
97
98 #define MC_TEST_ERR_RCV0 0x64
99 #define DIMM1_COR_ERR(r) (((r) >> 16) & 0x7fff)
100 #define DIMM0_COR_ERR(r) ((r) & 0x7fff)
101
102 /* OFFSETS for Device 3 Function 2, as indicated on Xeon 5500 datasheet */
103 #define MC_SSRCONTROL 0x48
104 #define SSR_MODE_DISABLE 0x00
105 #define SSR_MODE_ENABLE 0x01
106 #define SSR_MODE_MASK 0x03
107
108 #define MC_SCRUB_CONTROL 0x4c
109 #define STARTSCRUB (1 << 24)
110 #define SCRUBINTERVAL_MASK 0xffffff
111
112 #define MC_COR_ECC_CNT_0 0x80
113 #define MC_COR_ECC_CNT_1 0x84
114 #define MC_COR_ECC_CNT_2 0x88
115 #define MC_COR_ECC_CNT_3 0x8c
116 #define MC_COR_ECC_CNT_4 0x90
117 #define MC_COR_ECC_CNT_5 0x94
118
119 #define DIMM_TOP_COR_ERR(r) (((r) >> 16) & 0x7fff)
120 #define DIMM_BOT_COR_ERR(r) ((r) & 0x7fff)
121
122
123 /* OFFSETS for Devices 4,5 and 6 Function 0 */
124
125 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
126 #define THREE_DIMMS_PRESENT (1 << 24)
127 #define SINGLE_QUAD_RANK_PRESENT (1 << 23)
128 #define QUAD_RANK_PRESENT (1 << 22)
129 #define REGISTERED_DIMM (1 << 15)
130
131 #define MC_CHANNEL_MAPPER 0x60
132 #define RDLCH(r, ch) ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
133 #define WRLCH(r, ch) ((((r) >> (ch * 6)) & 0x07) - 1)
134
135 #define MC_CHANNEL_RANK_PRESENT 0x7c
136 #define RANK_PRESENT_MASK 0xffff
137
138 #define MC_CHANNEL_ADDR_MATCH 0xf0
139 #define MC_CHANNEL_ERROR_MASK 0xf8
140 #define MC_CHANNEL_ERROR_INJECT 0xfc
141 #define INJECT_ADDR_PARITY 0x10
142 #define INJECT_ECC 0x08
143 #define MASK_CACHELINE 0x06
144 #define MASK_FULL_CACHELINE 0x06
145 #define MASK_MSB32_CACHELINE 0x04
146 #define MASK_LSB32_CACHELINE 0x02
147 #define NO_MASK_CACHELINE 0x00
148 #define REPEAT_EN 0x01
149
150 /* OFFSETS for Devices 4,5 and 6 Function 1 */
151
152 #define MC_DOD_CH_DIMM0 0x48
153 #define MC_DOD_CH_DIMM1 0x4c
154 #define MC_DOD_CH_DIMM2 0x50
155 #define RANKOFFSET_MASK ((1 << 12) | (1 << 11) | (1 << 10))
156 #define RANKOFFSET(x) ((x & RANKOFFSET_MASK) >> 10)
157 #define DIMM_PRESENT_MASK (1 << 9)
158 #define DIMM_PRESENT(x) (((x) & DIMM_PRESENT_MASK) >> 9)
159 #define MC_DOD_NUMBANK_MASK ((1 << 8) | (1 << 7))
160 #define MC_DOD_NUMBANK(x) (((x) & MC_DOD_NUMBANK_MASK) >> 7)
161 #define MC_DOD_NUMRANK_MASK ((1 << 6) | (1 << 5))
162 #define MC_DOD_NUMRANK(x) (((x) & MC_DOD_NUMRANK_MASK) >> 5)
163 #define MC_DOD_NUMROW_MASK ((1 << 4) | (1 << 3) | (1 << 2))
164 #define MC_DOD_NUMROW(x) (((x) & MC_DOD_NUMROW_MASK) >> 2)
165 #define MC_DOD_NUMCOL_MASK 3
166 #define MC_DOD_NUMCOL(x) ((x) & MC_DOD_NUMCOL_MASK)
167
168 #define MC_RANK_PRESENT 0x7c
169
170 #define MC_SAG_CH_0 0x80
171 #define MC_SAG_CH_1 0x84
172 #define MC_SAG_CH_2 0x88
173 #define MC_SAG_CH_3 0x8c
174 #define MC_SAG_CH_4 0x90
175 #define MC_SAG_CH_5 0x94
176 #define MC_SAG_CH_6 0x98
177 #define MC_SAG_CH_7 0x9c
178
179 #define MC_RIR_LIMIT_CH_0 0x40
180 #define MC_RIR_LIMIT_CH_1 0x44
181 #define MC_RIR_LIMIT_CH_2 0x48
182 #define MC_RIR_LIMIT_CH_3 0x4C
183 #define MC_RIR_LIMIT_CH_4 0x50
184 #define MC_RIR_LIMIT_CH_5 0x54
185 #define MC_RIR_LIMIT_CH_6 0x58
186 #define MC_RIR_LIMIT_CH_7 0x5C
187 #define MC_RIR_LIMIT_MASK ((1 << 10) - 1)
188
189 #define MC_RIR_WAY_CH 0x80
190 #define MC_RIR_WAY_OFFSET_MASK (((1 << 14) - 1) & ~0x7)
191 #define MC_RIR_WAY_RANK_MASK 0x7
192
193 /*
194 * i7core structs
195 */
196
197 #define NUM_CHANS 3
198 #define MAX_DIMMS 3 /* Max DIMMS per channel */
199 #define MAX_MCR_FUNC 4
200 #define MAX_CHAN_FUNC 3
201
202 struct i7core_info {
203 u32 mc_control;
204 u32 mc_status;
205 u32 max_dod;
206 u32 ch_map;
207 };
208
209
210 struct i7core_inject {
211 int enable;
212
213 u32 section;
214 u32 type;
215 u32 eccmask;
216
217 /* Error address mask */
218 int channel, dimm, rank, bank, page, col;
219 };
220
221 struct i7core_channel {
222 bool is_3dimms_present;
223 bool is_single_4rank;
224 bool has_4rank;
225 u32 dimms;
226 };
227
228 struct pci_id_descr {
229 int dev;
230 int func;
231 int dev_id;
232 int optional;
233 };
234
235 struct pci_id_table {
236 const struct pci_id_descr *descr;
237 int n_devs;
238 };
239
240 struct i7core_dev {
241 struct list_head list;
242 u8 socket;
243 struct pci_dev **pdev;
244 int n_devs;
245 struct mem_ctl_info *mci;
246 };
247
248 struct i7core_pvt {
249 struct device *addrmatch_dev, *chancounts_dev;
250
251 struct pci_dev *pci_noncore;
252 struct pci_dev *pci_mcr[MAX_MCR_FUNC + 1];
253 struct pci_dev *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
254
255 struct i7core_dev *i7core_dev;
256
257 struct i7core_info info;
258 struct i7core_inject inject;
259 struct i7core_channel channel[NUM_CHANS];
260
261 int ce_count_available;
262
263 /* ECC corrected errors counts per udimm */
264 unsigned long udimm_ce_count[MAX_DIMMS];
265 int udimm_last_ce_count[MAX_DIMMS];
266 /* ECC corrected errors counts per rdimm */
267 unsigned long rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
268 int rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
269
270 bool is_registered, enable_scrub;
271
272 /* DCLK Frequency used for computing scrub rate */
273 int dclk_freq;
274
275 /* Struct to control EDAC polling */
276 struct edac_pci_ctl_info *i7core_pci;
277 };
278
279 #define PCI_DESCR(device, function, device_id) \
280 .dev = (device), \
281 .func = (function), \
282 .dev_id = (device_id)
283
284 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
285 /* Memory controller */
286 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR) },
287 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD) },
288 /* Exists only for RDIMM */
289 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1 },
290 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
291
292 /* Channel 0 */
293 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
294 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
295 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
296 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC) },
297
298 /* Channel 1 */
299 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
300 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
301 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
302 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC) },
303
304 /* Channel 2 */
305 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
306 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
307 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
308 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC) },
309
310 /* Generic Non-core registers */
311 /*
312 * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
313 * On Xeon 55xx, however, it has a different id (8086:2c40). So,
314 * the probing code needs to test for the other address in case of
315 * failure of this one
316 */
317 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE) },
318
319 };
320
321 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
322 { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR) },
323 { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD) },
324 { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST) },
325
326 { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
327 { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
328 { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
329 { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC) },
330
331 { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
332 { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
333 { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
334 { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC) },
335
336 /*
337 * This is the PCI device has an alternate address on some
338 * processors like Core i7 860
339 */
340 { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE) },
341 };
342
343 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
344 /* Memory controller */
345 { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2) },
346 { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2) },
347 /* Exists only for RDIMM */
348 { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1 },
349 { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
350
351 /* Channel 0 */
352 { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
353 { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
354 { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
355 { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2) },
356
357 /* Channel 1 */
358 { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
359 { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
360 { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
361 { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2) },
362
363 /* Channel 2 */
364 { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
365 { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
366 { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
367 { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2) },
368
369 /* Generic Non-core registers */
370 { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2) },
371
372 };
373
374 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
375 static const struct pci_id_table pci_dev_table[] = {
376 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
377 PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
378 PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
379 { NULL, }
380 };
381
382 /*
383 * pci_device_id table for which devices we are looking for
384 */
385 static const struct pci_device_id i7core_pci_tbl[] = {
386 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
387 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
388 { 0, }
389 };
390
391 /****************************************************************************
392 Ancillary status routines
393 ****************************************************************************/
394
395 /* MC_CONTROL bits */
396 #define CH_ACTIVE(pvt, ch) ((pvt)->info.mc_control & (1 << (8 + ch)))
397 #define ECCx8(pvt) ((pvt)->info.mc_control & (1 << 1))
398
399 /* MC_STATUS bits */
400 #define ECC_ENABLED(pvt) ((pvt)->info.mc_status & (1 << 4))
401 #define CH_DISABLED(pvt, ch) ((pvt)->info.mc_status & (1 << ch))
402
403 /* MC_MAX_DOD read functions */
numdimms(u32 dimms)404 static inline int numdimms(u32 dimms)
405 {
406 return (dimms & 0x3) + 1;
407 }
408
numrank(u32 rank)409 static inline int numrank(u32 rank)
410 {
411 static const int ranks[] = { 1, 2, 4, -EINVAL };
412
413 return ranks[rank & 0x3];
414 }
415
numbank(u32 bank)416 static inline int numbank(u32 bank)
417 {
418 static const int banks[] = { 4, 8, 16, -EINVAL };
419
420 return banks[bank & 0x3];
421 }
422
numrow(u32 row)423 static inline int numrow(u32 row)
424 {
425 static const int rows[] = {
426 1 << 12, 1 << 13, 1 << 14, 1 << 15,
427 1 << 16, -EINVAL, -EINVAL, -EINVAL,
428 };
429
430 return rows[row & 0x7];
431 }
432
numcol(u32 col)433 static inline int numcol(u32 col)
434 {
435 static const int cols[] = {
436 1 << 10, 1 << 11, 1 << 12, -EINVAL,
437 };
438 return cols[col & 0x3];
439 }
440
get_i7core_dev(u8 socket)441 static struct i7core_dev *get_i7core_dev(u8 socket)
442 {
443 struct i7core_dev *i7core_dev;
444
445 list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
446 if (i7core_dev->socket == socket)
447 return i7core_dev;
448 }
449
450 return NULL;
451 }
452
alloc_i7core_dev(u8 socket,const struct pci_id_table * table)453 static struct i7core_dev *alloc_i7core_dev(u8 socket,
454 const struct pci_id_table *table)
455 {
456 struct i7core_dev *i7core_dev;
457
458 i7core_dev = kzalloc_obj(*i7core_dev);
459 if (!i7core_dev)
460 return NULL;
461
462 i7core_dev->pdev = kzalloc_objs(*i7core_dev->pdev, table->n_devs);
463 if (!i7core_dev->pdev) {
464 kfree(i7core_dev);
465 return NULL;
466 }
467
468 i7core_dev->socket = socket;
469 i7core_dev->n_devs = table->n_devs;
470 list_add_tail(&i7core_dev->list, &i7core_edac_list);
471
472 return i7core_dev;
473 }
474
free_i7core_dev(struct i7core_dev * i7core_dev)475 static void free_i7core_dev(struct i7core_dev *i7core_dev)
476 {
477 list_del(&i7core_dev->list);
478 kfree(i7core_dev->pdev);
479 kfree(i7core_dev);
480 }
481
482 /****************************************************************************
483 Memory check routines
484 ****************************************************************************/
485
get_dimm_config(struct mem_ctl_info * mci)486 static int get_dimm_config(struct mem_ctl_info *mci)
487 {
488 struct i7core_pvt *pvt = mci->pvt_info;
489 struct pci_dev *pdev;
490 int i, j;
491 enum edac_type mode;
492 enum mem_type mtype;
493 struct dimm_info *dimm;
494
495 /* Get data from the MC register, function 0 */
496 pdev = pvt->pci_mcr[0];
497 if (!pdev)
498 return -ENODEV;
499
500 /* Device 3 function 0 reads */
501 pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
502 pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
503 pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
504 pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
505
506 edac_dbg(0, "QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
507 pvt->i7core_dev->socket, pvt->info.mc_control,
508 pvt->info.mc_status, pvt->info.max_dod, pvt->info.ch_map);
509
510 if (ECC_ENABLED(pvt)) {
511 edac_dbg(0, "ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
512 if (ECCx8(pvt))
513 mode = EDAC_S8ECD8ED;
514 else
515 mode = EDAC_S4ECD4ED;
516 } else {
517 edac_dbg(0, "ECC disabled\n");
518 mode = EDAC_NONE;
519 }
520
521 /* FIXME: need to handle the error codes */
522 edac_dbg(0, "DOD Max limits: DIMMS: %d, %d-ranked, %d-banked x%x x 0x%x\n",
523 numdimms(pvt->info.max_dod),
524 numrank(pvt->info.max_dod >> 2),
525 numbank(pvt->info.max_dod >> 4),
526 numrow(pvt->info.max_dod >> 6),
527 numcol(pvt->info.max_dod >> 9));
528
529 for (i = 0; i < NUM_CHANS; i++) {
530 u32 data, dimm_dod[3], value[8];
531
532 if (!pvt->pci_ch[i][0])
533 continue;
534
535 if (!CH_ACTIVE(pvt, i)) {
536 edac_dbg(0, "Channel %i is not active\n", i);
537 continue;
538 }
539 if (CH_DISABLED(pvt, i)) {
540 edac_dbg(0, "Channel %i is disabled\n", i);
541 continue;
542 }
543
544 /* Devices 4-6 function 0 */
545 pci_read_config_dword(pvt->pci_ch[i][0],
546 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
547
548
549 if (data & THREE_DIMMS_PRESENT)
550 pvt->channel[i].is_3dimms_present = true;
551
552 if (data & SINGLE_QUAD_RANK_PRESENT)
553 pvt->channel[i].is_single_4rank = true;
554
555 if (data & QUAD_RANK_PRESENT)
556 pvt->channel[i].has_4rank = true;
557
558 if (data & REGISTERED_DIMM)
559 mtype = MEM_RDDR3;
560 else
561 mtype = MEM_DDR3;
562
563 /* Devices 4-6 function 1 */
564 pci_read_config_dword(pvt->pci_ch[i][1],
565 MC_DOD_CH_DIMM0, &dimm_dod[0]);
566 pci_read_config_dword(pvt->pci_ch[i][1],
567 MC_DOD_CH_DIMM1, &dimm_dod[1]);
568 pci_read_config_dword(pvt->pci_ch[i][1],
569 MC_DOD_CH_DIMM2, &dimm_dod[2]);
570
571 edac_dbg(0, "Ch%d phy rd%d, wr%d (0x%08x): %s%s%s%cDIMMs\n",
572 i,
573 RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
574 data,
575 pvt->channel[i].is_3dimms_present ? "3DIMMS " : "",
576 pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "",
577 pvt->channel[i].has_4rank ? "HAS_4R " : "",
578 (data & REGISTERED_DIMM) ? 'R' : 'U');
579
580 for (j = 0; j < 3; j++) {
581 u32 banks, ranks, rows, cols;
582 u32 size, npages;
583
584 if (!DIMM_PRESENT(dimm_dod[j]))
585 continue;
586
587 dimm = edac_get_dimm(mci, i, j, 0);
588 banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
589 ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
590 rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
591 cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
592
593 /* DDR3 has 8 I/O banks */
594 size = (rows * cols * banks * ranks) >> (20 - 3);
595
596 edac_dbg(0, "\tdimm %d %d MiB offset: %x, bank: %d, rank: %d, row: %#x, col: %#x\n",
597 j, size,
598 RANKOFFSET(dimm_dod[j]),
599 banks, ranks, rows, cols);
600
601 npages = MiB_TO_PAGES(size);
602
603 dimm->nr_pages = npages;
604
605 switch (banks) {
606 case 4:
607 dimm->dtype = DEV_X4;
608 break;
609 case 8:
610 dimm->dtype = DEV_X8;
611 break;
612 case 16:
613 dimm->dtype = DEV_X16;
614 break;
615 default:
616 dimm->dtype = DEV_UNKNOWN;
617 }
618
619 snprintf(dimm->label, sizeof(dimm->label),
620 "CPU#%uChannel#%u_DIMM#%u",
621 pvt->i7core_dev->socket, i, j);
622 dimm->grain = 8;
623 dimm->edac_mode = mode;
624 dimm->mtype = mtype;
625 }
626
627 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
628 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
629 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
630 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
631 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
632 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
633 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
634 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
635 edac_dbg(1, "\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
636 for (j = 0; j < 8; j++)
637 edac_dbg(1, "\t\t%#x\t%#x\t%#x\n",
638 (value[j] >> 27) & 0x1,
639 (value[j] >> 24) & 0x7,
640 (value[j] & ((1 << 24) - 1)));
641 }
642
643 return 0;
644 }
645
646 /****************************************************************************
647 Error insertion routines
648 ****************************************************************************/
649
650 #define to_mci(k) container_of(k, struct mem_ctl_info, dev)
651
652 /* The i7core has independent error injection features per channel.
653 However, to have a simpler code, we don't allow enabling error injection
654 on more than one channel.
655 Also, since a change at an inject parameter will be applied only at enable,
656 we're disabling error injection on all write calls to the sysfs nodes that
657 controls the error code injection.
658 */
disable_inject(const struct mem_ctl_info * mci)659 static int disable_inject(const struct mem_ctl_info *mci)
660 {
661 struct i7core_pvt *pvt = mci->pvt_info;
662
663 pvt->inject.enable = 0;
664
665 if (!pvt->pci_ch[pvt->inject.channel][0])
666 return -ENODEV;
667
668 pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
669 MC_CHANNEL_ERROR_INJECT, 0);
670
671 return 0;
672 }
673
674 /*
675 * i7core inject inject.section
676 *
677 * accept and store error injection inject.section value
678 * bit 0 - refers to the lower 32-byte half cacheline
679 * bit 1 - refers to the upper 32-byte half cacheline
680 */
i7core_inject_section_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)681 static ssize_t i7core_inject_section_store(struct device *dev,
682 struct device_attribute *mattr,
683 const char *data, size_t count)
684 {
685 struct mem_ctl_info *mci = to_mci(dev);
686 struct i7core_pvt *pvt = mci->pvt_info;
687 unsigned long value;
688 int rc;
689
690 if (pvt->inject.enable)
691 disable_inject(mci);
692
693 rc = kstrtoul(data, 10, &value);
694 if ((rc < 0) || (value > 3))
695 return -EIO;
696
697 pvt->inject.section = (u32) value;
698 return count;
699 }
700
i7core_inject_section_show(struct device * dev,struct device_attribute * mattr,char * data)701 static ssize_t i7core_inject_section_show(struct device *dev,
702 struct device_attribute *mattr,
703 char *data)
704 {
705 struct mem_ctl_info *mci = to_mci(dev);
706 struct i7core_pvt *pvt = mci->pvt_info;
707 return sprintf(data, "0x%08x\n", pvt->inject.section);
708 }
709
710 /*
711 * i7core inject.type
712 *
713 * accept and store error injection inject.section value
714 * bit 0 - repeat enable - Enable error repetition
715 * bit 1 - inject ECC error
716 * bit 2 - inject parity error
717 */
i7core_inject_type_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)718 static ssize_t i7core_inject_type_store(struct device *dev,
719 struct device_attribute *mattr,
720 const char *data, size_t count)
721 {
722 struct mem_ctl_info *mci = to_mci(dev);
723 struct i7core_pvt *pvt = mci->pvt_info;
724 unsigned long value;
725 int rc;
726
727 if (pvt->inject.enable)
728 disable_inject(mci);
729
730 rc = kstrtoul(data, 10, &value);
731 if ((rc < 0) || (value > 7))
732 return -EIO;
733
734 pvt->inject.type = (u32) value;
735 return count;
736 }
737
i7core_inject_type_show(struct device * dev,struct device_attribute * mattr,char * data)738 static ssize_t i7core_inject_type_show(struct device *dev,
739 struct device_attribute *mattr,
740 char *data)
741 {
742 struct mem_ctl_info *mci = to_mci(dev);
743 struct i7core_pvt *pvt = mci->pvt_info;
744
745 return sprintf(data, "0x%08x\n", pvt->inject.type);
746 }
747
748 /*
749 * i7core_inject_inject.eccmask_store
750 *
751 * The type of error (UE/CE) will depend on the inject.eccmask value:
752 * Any bits set to a 1 will flip the corresponding ECC bit
753 * Correctable errors can be injected by flipping 1 bit or the bits within
754 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
755 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an
756 * uncorrectable error to be injected.
757 */
i7core_inject_eccmask_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)758 static ssize_t i7core_inject_eccmask_store(struct device *dev,
759 struct device_attribute *mattr,
760 const char *data, size_t count)
761 {
762 struct mem_ctl_info *mci = to_mci(dev);
763 struct i7core_pvt *pvt = mci->pvt_info;
764 unsigned long value;
765 int rc;
766
767 if (pvt->inject.enable)
768 disable_inject(mci);
769
770 rc = kstrtoul(data, 10, &value);
771 if (rc < 0)
772 return -EIO;
773
774 pvt->inject.eccmask = (u32) value;
775 return count;
776 }
777
i7core_inject_eccmask_show(struct device * dev,struct device_attribute * mattr,char * data)778 static ssize_t i7core_inject_eccmask_show(struct device *dev,
779 struct device_attribute *mattr,
780 char *data)
781 {
782 struct mem_ctl_info *mci = to_mci(dev);
783 struct i7core_pvt *pvt = mci->pvt_info;
784
785 return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
786 }
787
788 /*
789 * i7core_addrmatch
790 *
791 * The type of error (UE/CE) will depend on the inject.eccmask value:
792 * Any bits set to a 1 will flip the corresponding ECC bit
793 * Correctable errors can be injected by flipping 1 bit or the bits within
794 * a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
795 * 23:16 and 31:24). Flipping bits in two symbol pairs will cause an
796 * uncorrectable error to be injected.
797 */
798
799 #define DECLARE_ADDR_MATCH(param, limit) \
800 static ssize_t i7core_inject_store_##param( \
801 struct device *dev, \
802 struct device_attribute *mattr, \
803 const char *data, size_t count) \
804 { \
805 struct mem_ctl_info *mci = dev_get_drvdata(dev); \
806 struct i7core_pvt *pvt; \
807 long value; \
808 int rc; \
809 \
810 edac_dbg(1, "\n"); \
811 pvt = mci->pvt_info; \
812 \
813 if (pvt->inject.enable) \
814 disable_inject(mci); \
815 \
816 if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
817 value = -1; \
818 else { \
819 rc = kstrtoul(data, 10, &value); \
820 if ((rc < 0) || (value >= limit)) \
821 return -EIO; \
822 } \
823 \
824 pvt->inject.param = value; \
825 \
826 return count; \
827 } \
828 \
829 static ssize_t i7core_inject_show_##param( \
830 struct device *dev, \
831 struct device_attribute *mattr, \
832 char *data) \
833 { \
834 struct mem_ctl_info *mci = dev_get_drvdata(dev); \
835 struct i7core_pvt *pvt; \
836 \
837 pvt = mci->pvt_info; \
838 edac_dbg(1, "pvt=%p\n", pvt); \
839 if (pvt->inject.param < 0) \
840 return sprintf(data, "any\n"); \
841 else \
842 return sprintf(data, "%d\n", pvt->inject.param);\
843 }
844
845 #define ATTR_ADDR_MATCH(param) \
846 static DEVICE_ATTR(param, S_IRUGO | S_IWUSR, \
847 i7core_inject_show_##param, \
848 i7core_inject_store_##param)
849
850 DECLARE_ADDR_MATCH(channel, 3);
851 DECLARE_ADDR_MATCH(dimm, 3);
852 DECLARE_ADDR_MATCH(rank, 4);
853 DECLARE_ADDR_MATCH(bank, 32);
854 DECLARE_ADDR_MATCH(page, 0x10000);
855 DECLARE_ADDR_MATCH(col, 0x4000);
856
857 ATTR_ADDR_MATCH(channel);
858 ATTR_ADDR_MATCH(dimm);
859 ATTR_ADDR_MATCH(rank);
860 ATTR_ADDR_MATCH(bank);
861 ATTR_ADDR_MATCH(page);
862 ATTR_ADDR_MATCH(col);
863
write_and_test(struct pci_dev * dev,const int where,const u32 val)864 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
865 {
866 u32 read;
867 int count;
868
869 edac_dbg(0, "setting pci %02x:%02x.%x reg=%02x value=%08x\n",
870 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
871 where, val);
872
873 for (count = 0; count < 10; count++) {
874 if (count)
875 msleep(100);
876 pci_write_config_dword(dev, where, val);
877 pci_read_config_dword(dev, where, &read);
878
879 if (read == val)
880 return 0;
881 }
882
883 i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
884 "write=%08x. Read=%08x\n",
885 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
886 where, val, read);
887
888 return -EINVAL;
889 }
890
891 /*
892 * This routine prepares the Memory Controller for error injection.
893 * The error will be injected when some process tries to write to the
894 * memory that matches the given criteria.
895 * The criteria can be set in terms of a mask where dimm, rank, bank, page
896 * and col can be specified.
897 * A -1 value for any of the mask items will make the MCU to ignore
898 * that matching criteria for error injection.
899 *
900 * It should be noticed that the error will only happen after a write operation
901 * on a memory that matches the condition. if REPEAT_EN is not enabled at
902 * inject mask, then it will produce just one error. Otherwise, it will repeat
903 * until the injectmask would be cleaned.
904 *
905 * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
906 * is reliable enough to check if the MC is using the
907 * three channels. However, this is not clear at the datasheet.
908 */
i7core_inject_enable_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)909 static ssize_t i7core_inject_enable_store(struct device *dev,
910 struct device_attribute *mattr,
911 const char *data, size_t count)
912 {
913 struct mem_ctl_info *mci = to_mci(dev);
914 struct i7core_pvt *pvt = mci->pvt_info;
915 u32 injectmask;
916 u64 mask = 0;
917 int rc;
918 long enable;
919
920 if (!pvt->pci_ch[pvt->inject.channel][0])
921 return 0;
922
923 rc = kstrtoul(data, 10, &enable);
924 if ((rc < 0))
925 return 0;
926
927 if (enable) {
928 pvt->inject.enable = 1;
929 } else {
930 disable_inject(mci);
931 return count;
932 }
933
934 /* Sets pvt->inject.dimm mask */
935 if (pvt->inject.dimm < 0)
936 mask |= 1LL << 41;
937 else {
938 if (pvt->channel[pvt->inject.channel].dimms > 2)
939 mask |= (pvt->inject.dimm & 0x3LL) << 35;
940 else
941 mask |= (pvt->inject.dimm & 0x1LL) << 36;
942 }
943
944 /* Sets pvt->inject.rank mask */
945 if (pvt->inject.rank < 0)
946 mask |= 1LL << 40;
947 else {
948 if (pvt->channel[pvt->inject.channel].dimms > 2)
949 mask |= (pvt->inject.rank & 0x1LL) << 34;
950 else
951 mask |= (pvt->inject.rank & 0x3LL) << 34;
952 }
953
954 /* Sets pvt->inject.bank mask */
955 if (pvt->inject.bank < 0)
956 mask |= 1LL << 39;
957 else
958 mask |= (pvt->inject.bank & 0x15LL) << 30;
959
960 /* Sets pvt->inject.page mask */
961 if (pvt->inject.page < 0)
962 mask |= 1LL << 38;
963 else
964 mask |= (pvt->inject.page & 0xffff) << 14;
965
966 /* Sets pvt->inject.column mask */
967 if (pvt->inject.col < 0)
968 mask |= 1LL << 37;
969 else
970 mask |= (pvt->inject.col & 0x3fff);
971
972 /*
973 * bit 0: REPEAT_EN
974 * bits 1-2: MASK_HALF_CACHELINE
975 * bit 3: INJECT_ECC
976 * bit 4: INJECT_ADDR_PARITY
977 */
978
979 injectmask = (pvt->inject.type & 1) |
980 (pvt->inject.section & 0x3) << 1 |
981 (pvt->inject.type & 0x6) << (3 - 1);
982
983 /* Unlock writes to registers - this register is write only */
984 pci_write_config_dword(pvt->pci_noncore,
985 MC_CFG_CONTROL, 0x2);
986
987 write_and_test(pvt->pci_ch[pvt->inject.channel][0],
988 MC_CHANNEL_ADDR_MATCH, mask);
989 write_and_test(pvt->pci_ch[pvt->inject.channel][0],
990 MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
991
992 write_and_test(pvt->pci_ch[pvt->inject.channel][0],
993 MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
994
995 write_and_test(pvt->pci_ch[pvt->inject.channel][0],
996 MC_CHANNEL_ERROR_INJECT, injectmask);
997
998 /*
999 * This is something undocumented, based on my tests
1000 * Without writing 8 to this register, errors aren't injected. Not sure
1001 * why.
1002 */
1003 pci_write_config_dword(pvt->pci_noncore,
1004 MC_CFG_CONTROL, 8);
1005
1006 edac_dbg(0, "Error inject addr match 0x%016llx, ecc 0x%08x, inject 0x%08x\n",
1007 mask, pvt->inject.eccmask, injectmask);
1008
1009
1010 return count;
1011 }
1012
i7core_inject_enable_show(struct device * dev,struct device_attribute * mattr,char * data)1013 static ssize_t i7core_inject_enable_show(struct device *dev,
1014 struct device_attribute *mattr,
1015 char *data)
1016 {
1017 struct mem_ctl_info *mci = to_mci(dev);
1018 struct i7core_pvt *pvt = mci->pvt_info;
1019 u32 injectmask;
1020
1021 if (!pvt->pci_ch[pvt->inject.channel][0])
1022 return 0;
1023
1024 pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1025 MC_CHANNEL_ERROR_INJECT, &injectmask);
1026
1027 edac_dbg(0, "Inject error read: 0x%018x\n", injectmask);
1028
1029 if (injectmask & 0x0c)
1030 pvt->inject.enable = 1;
1031
1032 return sprintf(data, "%d\n", pvt->inject.enable);
1033 }
1034
1035 #define DECLARE_COUNTER(param) \
1036 static ssize_t i7core_show_counter_##param( \
1037 struct device *dev, \
1038 struct device_attribute *mattr, \
1039 char *data) \
1040 { \
1041 struct mem_ctl_info *mci = dev_get_drvdata(dev); \
1042 struct i7core_pvt *pvt = mci->pvt_info; \
1043 \
1044 edac_dbg(1, "\n"); \
1045 if (!pvt->ce_count_available || (pvt->is_registered)) \
1046 return sprintf(data, "data unavailable\n"); \
1047 return sprintf(data, "%lu\n", \
1048 pvt->udimm_ce_count[param]); \
1049 }
1050
1051 #define ATTR_COUNTER(param) \
1052 static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR, \
1053 i7core_show_counter_##param, \
1054 NULL)
1055
1056 DECLARE_COUNTER(0);
1057 DECLARE_COUNTER(1);
1058 DECLARE_COUNTER(2);
1059
1060 ATTR_COUNTER(0);
1061 ATTR_COUNTER(1);
1062 ATTR_COUNTER(2);
1063
1064 /*
1065 * inject_addrmatch device sysfs struct
1066 */
1067
1068 static struct attribute *i7core_addrmatch_attrs[] = {
1069 &dev_attr_channel.attr,
1070 &dev_attr_dimm.attr,
1071 &dev_attr_rank.attr,
1072 &dev_attr_bank.attr,
1073 &dev_attr_page.attr,
1074 &dev_attr_col.attr,
1075 NULL
1076 };
1077
1078 static const struct attribute_group addrmatch_grp = {
1079 .attrs = i7core_addrmatch_attrs,
1080 };
1081
1082 static const struct attribute_group *addrmatch_groups[] = {
1083 &addrmatch_grp,
1084 NULL
1085 };
1086
addrmatch_release(struct device * device)1087 static void addrmatch_release(struct device *device)
1088 {
1089 edac_dbg(1, "Releasing device %s\n", dev_name(device));
1090 kfree(device);
1091 }
1092
1093 static const struct device_type addrmatch_type = {
1094 .groups = addrmatch_groups,
1095 .release = addrmatch_release,
1096 };
1097
1098 /*
1099 * all_channel_counts sysfs struct
1100 */
1101
1102 static struct attribute *i7core_udimm_counters_attrs[] = {
1103 &dev_attr_udimm0.attr,
1104 &dev_attr_udimm1.attr,
1105 &dev_attr_udimm2.attr,
1106 NULL
1107 };
1108
1109 static const struct attribute_group all_channel_counts_grp = {
1110 .attrs = i7core_udimm_counters_attrs,
1111 };
1112
1113 static const struct attribute_group *all_channel_counts_groups[] = {
1114 &all_channel_counts_grp,
1115 NULL
1116 };
1117
all_channel_counts_release(struct device * device)1118 static void all_channel_counts_release(struct device *device)
1119 {
1120 edac_dbg(1, "Releasing device %s\n", dev_name(device));
1121 kfree(device);
1122 }
1123
1124 static const struct device_type all_channel_counts_type = {
1125 .groups = all_channel_counts_groups,
1126 .release = all_channel_counts_release,
1127 };
1128
1129 /*
1130 * inject sysfs attributes
1131 */
1132
1133 static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
1134 i7core_inject_section_show, i7core_inject_section_store);
1135
1136 static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR,
1137 i7core_inject_type_show, i7core_inject_type_store);
1138
1139
1140 static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR,
1141 i7core_inject_eccmask_show, i7core_inject_eccmask_store);
1142
1143 static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR,
1144 i7core_inject_enable_show, i7core_inject_enable_store);
1145
1146 static struct attribute *i7core_dev_attrs[] = {
1147 &dev_attr_inject_section.attr,
1148 &dev_attr_inject_type.attr,
1149 &dev_attr_inject_eccmask.attr,
1150 &dev_attr_inject_enable.attr,
1151 NULL
1152 };
1153
1154 ATTRIBUTE_GROUPS(i7core_dev);
1155
i7core_create_sysfs_devices(struct mem_ctl_info * mci)1156 static int i7core_create_sysfs_devices(struct mem_ctl_info *mci)
1157 {
1158 struct i7core_pvt *pvt = mci->pvt_info;
1159 int rc;
1160
1161 pvt->addrmatch_dev = kzalloc_obj(*pvt->addrmatch_dev);
1162 if (!pvt->addrmatch_dev)
1163 return -ENOMEM;
1164
1165 pvt->addrmatch_dev->type = &addrmatch_type;
1166 pvt->addrmatch_dev->bus = mci->dev.bus;
1167 device_initialize(pvt->addrmatch_dev);
1168 pvt->addrmatch_dev->parent = &mci->dev;
1169 dev_set_name(pvt->addrmatch_dev, "inject_addrmatch");
1170 dev_set_drvdata(pvt->addrmatch_dev, mci);
1171
1172 edac_dbg(1, "creating %s\n", dev_name(pvt->addrmatch_dev));
1173
1174 rc = device_add(pvt->addrmatch_dev);
1175 if (rc < 0)
1176 goto err_put_addrmatch;
1177
1178 if (!pvt->is_registered) {
1179 pvt->chancounts_dev = kzalloc_obj(*pvt->chancounts_dev);
1180 if (!pvt->chancounts_dev) {
1181 rc = -ENOMEM;
1182 goto err_del_addrmatch;
1183 }
1184
1185 pvt->chancounts_dev->type = &all_channel_counts_type;
1186 pvt->chancounts_dev->bus = mci->dev.bus;
1187 device_initialize(pvt->chancounts_dev);
1188 pvt->chancounts_dev->parent = &mci->dev;
1189 dev_set_name(pvt->chancounts_dev, "all_channel_counts");
1190 dev_set_drvdata(pvt->chancounts_dev, mci);
1191
1192 edac_dbg(1, "creating %s\n", dev_name(pvt->chancounts_dev));
1193
1194 rc = device_add(pvt->chancounts_dev);
1195 if (rc < 0)
1196 goto err_put_chancounts;
1197 }
1198 return 0;
1199
1200 err_put_chancounts:
1201 put_device(pvt->chancounts_dev);
1202 err_del_addrmatch:
1203 device_del(pvt->addrmatch_dev);
1204 err_put_addrmatch:
1205 put_device(pvt->addrmatch_dev);
1206
1207 return rc;
1208 }
1209
i7core_delete_sysfs_devices(struct mem_ctl_info * mci)1210 static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci)
1211 {
1212 struct i7core_pvt *pvt = mci->pvt_info;
1213
1214 edac_dbg(1, "\n");
1215
1216 if (!pvt->is_registered) {
1217 device_del(pvt->chancounts_dev);
1218 put_device(pvt->chancounts_dev);
1219 }
1220 device_del(pvt->addrmatch_dev);
1221 put_device(pvt->addrmatch_dev);
1222 }
1223
1224 /****************************************************************************
1225 Device initialization routines: put/get, init/exit
1226 ****************************************************************************/
1227
1228 /*
1229 * i7core_put_all_devices 'put' all the devices that we have
1230 * reserved via 'get'
1231 */
i7core_put_devices(struct i7core_dev * i7core_dev)1232 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1233 {
1234 int i;
1235
1236 edac_dbg(0, "\n");
1237 for (i = 0; i < i7core_dev->n_devs; i++) {
1238 struct pci_dev *pdev = i7core_dev->pdev[i];
1239 if (!pdev)
1240 continue;
1241 edac_dbg(0, "Removing dev %02x:%02x.%d\n",
1242 pdev->bus->number,
1243 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1244 pci_dev_put(pdev);
1245 }
1246 }
1247
i7core_put_all_devices(void)1248 static void i7core_put_all_devices(void)
1249 {
1250 struct i7core_dev *i7core_dev, *tmp;
1251
1252 list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1253 i7core_put_devices(i7core_dev);
1254 free_i7core_dev(i7core_dev);
1255 }
1256 }
1257
i7core_xeon_pci_fixup(const struct pci_id_table * table)1258 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1259 {
1260 struct pci_dev *pdev = NULL;
1261 int i;
1262
1263 /*
1264 * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1265 * aren't announced by acpi. So, we need to use a legacy scan probing
1266 * to detect them
1267 */
1268 while (table && table->descr) {
1269 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1270 if (unlikely(!pdev)) {
1271 for (i = 0; i < MAX_SOCKET_BUSES; i++)
1272 pcibios_scan_specific_bus(255-i);
1273 }
1274 pci_dev_put(pdev);
1275 table++;
1276 }
1277 }
1278
i7core_pci_lastbus(void)1279 static unsigned i7core_pci_lastbus(void)
1280 {
1281 int last_bus = 0, bus;
1282 struct pci_bus *b = NULL;
1283
1284 while ((b = pci_find_next_bus(b)) != NULL) {
1285 bus = b->number;
1286 edac_dbg(0, "Found bus %d\n", bus);
1287 if (bus > last_bus)
1288 last_bus = bus;
1289 }
1290
1291 edac_dbg(0, "Last bus %d\n", last_bus);
1292
1293 return last_bus;
1294 }
1295
1296 /*
1297 * i7core_get_all_devices Find and perform 'get' operation on the MCH's
1298 * device/functions we want to reference for this driver
1299 *
1300 * Need to 'get' device 16 func 1 and func 2
1301 */
i7core_get_onedevice(struct pci_dev ** prev,const struct pci_id_table * table,const unsigned devno,const unsigned last_bus)1302 static int i7core_get_onedevice(struct pci_dev **prev,
1303 const struct pci_id_table *table,
1304 const unsigned devno,
1305 const unsigned last_bus)
1306 {
1307 struct i7core_dev *i7core_dev;
1308 const struct pci_id_descr *dev_descr = &table->descr[devno];
1309
1310 struct pci_dev *pdev = NULL;
1311 u8 bus = 0;
1312 u8 socket = 0;
1313
1314 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1315 dev_descr->dev_id, *prev);
1316
1317 /*
1318 * On Xeon 55xx, the Intel QuickPath Arch Generic Non-core regs
1319 * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1320 * to probe for the alternate address in case of failure
1321 */
1322 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev) {
1323 pci_dev_get(*prev); /* pci_get_device will put it */
1324 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1325 PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1326 }
1327
1328 if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE &&
1329 !pdev) {
1330 pci_dev_get(*prev); /* pci_get_device will put it */
1331 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1332 PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1333 *prev);
1334 }
1335
1336 if (!pdev) {
1337 if (*prev) {
1338 *prev = pdev;
1339 return 0;
1340 }
1341
1342 if (dev_descr->optional)
1343 return 0;
1344
1345 if (devno == 0)
1346 return -ENODEV;
1347
1348 i7core_printk(KERN_INFO,
1349 "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1350 dev_descr->dev, dev_descr->func,
1351 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1352
1353 /* End of list, leave */
1354 return -ENODEV;
1355 }
1356 bus = pdev->bus->number;
1357
1358 socket = last_bus - bus;
1359
1360 i7core_dev = get_i7core_dev(socket);
1361 if (!i7core_dev) {
1362 i7core_dev = alloc_i7core_dev(socket, table);
1363 if (!i7core_dev) {
1364 pci_dev_put(pdev);
1365 return -ENOMEM;
1366 }
1367 }
1368
1369 if (i7core_dev->pdev[devno]) {
1370 i7core_printk(KERN_ERR,
1371 "Duplicated device for "
1372 "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1373 bus, dev_descr->dev, dev_descr->func,
1374 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1375 pci_dev_put(pdev);
1376 return -ENODEV;
1377 }
1378
1379 i7core_dev->pdev[devno] = pdev;
1380
1381 /* Sanity check */
1382 if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1383 PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1384 i7core_printk(KERN_ERR,
1385 "Device PCI ID %04x:%04x "
1386 "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1387 PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1388 bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1389 bus, dev_descr->dev, dev_descr->func);
1390 return -ENODEV;
1391 }
1392
1393 /* Be sure that the device is enabled */
1394 if (unlikely(pci_enable_device(pdev) < 0)) {
1395 i7core_printk(KERN_ERR,
1396 "Couldn't enable "
1397 "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1398 bus, dev_descr->dev, dev_descr->func,
1399 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1400 return -ENODEV;
1401 }
1402
1403 edac_dbg(0, "Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1404 socket, bus, dev_descr->dev,
1405 dev_descr->func,
1406 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1407
1408 /*
1409 * As stated on drivers/pci/search.c, the reference count for
1410 * @from is always decremented if it is not %NULL. So, as we need
1411 * to get all devices up to null, we need to do a get for the device
1412 */
1413 pci_dev_get(pdev);
1414
1415 *prev = pdev;
1416
1417 return 0;
1418 }
1419
i7core_get_all_devices(void)1420 static int i7core_get_all_devices(void)
1421 {
1422 int i, rc, last_bus;
1423 struct pci_dev *pdev = NULL;
1424 const struct pci_id_table *table = pci_dev_table;
1425
1426 last_bus = i7core_pci_lastbus();
1427
1428 while (table && table->descr) {
1429 for (i = 0; i < table->n_devs; i++) {
1430 pdev = NULL;
1431 do {
1432 rc = i7core_get_onedevice(&pdev, table, i,
1433 last_bus);
1434 if (rc < 0) {
1435 if (i == 0) {
1436 i = table->n_devs;
1437 break;
1438 }
1439 i7core_put_all_devices();
1440 return -ENODEV;
1441 }
1442 } while (pdev);
1443 }
1444 table++;
1445 }
1446
1447 return 0;
1448 }
1449
mci_bind_devs(struct mem_ctl_info * mci,struct i7core_dev * i7core_dev)1450 static int mci_bind_devs(struct mem_ctl_info *mci,
1451 struct i7core_dev *i7core_dev)
1452 {
1453 struct i7core_pvt *pvt = mci->pvt_info;
1454 struct pci_dev *pdev;
1455 int i, func, slot;
1456 char *family;
1457
1458 pvt->is_registered = false;
1459 pvt->enable_scrub = false;
1460 for (i = 0; i < i7core_dev->n_devs; i++) {
1461 pdev = i7core_dev->pdev[i];
1462 if (!pdev)
1463 continue;
1464
1465 func = PCI_FUNC(pdev->devfn);
1466 slot = PCI_SLOT(pdev->devfn);
1467 if (slot == 3) {
1468 if (unlikely(func > MAX_MCR_FUNC))
1469 goto error;
1470 pvt->pci_mcr[func] = pdev;
1471 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1472 if (unlikely(func > MAX_CHAN_FUNC))
1473 goto error;
1474 pvt->pci_ch[slot - 4][func] = pdev;
1475 } else if (!slot && !func) {
1476 pvt->pci_noncore = pdev;
1477
1478 /* Detect the processor family */
1479 switch (pdev->device) {
1480 case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1481 family = "Xeon 35xx/ i7core";
1482 pvt->enable_scrub = false;
1483 break;
1484 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1485 family = "i7-800/i5-700";
1486 pvt->enable_scrub = false;
1487 break;
1488 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1489 family = "Xeon 34xx";
1490 pvt->enable_scrub = false;
1491 break;
1492 case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1493 family = "Xeon 55xx";
1494 pvt->enable_scrub = true;
1495 break;
1496 case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1497 family = "Xeon 56xx / i7-900";
1498 pvt->enable_scrub = true;
1499 break;
1500 default:
1501 family = "unknown";
1502 pvt->enable_scrub = false;
1503 }
1504 edac_dbg(0, "Detected a processor type %s\n", family);
1505 } else
1506 goto error;
1507
1508 edac_dbg(0, "Associated fn %d.%d, dev = %p, socket %d\n",
1509 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1510 pdev, i7core_dev->socket);
1511
1512 if (PCI_SLOT(pdev->devfn) == 3 &&
1513 PCI_FUNC(pdev->devfn) == 2)
1514 pvt->is_registered = true;
1515 }
1516
1517 return 0;
1518
1519 error:
1520 i7core_printk(KERN_ERR, "Device %d, function %d "
1521 "is out of the expected range\n",
1522 slot, func);
1523 return -EINVAL;
1524 }
1525
1526 /****************************************************************************
1527 Error check routines
1528 ****************************************************************************/
1529
i7core_rdimm_update_ce_count(struct mem_ctl_info * mci,const int chan,const int new0,const int new1,const int new2)1530 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1531 const int chan,
1532 const int new0,
1533 const int new1,
1534 const int new2)
1535 {
1536 struct i7core_pvt *pvt = mci->pvt_info;
1537 int add0 = 0, add1 = 0, add2 = 0;
1538 /* Updates CE counters if it is not the first time here */
1539 if (pvt->ce_count_available) {
1540 /* Updates CE counters */
1541
1542 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1543 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1544 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1545
1546 if (add2 < 0)
1547 add2 += 0x7fff;
1548 pvt->rdimm_ce_count[chan][2] += add2;
1549
1550 if (add1 < 0)
1551 add1 += 0x7fff;
1552 pvt->rdimm_ce_count[chan][1] += add1;
1553
1554 if (add0 < 0)
1555 add0 += 0x7fff;
1556 pvt->rdimm_ce_count[chan][0] += add0;
1557 } else
1558 pvt->ce_count_available = 1;
1559
1560 /* Store the new values */
1561 pvt->rdimm_last_ce_count[chan][2] = new2;
1562 pvt->rdimm_last_ce_count[chan][1] = new1;
1563 pvt->rdimm_last_ce_count[chan][0] = new0;
1564
1565 /*updated the edac core */
1566 if (add0 != 0)
1567 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add0,
1568 0, 0, 0,
1569 chan, 0, -1, "error", "");
1570 if (add1 != 0)
1571 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add1,
1572 0, 0, 0,
1573 chan, 1, -1, "error", "");
1574 if (add2 != 0)
1575 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add2,
1576 0, 0, 0,
1577 chan, 2, -1, "error", "");
1578 }
1579
i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info * mci)1580 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1581 {
1582 struct i7core_pvt *pvt = mci->pvt_info;
1583 u32 rcv[3][2];
1584 int i, new0, new1, new2;
1585
1586 /*Read DEV 3: FUN 2: MC_COR_ECC_CNT regs directly*/
1587 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1588 &rcv[0][0]);
1589 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1590 &rcv[0][1]);
1591 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1592 &rcv[1][0]);
1593 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1594 &rcv[1][1]);
1595 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1596 &rcv[2][0]);
1597 pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1598 &rcv[2][1]);
1599 for (i = 0 ; i < 3; i++) {
1600 edac_dbg(3, "MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1601 (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1602 /*if the channel has 3 dimms*/
1603 if (pvt->channel[i].dimms > 2) {
1604 new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1605 new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1606 new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1607 } else {
1608 new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1609 DIMM_BOT_COR_ERR(rcv[i][0]);
1610 new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1611 DIMM_BOT_COR_ERR(rcv[i][1]);
1612 new2 = 0;
1613 }
1614
1615 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1616 }
1617 }
1618
1619 /* This function is based on the device 3 function 4 registers as described on:
1620 * Intel Xeon Processor 5500 Series Datasheet Volume 2
1621 * http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1622 * also available at:
1623 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1624 */
i7core_udimm_check_mc_ecc_err(struct mem_ctl_info * mci)1625 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1626 {
1627 struct i7core_pvt *pvt = mci->pvt_info;
1628 u32 rcv1, rcv0;
1629 int new0, new1, new2;
1630
1631 if (!pvt->pci_mcr[4]) {
1632 edac_dbg(0, "MCR registers not found\n");
1633 return;
1634 }
1635
1636 /* Corrected test errors */
1637 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1638 pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1639
1640 /* Store the new values */
1641 new2 = DIMM2_COR_ERR(rcv1);
1642 new1 = DIMM1_COR_ERR(rcv0);
1643 new0 = DIMM0_COR_ERR(rcv0);
1644
1645 /* Updates CE counters if it is not the first time here */
1646 if (pvt->ce_count_available) {
1647 /* Updates CE counters */
1648 int add0, add1, add2;
1649
1650 add2 = new2 - pvt->udimm_last_ce_count[2];
1651 add1 = new1 - pvt->udimm_last_ce_count[1];
1652 add0 = new0 - pvt->udimm_last_ce_count[0];
1653
1654 if (add2 < 0)
1655 add2 += 0x7fff;
1656 pvt->udimm_ce_count[2] += add2;
1657
1658 if (add1 < 0)
1659 add1 += 0x7fff;
1660 pvt->udimm_ce_count[1] += add1;
1661
1662 if (add0 < 0)
1663 add0 += 0x7fff;
1664 pvt->udimm_ce_count[0] += add0;
1665
1666 if (add0 | add1 | add2)
1667 i7core_printk(KERN_ERR, "New Corrected error(s): "
1668 "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1669 add0, add1, add2);
1670 } else
1671 pvt->ce_count_available = 1;
1672
1673 /* Store the new values */
1674 pvt->udimm_last_ce_count[2] = new2;
1675 pvt->udimm_last_ce_count[1] = new1;
1676 pvt->udimm_last_ce_count[0] = new0;
1677 }
1678
1679 /*
1680 * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1681 * Architectures Software Developer’s Manual Volume 3B.
1682 * Nehalem are defined as family 0x06, model 0x1a
1683 *
1684 * The MCA registers used here are the following ones:
1685 * struct mce field MCA Register
1686 * m->status MSR_IA32_MC8_STATUS
1687 * m->addr MSR_IA32_MC8_ADDR
1688 * m->misc MSR_IA32_MC8_MISC
1689 * In the case of Nehalem, the error information is masked at .status and .misc
1690 * fields
1691 */
i7core_mce_output_error(struct mem_ctl_info * mci,const struct mce * m)1692 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1693 const struct mce *m)
1694 {
1695 struct i7core_pvt *pvt = mci->pvt_info;
1696 char *optype, *err;
1697 enum hw_event_mc_err_type tp_event;
1698 unsigned long error = m->status & 0x1ff0000l;
1699 bool uncorrected_error = m->mcgstatus & 1ll << 61;
1700 bool ripv = m->mcgstatus & 1;
1701 u32 optypenum = (m->status >> 4) & 0x07;
1702 u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1703 u32 dimm = (m->misc >> 16) & 0x3;
1704 u32 channel = (m->misc >> 18) & 0x3;
1705 u32 syndrome = m->misc >> 32;
1706 u32 errnum = find_first_bit(&error, 32);
1707
1708 if (uncorrected_error) {
1709 core_err_cnt = 1;
1710 if (ripv)
1711 tp_event = HW_EVENT_ERR_UNCORRECTED;
1712 else
1713 tp_event = HW_EVENT_ERR_FATAL;
1714 } else {
1715 tp_event = HW_EVENT_ERR_CORRECTED;
1716 }
1717
1718 switch (optypenum) {
1719 case 0:
1720 optype = "generic undef request";
1721 break;
1722 case 1:
1723 optype = "read error";
1724 break;
1725 case 2:
1726 optype = "write error";
1727 break;
1728 case 3:
1729 optype = "addr/cmd error";
1730 break;
1731 case 4:
1732 optype = "scrubbing error";
1733 break;
1734 default:
1735 optype = "reserved";
1736 break;
1737 }
1738
1739 switch (errnum) {
1740 case 16:
1741 err = "read ECC error";
1742 break;
1743 case 17:
1744 err = "RAS ECC error";
1745 break;
1746 case 18:
1747 err = "write parity error";
1748 break;
1749 case 19:
1750 err = "redundancy loss";
1751 break;
1752 case 20:
1753 err = "reserved";
1754 break;
1755 case 21:
1756 err = "memory range error";
1757 break;
1758 case 22:
1759 err = "RTID out of range";
1760 break;
1761 case 23:
1762 err = "address parity error";
1763 break;
1764 case 24:
1765 err = "byte enable parity error";
1766 break;
1767 default:
1768 err = "unknown";
1769 }
1770
1771 /*
1772 * Call the helper to output message
1773 * FIXME: what to do if core_err_cnt > 1? Currently, it generates
1774 * only one event
1775 */
1776 if (uncorrected_error || !pvt->is_registered)
1777 edac_mc_handle_error(tp_event, mci, core_err_cnt,
1778 m->addr >> PAGE_SHIFT,
1779 m->addr & ~PAGE_MASK,
1780 syndrome,
1781 channel, dimm, -1,
1782 err, optype);
1783 }
1784
1785 /*
1786 * i7core_check_error Retrieve and process errors reported by the
1787 * hardware. Called by the Core module.
1788 */
i7core_check_error(struct mem_ctl_info * mci,struct mce * m)1789 static void i7core_check_error(struct mem_ctl_info *mci, struct mce *m)
1790 {
1791 struct i7core_pvt *pvt = mci->pvt_info;
1792
1793 i7core_mce_output_error(mci, m);
1794
1795 /*
1796 * Now, let's increment CE error counts
1797 */
1798 if (!pvt->is_registered)
1799 i7core_udimm_check_mc_ecc_err(mci);
1800 else
1801 i7core_rdimm_check_mc_ecc_err(mci);
1802 }
1803
1804 /*
1805 * Check that logging is enabled and that this is the right type
1806 * of error for us to handle.
1807 */
i7core_mce_check_error(struct notifier_block * nb,unsigned long val,void * data)1808 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1809 void *data)
1810 {
1811 struct mce *mce = (struct mce *)data;
1812 struct i7core_dev *i7_dev;
1813 struct mem_ctl_info *mci;
1814
1815 i7_dev = get_i7core_dev(mce->socketid);
1816 if (!i7_dev || (mce->kflags & MCE_HANDLED_CEC))
1817 return NOTIFY_DONE;
1818
1819 mci = i7_dev->mci;
1820
1821 /*
1822 * Just let mcelog handle it if the error is
1823 * outside the memory controller
1824 */
1825 if (((mce->status & 0xffff) >> 7) != 1)
1826 return NOTIFY_DONE;
1827
1828 /* Bank 8 registers are the only ones that we know how to handle */
1829 if (mce->bank != 8)
1830 return NOTIFY_DONE;
1831
1832 i7core_check_error(mci, mce);
1833
1834 /* Advise mcelog that the errors were handled */
1835 mce->kflags |= MCE_HANDLED_EDAC;
1836 return NOTIFY_OK;
1837 }
1838
1839 static struct notifier_block i7_mce_dec = {
1840 .notifier_call = i7core_mce_check_error,
1841 .priority = MCE_PRIO_EDAC,
1842 };
1843
1844 struct memdev_dmi_entry {
1845 u8 type;
1846 u8 length;
1847 u16 handle;
1848 u16 phys_mem_array_handle;
1849 u16 mem_err_info_handle;
1850 u16 total_width;
1851 u16 data_width;
1852 u16 size;
1853 u8 form;
1854 u8 device_set;
1855 u8 device_locator;
1856 u8 bank_locator;
1857 u8 memory_type;
1858 u16 type_detail;
1859 u16 speed;
1860 u8 manufacturer;
1861 u8 serial_number;
1862 u8 asset_tag;
1863 u8 part_number;
1864 u8 attributes;
1865 u32 extended_size;
1866 u16 conf_mem_clk_speed;
1867 } __attribute__((__packed__));
1868
1869
1870 /*
1871 * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1872 * memory devices show the same speed, and if they don't then consider
1873 * all speeds to be invalid.
1874 */
decode_dclk(const struct dmi_header * dh,void * _dclk_freq)1875 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1876 {
1877 int *dclk_freq = _dclk_freq;
1878 u16 dmi_mem_clk_speed;
1879
1880 if (*dclk_freq == -1)
1881 return;
1882
1883 if (dh->type == DMI_ENTRY_MEM_DEVICE) {
1884 struct memdev_dmi_entry *memdev_dmi_entry =
1885 (struct memdev_dmi_entry *)dh;
1886 unsigned long conf_mem_clk_speed_offset =
1887 (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
1888 (unsigned long)&memdev_dmi_entry->type;
1889 unsigned long speed_offset =
1890 (unsigned long)&memdev_dmi_entry->speed -
1891 (unsigned long)&memdev_dmi_entry->type;
1892
1893 /* Check that a DIMM is present */
1894 if (memdev_dmi_entry->size == 0)
1895 return;
1896
1897 /*
1898 * Pick the configured speed if it's available, otherwise
1899 * pick the DIMM speed, or we don't have a speed.
1900 */
1901 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
1902 dmi_mem_clk_speed =
1903 memdev_dmi_entry->conf_mem_clk_speed;
1904 } else if (memdev_dmi_entry->length > speed_offset) {
1905 dmi_mem_clk_speed = memdev_dmi_entry->speed;
1906 } else {
1907 *dclk_freq = -1;
1908 return;
1909 }
1910
1911 if (*dclk_freq == 0) {
1912 /* First pass, speed was 0 */
1913 if (dmi_mem_clk_speed > 0) {
1914 /* Set speed if a valid speed is read */
1915 *dclk_freq = dmi_mem_clk_speed;
1916 } else {
1917 /* Otherwise we don't have a valid speed */
1918 *dclk_freq = -1;
1919 }
1920 } else if (*dclk_freq > 0 &&
1921 *dclk_freq != dmi_mem_clk_speed) {
1922 /*
1923 * If we have a speed, check that all DIMMS are the same
1924 * speed, otherwise set the speed as invalid.
1925 */
1926 *dclk_freq = -1;
1927 }
1928 }
1929 }
1930
1931 /*
1932 * The default DCLK frequency is used as a fallback if we
1933 * fail to find anything reliable in the DMI. The value
1934 * is taken straight from the datasheet.
1935 */
1936 #define DEFAULT_DCLK_FREQ 800
1937
get_dclk_freq(void)1938 static int get_dclk_freq(void)
1939 {
1940 int dclk_freq = 0;
1941
1942 dmi_walk(decode_dclk, (void *)&dclk_freq);
1943
1944 if (dclk_freq < 1)
1945 return DEFAULT_DCLK_FREQ;
1946
1947 return dclk_freq;
1948 }
1949
1950 /*
1951 * set_sdram_scrub_rate This routine sets byte/sec bandwidth scrub rate
1952 * to hardware according to SCRUBINTERVAL formula
1953 * found in datasheet.
1954 */
set_sdram_scrub_rate(struct mem_ctl_info * mci,u32 new_bw)1955 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
1956 {
1957 struct i7core_pvt *pvt = mci->pvt_info;
1958 struct pci_dev *pdev;
1959 u32 dw_scrub;
1960 u32 dw_ssr;
1961
1962 /* Get data from the MC register, function 2 */
1963 pdev = pvt->pci_mcr[2];
1964 if (!pdev)
1965 return -ENODEV;
1966
1967 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
1968
1969 if (new_bw == 0) {
1970 /* Prepare to disable petrol scrub */
1971 dw_scrub &= ~STARTSCRUB;
1972 /* Stop the patrol scrub engine */
1973 write_and_test(pdev, MC_SCRUB_CONTROL,
1974 dw_scrub & ~SCRUBINTERVAL_MASK);
1975
1976 /* Get current status of scrub rate and set bit to disable */
1977 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
1978 dw_ssr &= ~SSR_MODE_MASK;
1979 dw_ssr |= SSR_MODE_DISABLE;
1980 } else {
1981 const int cache_line_size = 64;
1982 const u32 freq_dclk_mhz = pvt->dclk_freq;
1983 unsigned long long scrub_interval;
1984 /*
1985 * Translate the desired scrub rate to a register value and
1986 * program the corresponding register value.
1987 */
1988 scrub_interval = (unsigned long long)freq_dclk_mhz *
1989 cache_line_size * 1000000;
1990 do_div(scrub_interval, new_bw);
1991
1992 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
1993 return -EINVAL;
1994
1995 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
1996
1997 /* Start the patrol scrub engine */
1998 pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
1999 STARTSCRUB | dw_scrub);
2000
2001 /* Get current status of scrub rate and set bit to enable */
2002 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2003 dw_ssr &= ~SSR_MODE_MASK;
2004 dw_ssr |= SSR_MODE_ENABLE;
2005 }
2006 /* Disable or enable scrubbing */
2007 pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2008
2009 return new_bw;
2010 }
2011
2012 /*
2013 * get_sdram_scrub_rate This routine convert current scrub rate value
2014 * into byte/sec bandwidth according to
2015 * SCRUBINTERVAL formula found in datasheet.
2016 */
get_sdram_scrub_rate(struct mem_ctl_info * mci)2017 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2018 {
2019 struct i7core_pvt *pvt = mci->pvt_info;
2020 struct pci_dev *pdev;
2021 const u32 cache_line_size = 64;
2022 const u32 freq_dclk_mhz = pvt->dclk_freq;
2023 unsigned long long scrub_rate;
2024 u32 scrubval;
2025
2026 /* Get data from the MC register, function 2 */
2027 pdev = pvt->pci_mcr[2];
2028 if (!pdev)
2029 return -ENODEV;
2030
2031 /* Get current scrub control data */
2032 pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2033
2034 /* Mask highest 8-bits to 0 */
2035 scrubval &= SCRUBINTERVAL_MASK;
2036 if (!scrubval)
2037 return 0;
2038
2039 /* Calculate scrub rate value into byte/sec bandwidth */
2040 scrub_rate = (unsigned long long)freq_dclk_mhz *
2041 1000000 * cache_line_size;
2042 do_div(scrub_rate, scrubval);
2043 return (int)scrub_rate;
2044 }
2045
enable_sdram_scrub_setting(struct mem_ctl_info * mci)2046 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2047 {
2048 struct i7core_pvt *pvt = mci->pvt_info;
2049 u32 pci_lock;
2050
2051 /* Unlock writes to pci registers */
2052 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2053 pci_lock &= ~0x3;
2054 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2055 pci_lock | MC_CFG_UNLOCK);
2056
2057 mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2058 mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2059 }
2060
disable_sdram_scrub_setting(struct mem_ctl_info * mci)2061 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2062 {
2063 struct i7core_pvt *pvt = mci->pvt_info;
2064 u32 pci_lock;
2065
2066 /* Lock writes to pci registers */
2067 pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2068 pci_lock &= ~0x3;
2069 pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2070 pci_lock | MC_CFG_LOCK);
2071 }
2072
i7core_pci_ctl_create(struct i7core_pvt * pvt)2073 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2074 {
2075 pvt->i7core_pci = edac_pci_create_generic_ctl(
2076 &pvt->i7core_dev->pdev[0]->dev,
2077 EDAC_MOD_STR);
2078 if (unlikely(!pvt->i7core_pci))
2079 i7core_printk(KERN_WARNING,
2080 "Unable to setup PCI error report via EDAC\n");
2081 }
2082
i7core_pci_ctl_release(struct i7core_pvt * pvt)2083 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2084 {
2085 if (likely(pvt->i7core_pci))
2086 edac_pci_release_generic_ctl(pvt->i7core_pci);
2087 else
2088 i7core_printk(KERN_ERR,
2089 "Couldn't find mem_ctl_info for socket %d\n",
2090 pvt->i7core_dev->socket);
2091 pvt->i7core_pci = NULL;
2092 }
2093
i7core_unregister_mci(struct i7core_dev * i7core_dev)2094 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2095 {
2096 struct mem_ctl_info *mci = i7core_dev->mci;
2097 struct i7core_pvt *pvt;
2098
2099 if (unlikely(!mci || !mci->pvt_info)) {
2100 edac_dbg(0, "MC: dev = %p\n", &i7core_dev->pdev[0]->dev);
2101
2102 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2103 return;
2104 }
2105
2106 pvt = mci->pvt_info;
2107
2108 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev);
2109
2110 /* Disable scrubrate setting */
2111 if (pvt->enable_scrub)
2112 disable_sdram_scrub_setting(mci);
2113
2114 /* Disable EDAC polling */
2115 i7core_pci_ctl_release(pvt);
2116
2117 /* Remove MC sysfs nodes */
2118 i7core_delete_sysfs_devices(mci);
2119 edac_mc_del_mc(mci->pdev);
2120
2121 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
2122 kfree(mci->ctl_name);
2123 edac_mc_free(mci);
2124 i7core_dev->mci = NULL;
2125 }
2126
i7core_register_mci(struct i7core_dev * i7core_dev)2127 static int i7core_register_mci(struct i7core_dev *i7core_dev)
2128 {
2129 struct mem_ctl_info *mci;
2130 struct i7core_pvt *pvt;
2131 int rc;
2132 struct edac_mc_layer layers[2];
2133
2134 /* allocate a new MC control structure */
2135
2136 layers[0].type = EDAC_MC_LAYER_CHANNEL;
2137 layers[0].size = NUM_CHANS;
2138 layers[0].is_virt_csrow = false;
2139 layers[1].type = EDAC_MC_LAYER_SLOT;
2140 layers[1].size = MAX_DIMMS;
2141 layers[1].is_virt_csrow = true;
2142 mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers,
2143 sizeof(*pvt));
2144 if (unlikely(!mci))
2145 return -ENOMEM;
2146
2147 edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev);
2148
2149 pvt = mci->pvt_info;
2150 memset(pvt, 0, sizeof(*pvt));
2151
2152 /* Associates i7core_dev and mci for future usage */
2153 pvt->i7core_dev = i7core_dev;
2154 i7core_dev->mci = mci;
2155
2156 /*
2157 * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2158 * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2159 * memory channels
2160 */
2161 mci->mtype_cap = MEM_FLAG_DDR3;
2162 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2163 mci->edac_cap = EDAC_FLAG_NONE;
2164 mci->mod_name = "i7core_edac.c";
2165
2166 mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d", i7core_dev->socket);
2167 if (!mci->ctl_name) {
2168 rc = -ENOMEM;
2169 goto fail1;
2170 }
2171
2172 mci->dev_name = pci_name(i7core_dev->pdev[0]);
2173 mci->ctl_page_to_phys = NULL;
2174
2175 /* Store pci devices at mci for faster access */
2176 rc = mci_bind_devs(mci, i7core_dev);
2177 if (unlikely(rc < 0))
2178 goto fail0;
2179
2180
2181 /* Get dimm basic config */
2182 get_dimm_config(mci);
2183 /* record ptr to the generic device */
2184 mci->pdev = &i7core_dev->pdev[0]->dev;
2185
2186 /* Enable scrubrate setting */
2187 if (pvt->enable_scrub)
2188 enable_sdram_scrub_setting(mci);
2189
2190 /* add this new MC control structure to EDAC's list of MCs */
2191 if (unlikely(edac_mc_add_mc_with_groups(mci, i7core_dev_groups))) {
2192 edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
2193 /* FIXME: perhaps some code should go here that disables error
2194 * reporting if we just enabled it
2195 */
2196
2197 rc = -EINVAL;
2198 goto fail0;
2199 }
2200 if (i7core_create_sysfs_devices(mci)) {
2201 edac_dbg(0, "MC: failed to create sysfs nodes\n");
2202 edac_mc_del_mc(mci->pdev);
2203 rc = -EINVAL;
2204 goto fail0;
2205 }
2206
2207 /* Default error mask is any memory */
2208 pvt->inject.channel = 0;
2209 pvt->inject.dimm = -1;
2210 pvt->inject.rank = -1;
2211 pvt->inject.bank = -1;
2212 pvt->inject.page = -1;
2213 pvt->inject.col = -1;
2214
2215 /* allocating generic PCI control info */
2216 i7core_pci_ctl_create(pvt);
2217
2218 /* DCLK for scrub rate setting */
2219 pvt->dclk_freq = get_dclk_freq();
2220
2221 return 0;
2222
2223 fail0:
2224 kfree(mci->ctl_name);
2225
2226 fail1:
2227 edac_mc_free(mci);
2228 i7core_dev->mci = NULL;
2229 return rc;
2230 }
2231
2232 /*
2233 * i7core_probe Probe for ONE instance of device to see if it is
2234 * present.
2235 * return:
2236 * 0 for FOUND a device
2237 * < 0 for error code
2238 */
2239
i7core_probe(struct pci_dev * pdev,const struct pci_device_id * id)2240 static int i7core_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2241 {
2242 int rc, count = 0;
2243 struct i7core_dev *i7core_dev;
2244
2245 /* get the pci devices we want to reserve for our use */
2246 mutex_lock(&i7core_edac_lock);
2247
2248 /*
2249 * All memory controllers are allocated at the first pass.
2250 */
2251 if (unlikely(probed >= 1)) {
2252 mutex_unlock(&i7core_edac_lock);
2253 return -ENODEV;
2254 }
2255 probed++;
2256
2257 rc = i7core_get_all_devices();
2258 if (unlikely(rc < 0))
2259 goto fail0;
2260
2261 list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2262 count++;
2263 rc = i7core_register_mci(i7core_dev);
2264 if (unlikely(rc < 0))
2265 goto fail1;
2266 }
2267
2268 /*
2269 * Nehalem-EX uses a different memory controller. However, as the
2270 * memory controller is not visible on some Nehalem/Nehalem-EP, we
2271 * need to indirectly probe via a X58 PCI device. The same devices
2272 * are found on (some) Nehalem-EX. So, on those machines, the
2273 * probe routine needs to return -ENODEV, as the actual Memory
2274 * Controller registers won't be detected.
2275 */
2276 if (!count) {
2277 rc = -ENODEV;
2278 goto fail1;
2279 }
2280
2281 i7core_printk(KERN_INFO,
2282 "Driver loaded, %d memory controller(s) found.\n",
2283 count);
2284
2285 mutex_unlock(&i7core_edac_lock);
2286 return 0;
2287
2288 fail1:
2289 list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2290 i7core_unregister_mci(i7core_dev);
2291
2292 i7core_put_all_devices();
2293 fail0:
2294 mutex_unlock(&i7core_edac_lock);
2295 return rc;
2296 }
2297
2298 /*
2299 * i7core_remove destructor for one instance of device
2300 *
2301 */
i7core_remove(struct pci_dev * pdev)2302 static void i7core_remove(struct pci_dev *pdev)
2303 {
2304 struct i7core_dev *i7core_dev;
2305
2306 edac_dbg(0, "\n");
2307
2308 /*
2309 * we have a trouble here: pdev value for removal will be wrong, since
2310 * it will point to the X58 register used to detect that the machine
2311 * is a Nehalem or upper design. However, due to the way several PCI
2312 * devices are grouped together to provide MC functionality, we need
2313 * to use a different method for releasing the devices
2314 */
2315
2316 mutex_lock(&i7core_edac_lock);
2317
2318 if (unlikely(!probed)) {
2319 mutex_unlock(&i7core_edac_lock);
2320 return;
2321 }
2322
2323 list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2324 i7core_unregister_mci(i7core_dev);
2325
2326 /* Release PCI resources */
2327 i7core_put_all_devices();
2328
2329 probed--;
2330
2331 mutex_unlock(&i7core_edac_lock);
2332 }
2333
2334 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2335
2336 /*
2337 * i7core_driver pci_driver structure for this module
2338 *
2339 */
2340 static struct pci_driver i7core_driver = {
2341 .name = "i7core_edac",
2342 .probe = i7core_probe,
2343 .remove = i7core_remove,
2344 .id_table = i7core_pci_tbl,
2345 };
2346
2347 /*
2348 * i7core_init Module entry function
2349 * Try to initialize this module for its devices
2350 */
i7core_init(void)2351 static int __init i7core_init(void)
2352 {
2353 int pci_rc;
2354
2355 edac_dbg(2, "\n");
2356
2357 /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2358 opstate_init();
2359
2360 if (use_pci_fixup)
2361 i7core_xeon_pci_fixup(pci_dev_table);
2362
2363 pci_rc = pci_register_driver(&i7core_driver);
2364
2365 if (pci_rc >= 0) {
2366 mce_register_decode_chain(&i7_mce_dec);
2367 return 0;
2368 }
2369
2370 i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2371 pci_rc);
2372
2373 return pci_rc;
2374 }
2375
2376 /*
2377 * i7core_exit() Module exit function
2378 * Unregister the driver
2379 */
i7core_exit(void)2380 static void __exit i7core_exit(void)
2381 {
2382 edac_dbg(2, "\n");
2383 pci_unregister_driver(&i7core_driver);
2384 mce_unregister_decode_chain(&i7_mce_dec);
2385 }
2386
2387 module_init(i7core_init);
2388 module_exit(i7core_exit);
2389
2390 MODULE_LICENSE("GPL");
2391 MODULE_AUTHOR("Mauro Carvalho Chehab");
2392 MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
2393 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2394 I7CORE_REVISION);
2395
2396 module_param(edac_op_state, int, 0444);
2397 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
2398