1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel client SoC with integrated memory controller using IBECC 4 * 5 * Copyright (C) 2020 Intel Corporation 6 * 7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific 8 * regions of the physical memory space. It's used for memory controllers 9 * that don't support the out-of-band ECC which often needs an additional 10 * storage device to each channel for storing ECC data. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 #include <linux/irq_work.h> 18 #include <linux/llist.h> 19 #include <linux/genalloc.h> 20 #include <linux/edac.h> 21 #include <linux/bits.h> 22 #include <linux/bitfield.h> 23 #include <linux/io.h> 24 #include <asm/mach_traps.h> 25 #include <asm/nmi.h> 26 #include <asm/mce.h> 27 28 #include "edac_mc.h" 29 #include "edac_module.h" 30 31 #define IGEN6_REVISION "v2.5.1" 32 33 #define EDAC_MOD_STR "igen6_edac" 34 #define IGEN6_NMI_NAME "igen6_ibecc" 35 36 /* Debug macros */ 37 #define igen6_printk(level, fmt, arg...) \ 38 edac_printk(level, "igen6", fmt, ##arg) 39 40 #define igen6_mc_printk(mci, level, fmt, arg...) \ 41 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) 42 43 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 44 45 /* Probing upper bound, not a hardware capability limit. */ 46 #define MAX_IMC_TO_PROBE 8 47 #define NUM_CHANNELS 2 /* Max channels */ 48 #define NUM_DIMMS 2 /* Max DIMMs per channel */ 49 50 #define _4GB BIT_ULL(32) 51 52 /* Size of physical memory */ 53 #define TOM_OFFSET 0xa0 54 /* Top of low usable DRAM */ 55 #define TOLUD_OFFSET 0xbc 56 /* Capability register C */ 57 #define CAPID_C_OFFSET 0xec 58 #define CAPID_C_IBECC BIT(15) 59 60 /* Capability register E */ 61 #define CAPID_E_OFFSET 0xf0 62 #define CAPID_E_IBECC BIT(12) 63 #define CAPID_E_IBECC_BIT18 BIT(18) 64 65 /* Error Status */ 66 #define ERRSTS_OFFSET 0xc8 67 #define ERRSTS_CE BIT_ULL(6) 68 #define ERRSTS_UE BIT_ULL(7) 69 70 /* Error Command */ 71 #define ERRCMD_OFFSET 0xca 72 #define ERRCMD_CE BIT_ULL(6) 73 #define ERRCMD_UE BIT_ULL(7) 74 75 /* IBECC MMIO base address */ 76 #define IBECC_BASE (res_cfg->ibecc_base) 77 #define IBECC_ACTIVATE_OFFSET IBECC_BASE 78 #define IBECC_ACTIVATE_EN BIT(0) 79 80 /* IBECC error log */ 81 #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset) 82 #define ECC_ERROR_LOG_CE BIT_ULL(62) 83 #define ECC_ERROR_LOG_UE BIT_ULL(63) 84 #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) 85 86 /* Host MMIO base address */ 87 #define MCHBAR_OFFSET 0x48 88 #define MCHBAR_EN BIT_ULL(0) 89 #define MCHBAR_SIZE 0x10000 90 91 /* Parameters for the channel decode stage */ 92 #define IMC_BASE (res_cfg->imc_base) 93 #define MAD_INTER_CHANNEL_OFFSET IMC_BASE 94 #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) 95 #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) 96 #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) 97 #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) 98 99 /* Parameters for DRAM decode stage */ 100 #define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4) 101 #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) 102 103 /* DIMM characteristics */ 104 #define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc) 105 #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) 106 #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) 107 #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) 108 #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) 109 110 /* Hash for memory controller selection */ 111 #define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8) 112 #define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3) 113 114 /* Hash for channel selection */ 115 #define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24) 116 /* Hash for enhanced channel selection */ 117 #define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28) 118 #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 119 #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 120 #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) 121 122 /* Parameters for memory slice decode stage */ 123 #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 124 #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 125 126 /* 127 * A slice represents a portion of memory space participating in an 128 * interleave relationship within the memory hierarchy. 129 * 130 * It can represent in different levels such as: 131 * 132 * - a pair of memory controllers 133 * - a memory controller 134 * - a memory channel 135 * - a memory sub-channel / DIMM 136 * 137 * +--------+ 138 * | | 139 * | Zone 1 | 140 * | | 141 * +--------+ +--------+ 142 * | | | | 143 * | | | | 144 * | Zone 0 | | Zone 0 | 145 * | | | | 146 * | | | | 147 * +--------+ +--------+ 148 * 149 * Slice L Slice S 150 * 151 * Memory space is divided into: 152 * 153 * - Zone 0 : Interleaved region 154 * - Zone 1 : Non-interleaved region (upper part of the large slice). 155 */ 156 struct slice { 157 /* Slice address. */ 158 u64 addr; 159 /* Slice that @addr belongs to. */ 160 int id; 161 }; 162 163 struct igen6_imc { 164 int mc; 165 struct mem_ctl_info *mci; 166 struct pci_dev *pdev; 167 struct device dev; 168 void __iomem *window; 169 u64 size; 170 u64 ch_s_size; 171 int ch_l_map; 172 u64 dimm_s_size[NUM_CHANNELS]; 173 u64 dimm_l_size[NUM_CHANNELS]; 174 int dimm_l_map[NUM_CHANNELS]; 175 }; 176 177 static struct res_config { 178 bool machine_check; 179 /* The number of present memory controllers. */ 180 int num_imc; 181 /* Host MMIO configuration */ 182 u64 reg_mchbar_mask; 183 /* Top of memory */ 184 u64 reg_tom_mask; 185 /* Top of upper usable DRAM */ 186 u64 reg_touud_mask; 187 /* IBECC error log */ 188 u64 reg_eccerrlog_addr_mask; 189 /* MEMSS_PMA_CR registers. */ 190 u32 reg_mem_config_offset; 191 u32 reg_mem_config_ddr_type_mask; 192 u32 reg_mem_config_ibecc_en_mask; 193 u32 reg_capabilities_misc_offset; 194 u32 reg_capabilities_misc_ibecc_dis; 195 /* Memory controller registers. */ 196 u32 reg_mad_inter_size_mask[NUM_CHANNELS]; 197 u64 reg_mad_inter_size_granularity; 198 u32 reg_mad_intra_rank_mask[NUM_DIMMS]; 199 u32 reg_mad_intra_width_mask[NUM_DIMMS]; 200 u32 reg_mad_intra_density_mask[NUM_DIMMS]; 201 u32 imc_base; 202 u32 cmf_base; 203 u32 cmf_size; 204 u32 ms_hash_offset; 205 u32 ibecc_base; 206 u32 ibecc_error_log_offset; 207 /* Get memory type. */ 208 enum mem_type (*get_mem_type)(struct igen6_imc *imc); 209 /* Get DRAM chip type. */ 210 enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l); 211 /* Set imc->ch_{s_size,l_map}. */ 212 void (*set_chan_params)(struct igen6_imc *imc); 213 /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */ 214 void (*set_dimm_params)(struct igen6_imc *imc, int chan); 215 bool (*ibecc_available)(struct pci_dev *pdev); 216 /* Convert error address logged in IBECC to system physical address */ 217 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); 218 /* Convert error address logged in IBECC to integrated memory controller address */ 219 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); 220 } *res_cfg; 221 222 static struct igen6_pvt { 223 void __iomem *memss_pma_cr; 224 u64 ms_hash; 225 u64 ms_s_size; 226 int ms_l_map; 227 struct igen6_imc imc[]; 228 } *igen6_pvt; 229 230 /* The top of low usable DRAM */ 231 static u32 igen6_tolud; 232 /* The size of physical memory */ 233 static u64 igen6_tom; 234 235 struct decoded_addr { 236 int mc; 237 u64 imc_addr; 238 u64 sys_addr; 239 int channel_idx; 240 u64 channel_addr; 241 int sub_channel_idx; 242 u64 sub_channel_addr; 243 }; 244 245 struct ecclog_node { 246 struct llist_node llnode; 247 int mc; 248 u64 ecclog; 249 }; 250 251 /* 252 * In the NMI handler, the driver uses the lock-less memory allocator 253 * to allocate memory to store the IBECC error logs and links the logs 254 * to the lock-less list. Delay printk() and the work of error reporting 255 * to EDAC core in a worker. 256 */ 257 #define ECCLOG_POOL_SIZE PAGE_SIZE 258 static LLIST_HEAD(ecclog_llist); 259 static struct gen_pool *ecclog_pool; 260 static char ecclog_buf[ECCLOG_POOL_SIZE]; 261 static struct irq_work ecclog_irq_work; 262 static struct work_struct ecclog_work; 263 264 /* SoC compute die IDs with IBECC capability. */ 265 /* Elkhart Lake */ 266 #define DID_EHL_SKU5 0x4514 267 #define DID_EHL_SKU6 0x4528 268 #define DID_EHL_SKU7 0x452a 269 #define DID_EHL_SKU8 0x4516 270 #define DID_EHL_SKU9 0x452c 271 #define DID_EHL_SKU10 0x452e 272 #define DID_EHL_SKU11 0x4532 273 #define DID_EHL_SKU12 0x4518 274 #define DID_EHL_SKU13 0x451a 275 #define DID_EHL_SKU14 0x4534 276 #define DID_EHL_SKU15 0x4536 277 278 /* ICL-NNPI */ 279 #define DID_ICL_SKU8 0x4581 280 #define DID_ICL_SKU10 0x4585 281 #define DID_ICL_SKU11 0x4589 282 #define DID_ICL_SKU12 0x458d 283 284 /* Tiger Lake */ 285 #define DID_TGL_SKU 0x9a14 286 287 /* Alder Lake */ 288 #define DID_ADL_SKU1 0x4601 289 #define DID_ADL_SKU2 0x4602 290 #define DID_ADL_SKU3 0x4621 291 #define DID_ADL_SKU4 0x4641 292 293 /* Alder Lake-N */ 294 #define DID_ADL_N_SKU1 0x4614 295 #define DID_ADL_N_SKU2 0x4617 296 #define DID_ADL_N_SKU3 0x461b 297 #define DID_ADL_N_SKU4 0x461c 298 #define DID_ADL_N_SKU5 0x4673 299 #define DID_ADL_N_SKU6 0x4674 300 #define DID_ADL_N_SKU7 0x4675 301 #define DID_ADL_N_SKU8 0x4677 302 #define DID_ADL_N_SKU9 0x4678 303 #define DID_ADL_N_SKU10 0x4679 304 #define DID_ADL_N_SKU11 0x467c 305 #define DID_ADL_N_SKU12 0x4632 306 307 /* Arizona Beach */ 308 #define DID_AZB_SKU1 0x4676 309 310 /* Amston Lake */ 311 #define DID_ASL_SKU1 0x464a 312 #define DID_ASL_SKU2 0x4646 313 #define DID_ASL_SKU3 0x4652 314 315 /* Raptor Lake-P */ 316 #define DID_RPL_P_SKU1 0xa706 317 #define DID_RPL_P_SKU2 0xa707 318 #define DID_RPL_P_SKU3 0xa708 319 #define DID_RPL_P_SKU4 0xa716 320 #define DID_RPL_P_SKU5 0xa718 321 322 /* Meteor Lake-PS */ 323 #define DID_MTL_PS_SKU1 0x7d21 324 #define DID_MTL_PS_SKU2 0x7d22 325 #define DID_MTL_PS_SKU3 0x7d23 326 #define DID_MTL_PS_SKU4 0x7d24 327 328 /* Meteor Lake-P */ 329 #define DID_MTL_P_SKU1 0x7d01 330 #define DID_MTL_P_SKU2 0x7d02 331 #define DID_MTL_P_SKU3 0x7d14 332 333 /* Arrow Lake-UH */ 334 #define DID_ARL_UH_SKU1 0x7d06 335 #define DID_ARL_UH_SKU2 0x7d20 336 #define DID_ARL_UH_SKU3 0x7d30 337 338 /* Panther Lake-H */ 339 #define DID_PTL_H_SKU1 0xb000 340 #define DID_PTL_H_SKU2 0xb001 341 #define DID_PTL_H_SKU3 0xb002 342 #define DID_PTL_H_SKU4 0xb003 343 #define DID_PTL_H_SKU5 0xb004 344 #define DID_PTL_H_SKU6 0xb005 345 #define DID_PTL_H_SKU7 0xb008 346 #define DID_PTL_H_SKU8 0xb011 347 #define DID_PTL_H_SKU9 0xb014 348 #define DID_PTL_H_SKU10 0xb015 349 #define DID_PTL_H_SKU11 0xb028 350 #define DID_PTL_H_SKU12 0xb029 351 #define DID_PTL_H_SKU13 0xb02a 352 #define DID_PTL_H_SKU14 0xb00a 353 354 /* Starfire */ 355 #define DID_STF_SKU1 0xb02b 356 357 /* Wildcat Lake */ 358 #define DID_WCL_SKU1 0xfd00 359 360 /* Nova Lake-H/HX */ 361 #define DID_NVL_H_SKU1 0xd701 362 #define DID_NVL_H_SKU2 0xd702 363 #define DID_NVL_H_SKU3 0xd704 364 #define DID_NVL_H_SKU4 0xd705 365 366 /* Remove the interleave bit and shift upper part down to fill gap. */ 367 static u64 squeeze_addr(u64 addr, int intlv_bit) 368 { 369 u64 slice_addr; 370 371 slice_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; 372 slice_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); 373 374 return slice_addr; 375 } 376 377 /* Shift the upper bits up and insert a zero at the @intlv_bit bit position. */ 378 static u64 inflate_addr(u64 addr, int intlv_bit) 379 { 380 u64 inflated_addr; 381 382 /* Insert a zero at @intlv_bit position. */ 383 inflated_addr = GET_BITFIELD(addr, intlv_bit, 63) << (intlv_bit + 1); 384 inflated_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); 385 386 return inflated_addr; 387 } 388 389 static u64 compute_hash(u64 addr, u64 hash_mask, u64 hash_base, int intlv_bit) 390 { 391 u64 hash_addr; 392 int i; 393 394 /* 395 * In hash mode, @intlv_bit is the lowest selected bit of @addr 396 * to be XORed. While @mask may or may not include this @intlv_bit, 397 * we enforce that @mask includes @intlv_bit to ensure @intlv_bit is 398 * XORed exactly once. 399 */ 400 hash_mask |= BIT_ULL(intlv_bit); 401 hash_addr = addr & hash_mask; 402 403 for (i = 6; i < 20; i++) 404 hash_base ^= (hash_addr >> i) & 1; 405 406 return hash_base; 407 } 408 409 /* 410 * Converts a higher-level address (system / IMC / channel) into a lower-level 411 * slice address and identifier. 412 */ 413 static void translate_to_lower_level(u64 addr, u64 hash_mask, u64 hash_base, 414 int intlv_bit, u64 s_size, int l_map, 415 struct slice *slice) 416 { 417 /* In non-interleave zone. */ 418 if (addr >= 2 * s_size) { 419 slice->addr = addr - s_size; 420 slice->id = l_map; 421 return; 422 } 423 424 /* In interleave zone. */ 425 slice->addr = squeeze_addr(addr, intlv_bit); 426 427 /* Non-hash mode. */ 428 if (!hash_mask) { 429 slice->id = GET_BITFIELD(addr, intlv_bit, intlv_bit); 430 return; 431 } 432 433 /* Hash mode. */ 434 slice->id = compute_hash(addr, hash_mask, hash_base, intlv_bit); 435 } 436 437 /* Reconstruct address for upper memory hierarchy level. */ 438 static u64 translate_to_upper_level(u64 addr, u64 hash_mask, u64 hash_base, 439 int intlv_bit, u64 s_size) 440 { 441 u64 inflated_addr, hash_val; 442 443 /* In non-interleave zone. */ 444 if (addr >= s_size) 445 return addr + s_size; 446 447 /* 448 * In interleave zone. 449 * 450 * Insert a zero at @intlv_bit position. 451 */ 452 inflated_addr = inflate_addr(addr, intlv_bit); 453 454 /* 455 * Reconstruct the removed interleave bit and use it to replace 456 * the zero at @intlv_bit position. 457 */ 458 hash_val = compute_hash(inflated_addr, hash_mask, hash_base, intlv_bit); 459 return inflated_addr | (hash_val << intlv_bit); 460 } 461 462 static int get_mchbar(struct pci_dev *pdev, u64 *mchbar) 463 { 464 union { 465 u64 v; 466 struct { 467 u32 v_lo; 468 u32 v_hi; 469 }; 470 } u; 471 472 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { 473 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); 474 return -ENODEV; 475 } 476 477 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { 478 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); 479 return -ENODEV; 480 } 481 482 if (!(u.v & MCHBAR_EN)) { 483 igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); 484 return -ENODEV; 485 } 486 487 *mchbar = u.v & res_cfg->reg_mchbar_mask; 488 edac_dbg(2, "MCHBAR 0x%llx (reg 0x%llx)\n", *mchbar, u.v); 489 490 return 0; 491 } 492 493 /* Check whether the memory controller is absent. */ 494 static bool imc_absent(void __iomem *window) 495 { 496 return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0; 497 } 498 499 /* Return MMIO base address of the memory controller if it's present, otherwise return NULL. */ 500 static void __iomem *map_imc_window(u64 mchbar, int pmc) 501 { 502 void __iomem *window; 503 504 window = ioremap(mchbar + pmc * MCHBAR_SIZE, MCHBAR_SIZE); 505 if (!window) 506 return NULL; 507 508 if (imc_absent(window)) { 509 iounmap(window); 510 return NULL; 511 } 512 513 return window; 514 } 515 516 /* Return the number of present memory controllers. */ 517 static int get_imc_num(u64 mchbar) 518 { 519 void __iomem *window; 520 int lmc, pmc; 521 522 for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) { 523 window = map_imc_window(mchbar, pmc); 524 if (window) { 525 iounmap(window); 526 lmc++; 527 } 528 } 529 530 return lmc; 531 } 532 533 static bool ehl_ibecc_available(struct pci_dev *pdev) 534 { 535 u32 v; 536 537 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 538 return false; 539 540 return !!(CAPID_C_IBECC & v); 541 } 542 543 static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc) 544 { 545 return eaddr; 546 } 547 548 static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc) 549 { 550 if (eaddr < igen6_tolud) 551 return eaddr; 552 553 if (igen6_tom <= _4GB) 554 return eaddr + igen6_tolud - _4GB; 555 556 if (eaddr >= igen6_tom) 557 return eaddr + igen6_tolud - igen6_tom; 558 559 return eaddr; 560 } 561 562 static bool icl_ibecc_available(struct pci_dev *pdev) 563 { 564 u32 v; 565 566 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 567 return false; 568 569 return !(CAPID_C_IBECC & v) && 570 (boot_cpu_data.x86_stepping >= 1); 571 } 572 573 static bool tgl_ibecc_available(struct pci_dev *pdev) 574 { 575 u32 v; 576 577 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) 578 return false; 579 580 return !(CAPID_E_IBECC & v); 581 } 582 583 static bool mtl_p_ibecc_available(struct pci_dev *pdev) 584 { 585 u32 v; 586 587 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) 588 return false; 589 590 return !(CAPID_E_IBECC_BIT18 & v); 591 } 592 593 static bool generic_ibecc_available(struct pci_dev *pdev) 594 { 595 void __iomem *base = igen6_pvt->memss_pma_cr; 596 bool present; 597 u32 val; 598 599 if (res_cfg->reg_capabilities_misc_offset) { 600 val = readl(base + res_cfg->reg_capabilities_misc_offset); 601 present = !(val & res_cfg->reg_capabilities_misc_ibecc_dis); 602 edac_dbg(2, "capabilities misc reg 0x%x\n", val); 603 } else if (res_cfg->reg_mem_config_offset) { 604 val = readl(base + res_cfg->reg_mem_config_offset); 605 present = !!(val & res_cfg->reg_mem_config_ibecc_en_mask); 606 edac_dbg(2, "mem config reg 0x%x\n", val); 607 } else { 608 igen6_printk(KERN_ERR, "No register for detecting IBECC presence.\n"); 609 present = false; 610 } 611 612 return present; 613 } 614 615 static u64 mem_addr_to_sys_addr(u64 maddr) 616 { 617 if (maddr < igen6_tolud) 618 return maddr; 619 620 if (igen6_tom <= _4GB) 621 return maddr - igen6_tolud + _4GB; 622 623 if (maddr < _4GB) 624 return maddr - igen6_tolud + igen6_tom; 625 626 return maddr; 627 } 628 629 static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc) 630 { 631 u64 mask, ms_s_size; 632 int intlv_bit; 633 u32 ms_hash; 634 635 ms_s_size = igen6_pvt->ms_s_size; 636 if (eaddr >= ms_s_size) 637 return eaddr + ms_s_size; 638 639 ms_hash = igen6_pvt->ms_hash; 640 641 mask = MEM_SLICE_HASH_MASK(ms_hash); 642 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6; 643 644 return translate_to_upper_level(eaddr, mask, mc, intlv_bit, ms_s_size); 645 } 646 647 static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc) 648 { 649 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc); 650 651 return mem_addr_to_sys_addr(maddr); 652 } 653 654 static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc) 655 { 656 return eaddr; 657 } 658 659 static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc) 660 { 661 return mem_addr_to_sys_addr(eaddr); 662 } 663 664 static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc) 665 { 666 u64 ms_s_size = igen6_pvt->ms_s_size; 667 struct igen6_imc *imc = &igen6_pvt->imc[mc]; 668 struct slice slice; 669 int intlv_bit; 670 u32 mc_hash; 671 672 if (eaddr >= 2 * ms_s_size) 673 return eaddr - ms_s_size; 674 675 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET); 676 677 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6; 678 679 translate_to_lower_level(eaddr, 0, 0, intlv_bit, ms_s_size, 0, &slice); 680 return slice.addr; 681 } 682 683 static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc) 684 { 685 u32 mtype, val; 686 687 val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset); 688 mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val); 689 690 edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val); 691 692 switch (mtype) { 693 case 1: 694 return MEM_DDR5; 695 case 2: 696 return MEM_LPDDR5; 697 case 3: 698 return MEM_LPDDR4; 699 default: 700 return MEM_UNKNOWN; 701 } 702 } 703 704 static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm) 705 { 706 u32 width, val; 707 708 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 709 width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val); 710 711 switch (width) { 712 case 1: 713 return DEV_X8; 714 default: 715 return DEV_X16; 716 } 717 } 718 719 static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan) 720 { 721 u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 722 723 return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) * 724 res_cfg->reg_mad_inter_size_granularity; 725 } 726 727 static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm) 728 { 729 u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 730 u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val); 731 /* DRAM device density in Gb */ 732 u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4; 733 734 enum mem_type mtype = ptl_h_get_mem_type(imc); 735 enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm); 736 u64 sub_ch_width, dev_num; 737 738 switch (mtype) { 739 case MEM_DDR5: 740 sub_ch_width = 32; 741 break; 742 case MEM_LPDDR5: 743 case MEM_LPDDR4: 744 sub_ch_width = 16; 745 break; 746 default: 747 sub_ch_width = 0; 748 } 749 750 switch (dtype) { 751 case DEV_X8: 752 dev_num = sub_ch_width / 8; 753 break; 754 case DEV_X16: 755 dev_num = sub_ch_width / 16; 756 break; 757 default: 758 dev_num = 0; 759 } 760 761 edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val); 762 763 return ((dev_num * density / 8) * ranks) << 30; 764 } 765 766 static void ptl_h_set_chan_params(struct igen6_imc *imc) 767 { 768 u64 ch0_size = ptl_h_get_chan_size(imc, 0); 769 u64 ch1_size = ptl_h_get_chan_size(imc, 1); 770 771 if (ch0_size <= ch1_size) { 772 imc->ch_s_size = ch0_size; 773 imc->ch_l_map = 1; 774 } else { 775 imc->ch_s_size = ch1_size; 776 imc->ch_l_map = 0; 777 } 778 } 779 780 static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan) 781 { 782 u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0); 783 u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1); 784 785 if (dimm0_size <= dimm1_size) { 786 imc->dimm_s_size[chan] = dimm0_size; 787 imc->dimm_l_size[chan] = dimm1_size; 788 imc->dimm_l_map[chan] = 1; 789 } else { 790 imc->dimm_s_size[chan] = dimm1_size; 791 imc->dimm_l_size[chan] = dimm0_size; 792 imc->dimm_l_map[chan] = 0; 793 } 794 } 795 796 static struct res_config ehl_cfg = { 797 .num_imc = 1, 798 .reg_mchbar_mask = GENMASK_ULL(38, 16), 799 .reg_tom_mask = GENMASK_ULL(38, 20), 800 .reg_touud_mask = GENMASK_ULL(38, 20), 801 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 802 .imc_base = 0x5000, 803 .ibecc_base = 0xdc00, 804 .ibecc_available = ehl_ibecc_available, 805 .ibecc_error_log_offset = 0x170, 806 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 807 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 808 }; 809 810 static struct res_config icl_cfg = { 811 .num_imc = 1, 812 .reg_mchbar_mask = GENMASK_ULL(38, 16), 813 .reg_tom_mask = GENMASK_ULL(38, 20), 814 .reg_touud_mask = GENMASK_ULL(38, 20), 815 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 816 .imc_base = 0x5000, 817 .ibecc_base = 0xd800, 818 .ibecc_error_log_offset = 0x170, 819 .ibecc_available = icl_ibecc_available, 820 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 821 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 822 }; 823 824 static struct res_config tgl_cfg = { 825 .machine_check = true, 826 .num_imc = 2, 827 .reg_mchbar_mask = GENMASK_ULL(38, 17), 828 .reg_tom_mask = GENMASK_ULL(38, 20), 829 .reg_touud_mask = GENMASK_ULL(38, 20), 830 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 831 .imc_base = 0x5000, 832 .cmf_base = 0x11000, 833 .cmf_size = 0x800, 834 .ms_hash_offset = 0xac, 835 .ibecc_base = 0xd400, 836 .ibecc_error_log_offset = 0x170, 837 .ibecc_available = tgl_ibecc_available, 838 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr, 839 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr, 840 }; 841 842 /* Shared by Alder Lake, Alder Lake-N, Arizona Beach, Amston Lake, and Raptor Lake-P */ 843 static struct res_config adl_cfg = { 844 .machine_check = true, 845 .num_imc = 2, 846 .reg_mchbar_mask = GENMASK_ULL(41, 17), 847 .reg_tom_mask = GENMASK_ULL(41, 20), 848 .reg_touud_mask = GENMASK_ULL(41, 20), 849 .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5), 850 .imc_base = 0xd800, 851 .ibecc_base = 0xd400, 852 .ibecc_error_log_offset = 0x68, 853 .ibecc_available = tgl_ibecc_available, 854 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 855 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 856 }; 857 858 static struct res_config mtl_ps_cfg = { 859 .machine_check = true, 860 .num_imc = 2, 861 .reg_mchbar_mask = GENMASK_ULL(41, 17), 862 .reg_tom_mask = GENMASK_ULL(41, 20), 863 .reg_touud_mask = GENMASK_ULL(41, 20), 864 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 865 .reg_capabilities_misc_offset = 0x13c00, 866 .reg_capabilities_misc_ibecc_dis = BIT(6), 867 .imc_base = 0xd800, 868 .ibecc_base = 0xd400, 869 .ibecc_error_log_offset = 0x170, 870 .ibecc_available = generic_ibecc_available, 871 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 872 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 873 }; 874 875 /* Shared by Meteor Lake-P, Arrow Lake-UH, and Wildcat Lake */ 876 static struct res_config mtl_p_cfg = { 877 .machine_check = true, 878 .num_imc = 2, 879 .reg_mchbar_mask = GENMASK_ULL(41, 17), 880 .reg_tom_mask = GENMASK_ULL(41, 20), 881 .reg_touud_mask = GENMASK_ULL(41, 20), 882 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 883 .imc_base = 0xd800, 884 .ibecc_base = 0xd400, 885 .ibecc_error_log_offset = 0x170, 886 .ibecc_available = mtl_p_ibecc_available, 887 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 888 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 889 }; 890 891 /* Shared by Panther Lake-H and Starfire */ 892 static struct res_config ptl_h_cfg = { 893 .machine_check = true, 894 .num_imc = 2, 895 .reg_mchbar_mask = GENMASK_ULL(41, 17), 896 .reg_tom_mask = GENMASK_ULL(41, 20), 897 .reg_touud_mask = GENMASK_ULL(41, 20), 898 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 899 .reg_mem_config_offset = 0x13d04, 900 .reg_mem_config_ddr_type_mask = GENMASK(8, 6), 901 .reg_mad_inter_size_mask[0] = GENMASK(15, 8), 902 .reg_mad_inter_size_mask[1] = GENMASK(23, 16), 903 .reg_mad_inter_size_granularity = BIT_ULL(29), 904 .reg_mad_intra_rank_mask[0] = BIT(7), 905 .reg_mad_intra_rank_mask[1] = BIT(15), 906 .reg_mad_intra_width_mask[0] = BIT(6), 907 .reg_mad_intra_width_mask[1] = BIT(14), 908 .reg_mad_intra_density_mask[0] = GENMASK(3, 0), 909 .reg_mad_intra_density_mask[1] = GENMASK(11, 8), 910 .imc_base = 0xd800, 911 .ibecc_base = 0xd400, 912 .ibecc_error_log_offset = 0x170, 913 .get_mem_type = ptl_h_get_mem_type, 914 .get_dev_type = ptl_h_get_dev_type, 915 .set_chan_params = ptl_h_set_chan_params, 916 .set_dimm_params = ptl_h_set_dimm_params, 917 .ibecc_available = mtl_p_ibecc_available, 918 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 919 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 920 }; 921 922 static struct res_config nvl_h_cfg = { 923 .machine_check = true, 924 .num_imc = 2, 925 .reg_mchbar_mask = GENMASK_ULL(41, 17), 926 .reg_tom_mask = GENMASK_ULL(41, 20), 927 .reg_touud_mask = GENMASK_ULL(41, 20), 928 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 929 .reg_mem_config_offset = 0x12904, 930 .reg_mem_config_ddr_type_mask = GENMASK(8, 6), 931 .reg_mem_config_ibecc_en_mask = GENMASK(3, 2), 932 .reg_mad_inter_size_mask[0] = GENMASK(15, 8), 933 .reg_mad_inter_size_mask[1] = GENMASK(23, 16), 934 .reg_mad_inter_size_granularity = BIT_ULL(29), 935 .reg_mad_intra_rank_mask[0] = BIT(7), 936 .reg_mad_intra_rank_mask[1] = BIT(15), 937 .reg_mad_intra_width_mask[0] = BIT(6), 938 .reg_mad_intra_width_mask[1] = BIT(14), 939 .reg_mad_intra_density_mask[0] = GENMASK(3, 0), 940 .reg_mad_intra_density_mask[1] = GENMASK(11, 8), 941 .imc_base = 0xd800, 942 .ibecc_base = 0xd400, 943 .ibecc_error_log_offset = 0x170, 944 .get_mem_type = ptl_h_get_mem_type, 945 .get_dev_type = ptl_h_get_dev_type, 946 .set_chan_params = ptl_h_set_chan_params, 947 .set_dimm_params = ptl_h_set_dimm_params, 948 .ibecc_available = generic_ibecc_available, 949 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 950 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 951 }; 952 953 static struct pci_device_id igen6_pci_tbl[] = { 954 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), .driver_data = (kernel_ulong_t)&ehl_cfg }, 955 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), .driver_data = (kernel_ulong_t)&ehl_cfg }, 956 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), .driver_data = (kernel_ulong_t)&ehl_cfg }, 957 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), .driver_data = (kernel_ulong_t)&ehl_cfg }, 958 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), .driver_data = (kernel_ulong_t)&ehl_cfg }, 959 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), .driver_data = (kernel_ulong_t)&ehl_cfg }, 960 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), .driver_data = (kernel_ulong_t)&ehl_cfg }, 961 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), .driver_data = (kernel_ulong_t)&ehl_cfg }, 962 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), .driver_data = (kernel_ulong_t)&ehl_cfg }, 963 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), .driver_data = (kernel_ulong_t)&ehl_cfg }, 964 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), .driver_data = (kernel_ulong_t)&ehl_cfg }, 965 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), .driver_data = (kernel_ulong_t)&icl_cfg }, 966 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), .driver_data = (kernel_ulong_t)&icl_cfg }, 967 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), .driver_data = (kernel_ulong_t)&icl_cfg }, 968 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), .driver_data = (kernel_ulong_t)&icl_cfg }, 969 { PCI_VDEVICE(INTEL, DID_TGL_SKU), .driver_data = (kernel_ulong_t)&tgl_cfg }, 970 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 971 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, 972 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, 973 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, 974 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 975 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, 976 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, 977 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, 978 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg }, 979 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), .driver_data = (kernel_ulong_t)&adl_cfg }, 980 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), .driver_data = (kernel_ulong_t)&adl_cfg }, 981 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), .driver_data = (kernel_ulong_t)&adl_cfg }, 982 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), .driver_data = (kernel_ulong_t)&adl_cfg }, 983 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), .driver_data = (kernel_ulong_t)&adl_cfg }, 984 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), .driver_data = (kernel_ulong_t)&adl_cfg }, 985 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), .driver_data = (kernel_ulong_t)&adl_cfg }, 986 { PCI_VDEVICE(INTEL, DID_AZB_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 987 { PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 988 { PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, 989 { PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, 990 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 991 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, 992 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, 993 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, 994 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg }, 995 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 996 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 997 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 998 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 999 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1000 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1001 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1002 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1003 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1004 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1005 { PCI_VDEVICE(INTEL, DID_WCL_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 1006 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1007 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1008 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1009 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1010 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1011 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1012 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1013 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1014 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1015 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1016 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1017 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1018 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1019 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1020 { PCI_VDEVICE(INTEL, DID_STF_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 1021 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU1), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 1022 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU2), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 1023 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU3), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 1024 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU4), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 1025 { }, 1026 }; 1027 MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); 1028 1029 static enum mem_type get_mem_type(struct igen6_imc *imc) 1030 { 1031 u32 val; 1032 1033 if (res_cfg->get_mem_type) 1034 return res_cfg->get_mem_type(imc); 1035 1036 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 1037 1038 switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) { 1039 case 0: 1040 return MEM_DDR4; 1041 case 1: 1042 return MEM_DDR3; 1043 case 2: 1044 return MEM_LPDDR3; 1045 case 3: 1046 return MEM_LPDDR4; 1047 case 4: 1048 return MEM_WIO2; 1049 default: 1050 return MEM_UNKNOWN; 1051 } 1052 } 1053 1054 static bool large_dimm(struct igen6_imc *imc, int chan, int dimm) 1055 { 1056 return dimm == imc->dimm_l_map[chan]; 1057 } 1058 1059 static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm) 1060 { 1061 u32 width, val; 1062 1063 if (res_cfg->get_dev_type) 1064 return res_cfg->get_dev_type(imc, chan, dimm); 1065 1066 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); 1067 width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) : 1068 MAD_DIMM_CH_DSW(val); 1069 1070 switch (width) { 1071 case 0: 1072 return DEV_X8; 1073 case 1: 1074 return DEV_X16; 1075 case 2: 1076 return DEV_X32; 1077 default: 1078 return DEV_UNKNOWN; 1079 } 1080 } 1081 1082 static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm) 1083 { 1084 if (large_dimm(imc, chan, dimm)) 1085 return imc->dimm_l_size[chan]; 1086 1087 return imc->dimm_s_size[chan]; 1088 } 1089 1090 static void set_chan_params(struct igen6_imc *imc) 1091 { 1092 u32 val; 1093 1094 if (res_cfg->set_chan_params) { 1095 res_cfg->set_chan_params(imc); 1096 return; 1097 } 1098 1099 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 1100 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val); 1101 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val); 1102 } 1103 1104 static void set_dimm_params(struct igen6_imc *imc, int chan) 1105 { 1106 u32 val; 1107 1108 if (res_cfg->set_dimm_params) { 1109 res_cfg->set_dimm_params(imc, chan); 1110 return; 1111 } 1112 1113 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 1114 imc->dimm_l_map[chan] = MAD_INTRA_CH_DIMM_L_MAP(val); 1115 1116 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); 1117 imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val); 1118 imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val); 1119 } 1120 1121 static int igen6_decode(struct decoded_addr *res) 1122 { 1123 struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; 1124 u64 addr = res->imc_addr, s_size; 1125 int intlv_bit, l_map; 1126 u32 hash, hash_mask; 1127 struct slice slice; 1128 1129 if (addr >= igen6_tom) { 1130 edac_dbg(0, "Address 0x%llx out of range\n", addr); 1131 return -EINVAL; 1132 } 1133 1134 /* Decode channel */ 1135 hash = readl(imc->window + CHANNEL_HASH_OFFSET); 1136 s_size = imc->ch_s_size; 1137 l_map = imc->ch_l_map; 1138 hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0; 1139 intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; 1140 1141 translate_to_lower_level(addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice); 1142 1143 res->channel_idx = slice.id; 1144 res->channel_addr = slice.addr; 1145 1146 /* Decode sub-channel/DIMM */ 1147 hash = readl(imc->window + CHANNEL_EHASH_OFFSET); 1148 s_size = imc->dimm_s_size[res->channel_idx]; 1149 l_map = imc->dimm_l_map[res->channel_idx]; 1150 hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0; 1151 intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; 1152 1153 translate_to_lower_level(res->channel_addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice); 1154 1155 res->sub_channel_idx = slice.id; 1156 res->sub_channel_addr = slice.addr; 1157 1158 return 0; 1159 } 1160 1161 static void igen6_output_error(struct decoded_addr *res, 1162 struct mem_ctl_info *mci, u64 ecclog) 1163 { 1164 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? 1165 HW_EVENT_ERR_UNCORRECTED : 1166 HW_EVENT_ERR_CORRECTED; 1167 1168 edac_mc_handle_error(type, mci, 1, 1169 res->sys_addr >> PAGE_SHIFT, 1170 res->sys_addr & ~PAGE_MASK, 1171 ECC_ERROR_LOG_SYND(ecclog), 1172 res->channel_idx, res->sub_channel_idx, 1173 -1, "", ""); 1174 } 1175 1176 static struct gen_pool *ecclog_gen_pool_create(void) 1177 { 1178 struct gen_pool *pool; 1179 1180 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); 1181 if (!pool) 1182 return NULL; 1183 1184 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { 1185 gen_pool_destroy(pool); 1186 return NULL; 1187 } 1188 1189 return pool; 1190 } 1191 1192 static int ecclog_gen_pool_add(int mc, u64 ecclog) 1193 { 1194 struct ecclog_node *node; 1195 1196 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); 1197 if (!node) 1198 return -ENOMEM; 1199 1200 node->mc = mc; 1201 node->ecclog = ecclog; 1202 llist_add(&node->llnode, &ecclog_llist); 1203 1204 return 0; 1205 } 1206 1207 /* 1208 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI 1209 * configuration space status register ERRSTS can indicate whether a 1210 * correctable error or an uncorrectable error occurred. We only use the 1211 * ECC_ERROR_LOG register to check error type, but need to clear both 1212 * registers to enable future error events. 1213 */ 1214 static u64 ecclog_read_and_clear(struct igen6_imc *imc) 1215 { 1216 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); 1217 1218 /* 1219 * Quirk: The ECC_ERROR_LOG register of certain SoCs may contain 1220 * the invalid value ~0. This will result in a flood of invalid 1221 * error reports in polling mode. Skip it. 1222 */ 1223 if (ecclog == ~0) 1224 return 0; 1225 1226 /* Neither a CE nor a UE. Skip it.*/ 1227 if (!(ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE))) 1228 return 0; 1229 1230 /* Clear CE/UE bits by writing 1s */ 1231 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); 1232 1233 return ecclog; 1234 } 1235 1236 static void errsts_clear(struct igen6_imc *imc) 1237 { 1238 u16 errsts; 1239 1240 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { 1241 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); 1242 return; 1243 } 1244 1245 /* Clear CE/UE bits by writing 1s */ 1246 if (errsts & (ERRSTS_CE | ERRSTS_UE)) 1247 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); 1248 } 1249 1250 static int errcmd_enable_error_reporting(bool enable) 1251 { 1252 struct igen6_imc *imc = &igen6_pvt->imc[0]; 1253 u16 errcmd; 1254 int rc; 1255 1256 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); 1257 if (rc) 1258 return pcibios_err_to_errno(rc); 1259 1260 if (enable) 1261 errcmd |= ERRCMD_CE | ERRSTS_UE; 1262 else 1263 errcmd &= ~(ERRCMD_CE | ERRSTS_UE); 1264 1265 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); 1266 if (rc) 1267 return pcibios_err_to_errno(rc); 1268 1269 return 0; 1270 } 1271 1272 static int ecclog_handler(void) 1273 { 1274 struct igen6_imc *imc; 1275 int i, n = 0; 1276 u64 ecclog; 1277 1278 for (i = 0; i < res_cfg->num_imc; i++) { 1279 imc = &igen6_pvt->imc[i]; 1280 1281 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 1282 1283 ecclog = ecclog_read_and_clear(imc); 1284 if (!ecclog) 1285 continue; 1286 1287 if (!ecclog_gen_pool_add(i, ecclog)) 1288 irq_work_queue(&ecclog_irq_work); 1289 1290 n++; 1291 } 1292 1293 return n; 1294 } 1295 1296 static void ecclog_work_cb(struct work_struct *work) 1297 { 1298 struct ecclog_node *node, *tmp; 1299 struct mem_ctl_info *mci; 1300 struct llist_node *head; 1301 struct decoded_addr res; 1302 u64 eaddr; 1303 1304 head = llist_del_all(&ecclog_llist); 1305 if (!head) 1306 return; 1307 1308 llist_for_each_entry_safe(node, tmp, head, llnode) { 1309 memset(&res, 0, sizeof(res)); 1310 eaddr = node->ecclog & res_cfg->reg_eccerrlog_addr_mask; 1311 res.mc = node->mc; 1312 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc); 1313 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc); 1314 1315 mci = igen6_pvt->imc[res.mc].mci; 1316 1317 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); 1318 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); 1319 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); 1320 1321 if (!igen6_decode(&res)) 1322 igen6_output_error(&res, mci, node->ecclog); 1323 1324 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); 1325 } 1326 } 1327 1328 static void ecclog_irq_work_cb(struct irq_work *irq_work) 1329 { 1330 int i; 1331 1332 for (i = 0; i < res_cfg->num_imc; i++) 1333 errsts_clear(&igen6_pvt->imc[i]); 1334 1335 if (!llist_empty(&ecclog_llist)) 1336 schedule_work(&ecclog_work); 1337 } 1338 1339 static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1340 { 1341 unsigned char reason; 1342 1343 if (!ecclog_handler()) 1344 return NMI_DONE; 1345 1346 /* 1347 * Both In-Band ECC correctable error and uncorrectable error are 1348 * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) 1349 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to 1350 * re-enable the SERR# NMI after NMI handling. So clear this bit here 1351 * to re-enable SERR# NMI for receiving future In-Band ECC errors. 1352 */ 1353 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; 1354 reason |= NMI_REASON_CLEAR_SERR; 1355 outb(reason, NMI_REASON_PORT); 1356 reason &= ~NMI_REASON_CLEAR_SERR; 1357 outb(reason, NMI_REASON_PORT); 1358 1359 return NMI_HANDLED; 1360 } 1361 1362 static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val, 1363 void *data) 1364 { 1365 struct mce *mce = (struct mce *)data; 1366 char *type; 1367 1368 if (mce->kflags & MCE_HANDLED_CEC) 1369 return NOTIFY_DONE; 1370 1371 /* 1372 * Ignore unless this is a memory related error. 1373 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here, 1374 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3). 1375 */ 1376 if ((mce->status & 0xefff) >> 7 != 1) 1377 return NOTIFY_DONE; 1378 1379 if (mce->mcgstatus & MCG_STATUS_MCIP) 1380 type = "Exception"; 1381 else 1382 type = "Event"; 1383 1384 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n", 1385 mce->extcpu, type, mce->mcgstatus, 1386 mce->bank, mce->status); 1387 edac_dbg(0, "TSC 0x%llx\n", mce->tsc); 1388 edac_dbg(0, "ADDR 0x%llx\n", mce->addr); 1389 edac_dbg(0, "MISC 0x%llx\n", mce->misc); 1390 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n", 1391 mce->cpuvendor, mce->cpuid, mce->time, 1392 mce->socketid, mce->apicid); 1393 /* 1394 * We just use the Machine Check for the memory error notification. 1395 * Each memory controller is associated with an IBECC instance. 1396 * Directly read and clear the error information(error address and 1397 * error type) on all the IBECC instances so that we know on which 1398 * memory controller the memory error(s) occurred. 1399 */ 1400 if (!ecclog_handler()) 1401 return NOTIFY_DONE; 1402 1403 mce->kflags |= MCE_HANDLED_EDAC; 1404 1405 return NOTIFY_DONE; 1406 } 1407 1408 static struct notifier_block ecclog_mce_dec = { 1409 .notifier_call = ecclog_mce_handler, 1410 .priority = MCE_PRIO_EDAC, 1411 }; 1412 1413 static bool igen6_check_ecc(struct igen6_imc *imc) 1414 { 1415 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); 1416 1417 return !!(activate & IBECC_ACTIVATE_EN); 1418 } 1419 1420 static int igen6_get_dimm_config(struct mem_ctl_info *mci) 1421 { 1422 struct igen6_imc *imc = mci->pvt_info; 1423 int i, j, ndimms, mc = imc->mc; 1424 struct dimm_info *dimm; 1425 enum mem_type mtype; 1426 enum dev_type dtype; 1427 u64 dsize; 1428 bool ecc; 1429 1430 edac_dbg(2, "\n"); 1431 1432 mtype = get_mem_type(imc); 1433 ecc = igen6_check_ecc(imc); 1434 set_chan_params(imc); 1435 1436 for (i = 0; i < NUM_CHANNELS; i++) { 1437 set_dimm_params(imc, i); 1438 imc->size += imc->dimm_s_size[i]; 1439 imc->size += imc->dimm_l_size[i]; 1440 ndimms = 0; 1441 1442 for (j = 0; j < NUM_DIMMS; j++) { 1443 dimm = edac_get_dimm(mci, i, j, 0); 1444 dtype = get_dev_type(imc, i, j); 1445 dsize = get_dimm_size(imc, i, j); 1446 1447 if (!dsize) 1448 continue; 1449 1450 dimm->grain = 64; 1451 dimm->mtype = mtype; 1452 dimm->dtype = dtype; 1453 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); 1454 dimm->edac_mode = EDAC_SECDED; 1455 snprintf(dimm->label, sizeof(dimm->label), 1456 "MC#%d_Chan#%d_DIMM#%d", mc, i, j); 1457 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", 1458 mc, i, j, dsize >> 20, dimm->nr_pages); 1459 1460 ndimms++; 1461 } 1462 1463 if (ndimms && !ecc) { 1464 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); 1465 return -ENODEV; 1466 } 1467 } 1468 1469 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20); 1470 1471 return 0; 1472 } 1473 1474 #ifdef CONFIG_EDAC_DEBUG 1475 /* Top of upper usable DRAM */ 1476 static u64 igen6_touud; 1477 #define TOUUD_OFFSET 0xa8 1478 1479 static void igen6_reg_dump(struct igen6_imc *imc) 1480 { 1481 int i; 1482 1483 edac_dbg(2, "CHANNEL_HASH : 0x%x\n", 1484 readl(imc->window + CHANNEL_HASH_OFFSET)); 1485 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", 1486 readl(imc->window + CHANNEL_EHASH_OFFSET)); 1487 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", 1488 readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); 1489 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", 1490 readq(imc->window + ECC_ERROR_LOG_OFFSET)); 1491 1492 for (i = 0; i < NUM_CHANNELS; i++) { 1493 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, 1494 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); 1495 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, 1496 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); 1497 } 1498 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); 1499 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); 1500 edac_dbg(2, "TOM : 0x%llx", igen6_tom); 1501 } 1502 1503 static struct dentry *igen6_test; 1504 1505 static int debugfs_u64_set(void *data, u64 val) 1506 { 1507 u64 ecclog; 1508 1509 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { 1510 edac_dbg(0, "Address 0x%llx out of range\n", val); 1511 return 0; 1512 } 1513 1514 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); 1515 1516 ecclog = (val & res_cfg->reg_eccerrlog_addr_mask) | ECC_ERROR_LOG_CE; 1517 1518 if (!ecclog_gen_pool_add(0, ecclog)) 1519 irq_work_queue(&ecclog_irq_work); 1520 1521 return 0; 1522 } 1523 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 1524 1525 static void igen6_debug_setup(void) 1526 { 1527 igen6_test = edac_debugfs_create_dir("igen6_test"); 1528 if (!igen6_test) 1529 return; 1530 1531 if (!edac_debugfs_create_file("addr", 0200, igen6_test, 1532 NULL, &fops_u64_wo)) { 1533 debugfs_remove(igen6_test); 1534 igen6_test = NULL; 1535 } 1536 } 1537 1538 static void igen6_debug_teardown(void) 1539 { 1540 debugfs_remove_recursive(igen6_test); 1541 } 1542 #else 1543 static void igen6_reg_dump(struct igen6_imc *imc) {} 1544 static void igen6_debug_setup(void) {} 1545 static void igen6_debug_teardown(void) {} 1546 #endif 1547 1548 static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev) 1549 { 1550 void __iomem *memss_pma_cr; 1551 struct igen6_pvt *pvt; 1552 int imc_num, rc; 1553 u64 mchbar; 1554 1555 rc = get_mchbar(pdev, &mchbar); 1556 if (rc) 1557 return NULL; 1558 1559 imc_num = get_imc_num(mchbar); 1560 if (!imc_num) { 1561 igen6_printk(KERN_ERR, "No mc found.\n"); 1562 return NULL; 1563 } 1564 edac_dbg(2, "%d mcs found.\n", imc_num); 1565 1566 /* Use the runtime detected IMC count. */ 1567 if (res_cfg->num_imc != imc_num) 1568 res_cfg->num_imc = imc_num; 1569 1570 pvt = kzalloc_flex(*pvt, imc, imc_num); 1571 if (!pvt) 1572 return NULL; 1573 1574 memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2); 1575 if (!memss_pma_cr) { 1576 kfree(pvt); 1577 return NULL; 1578 } 1579 pvt->memss_pma_cr = memss_pma_cr; 1580 1581 return pvt; 1582 } 1583 1584 static void igen6_pvt_release(struct igen6_pvt *pvt) 1585 { 1586 iounmap(pvt->memss_pma_cr); 1587 kfree(pvt); 1588 } 1589 1590 static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) 1591 { 1592 union { 1593 u64 v; 1594 struct { 1595 u32 v_lo; 1596 u32 v_hi; 1597 }; 1598 } u; 1599 1600 edac_dbg(2, "\n"); 1601 1602 if (!res_cfg->ibecc_available(pdev)) { 1603 edac_dbg(2, "No In-Band ECC IP\n"); 1604 goto fail; 1605 } 1606 1607 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { 1608 igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); 1609 goto fail; 1610 } 1611 1612 igen6_tolud &= GENMASK(31, 20); 1613 1614 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { 1615 igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); 1616 goto fail; 1617 } 1618 1619 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { 1620 igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); 1621 goto fail; 1622 } 1623 1624 igen6_tom = u.v & res_cfg->reg_tom_mask; 1625 1626 if (get_mchbar(pdev, mchbar)) 1627 goto fail; 1628 1629 #ifdef CONFIG_EDAC_DEBUG 1630 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) 1631 edac_dbg(2, "Failed to read lower TOUUD\n"); 1632 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) 1633 edac_dbg(2, "Failed to read upper TOUUD\n"); 1634 else 1635 igen6_touud = u.v & res_cfg->reg_touud_mask; 1636 #endif 1637 1638 return 0; 1639 fail: 1640 return -ENODEV; 1641 } 1642 1643 static void igen6_check(struct mem_ctl_info *mci) 1644 { 1645 struct igen6_imc *imc = mci->pvt_info; 1646 u64 ecclog; 1647 1648 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 1649 ecclog = ecclog_read_and_clear(imc); 1650 if (!ecclog) 1651 return; 1652 1653 if (!ecclog_gen_pool_add(imc->mc, ecclog)) 1654 irq_work_queue(&ecclog_irq_work); 1655 } 1656 1657 static void imc_release(struct device *dev) 1658 { 1659 /* Nothing to do, the 'imc' owns the 'dev' and will also release it. */ 1660 } 1661 1662 static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev) 1663 { 1664 struct edac_mc_layer layers[2]; 1665 struct mem_ctl_info *mci; 1666 struct igen6_imc *imc; 1667 int rc; 1668 1669 edac_dbg(2, "\n"); 1670 1671 layers[0].type = EDAC_MC_LAYER_CHANNEL; 1672 layers[0].size = NUM_CHANNELS; 1673 layers[0].is_virt_csrow = false; 1674 layers[1].type = EDAC_MC_LAYER_SLOT; 1675 layers[1].size = NUM_DIMMS; 1676 layers[1].is_virt_csrow = true; 1677 1678 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); 1679 if (!mci) { 1680 rc = -ENOMEM; 1681 goto fail; 1682 } 1683 1684 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); 1685 if (!mci->ctl_name) { 1686 rc = -ENOMEM; 1687 goto fail2; 1688 } 1689 1690 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; 1691 mci->edac_ctl_cap = EDAC_FLAG_SECDED; 1692 mci->edac_cap = EDAC_FLAG_SECDED; 1693 mci->mod_name = EDAC_MOD_STR; 1694 mci->dev_name = pci_name(pdev); 1695 if (edac_op_state == EDAC_OPSTATE_POLL) 1696 mci->edac_check = igen6_check; 1697 mci->pvt_info = &igen6_pvt->imc[mc]; 1698 1699 imc = mci->pvt_info; 1700 imc->dev.release = imc_release; 1701 device_initialize(&imc->dev); 1702 /* 1703 * EDAC core uses mci->pdev(pointer of structure device) as 1704 * memory controller ID. The client SoCs attach one or more 1705 * memory controllers to single pci_dev (single pci_dev->dev 1706 * can be for multiple memory controllers). 1707 * 1708 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev 1709 * for the first memory controller and assign a unique imc->dev 1710 * to mci->pdev for each non-first memory controller. 1711 */ 1712 mci->pdev = mc ? &imc->dev : &pdev->dev; 1713 imc->mc = mc; 1714 imc->pdev = pdev; 1715 imc->window = window; 1716 1717 igen6_reg_dump(imc); 1718 1719 rc = igen6_get_dimm_config(mci); 1720 if (rc) 1721 goto fail3; 1722 1723 rc = edac_mc_add_mc(mci); 1724 if (rc) { 1725 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); 1726 goto fail3; 1727 } 1728 1729 imc->mci = mci; 1730 return 0; 1731 fail3: 1732 put_device(&imc->dev); 1733 mci->pvt_info = NULL; 1734 kfree(mci->ctl_name); 1735 fail2: 1736 edac_mc_free(mci); 1737 fail: 1738 return rc; 1739 } 1740 1741 static void igen6_unregister_mcis(void) 1742 { 1743 struct mem_ctl_info *mci; 1744 struct igen6_imc *imc; 1745 int i; 1746 1747 edac_dbg(2, "\n"); 1748 1749 for (i = 0; i < res_cfg->num_imc; i++) { 1750 imc = &igen6_pvt->imc[i]; 1751 mci = imc->mci; 1752 if (!mci) 1753 continue; 1754 1755 edac_mc_del_mc(mci->pdev); 1756 kfree(mci->ctl_name); 1757 mci->pvt_info = NULL; 1758 edac_mc_free(mci); 1759 put_device(&imc->dev); 1760 iounmap(imc->window); 1761 } 1762 } 1763 1764 static int igen6_register_mcis(struct pci_dev *pdev, u64 mchbar) 1765 { 1766 void __iomem *window; 1767 int lmc, pmc, rc; 1768 1769 for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) { 1770 window = map_imc_window(mchbar, pmc); 1771 if (!window) 1772 continue; 1773 1774 rc = igen6_register_mci(lmc, window, pdev); 1775 if (rc) 1776 goto err_unregister; 1777 1778 /* Done, if all present MCs are detected and registered. */ 1779 if (++lmc >= res_cfg->num_imc) 1780 break; 1781 } 1782 1783 if (!lmc) { 1784 igen6_printk(KERN_ERR, "No mc found.\n"); 1785 return -ENODEV; 1786 } 1787 1788 if (lmc < res_cfg->num_imc) { 1789 igen6_printk(KERN_DEBUG, "Expected %d mcs, but only %d detected.", 1790 res_cfg->num_imc, lmc); 1791 res_cfg->num_imc = lmc; 1792 } 1793 1794 return 0; 1795 1796 err_unregister: 1797 iounmap(window); 1798 igen6_unregister_mcis(); 1799 1800 return rc; 1801 } 1802 1803 static int igen6_mem_slice_setup(u64 mchbar) 1804 { 1805 struct igen6_imc *imc = &igen6_pvt->imc[0]; 1806 u64 base = mchbar + res_cfg->cmf_base; 1807 u32 offset = res_cfg->ms_hash_offset; 1808 u32 size = res_cfg->cmf_size; 1809 u64 ms_s_size, ms_hash; 1810 void __iomem *cmf; 1811 int ms_l_map; 1812 1813 edac_dbg(2, "\n"); 1814 1815 if (imc[0].size < imc[1].size) { 1816 ms_s_size = imc[0].size; 1817 ms_l_map = 1; 1818 } else { 1819 ms_s_size = imc[1].size; 1820 ms_l_map = 0; 1821 } 1822 1823 igen6_pvt->ms_s_size = ms_s_size; 1824 igen6_pvt->ms_l_map = ms_l_map; 1825 1826 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n", 1827 ms_s_size >> 20, ms_l_map); 1828 1829 if (!size) 1830 return 0; 1831 1832 cmf = ioremap(base, size); 1833 if (!cmf) { 1834 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base); 1835 return -ENODEV; 1836 } 1837 1838 ms_hash = readq(cmf + offset); 1839 igen6_pvt->ms_hash = ms_hash; 1840 1841 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash); 1842 1843 iounmap(cmf); 1844 1845 return 0; 1846 } 1847 1848 static int register_err_handler(void) 1849 { 1850 int rc; 1851 1852 if (res_cfg->machine_check) { 1853 mce_register_decode_chain(&ecclog_mce_dec); 1854 return 0; 1855 } 1856 1857 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, 1858 0, IGEN6_NMI_NAME); 1859 if (rc) { 1860 igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); 1861 return rc; 1862 } 1863 1864 return 0; 1865 } 1866 1867 static void unregister_err_handler(void) 1868 { 1869 if (res_cfg->machine_check) { 1870 mce_unregister_decode_chain(&ecclog_mce_dec); 1871 return; 1872 } 1873 1874 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1875 } 1876 1877 static void opstate_set(const struct res_config *cfg, const struct pci_device_id *ent) 1878 { 1879 /* 1880 * Quirk: Certain SoCs' error reporting interrupts don't work. 1881 * Force polling mode for them to ensure that memory error 1882 * events can be handled. 1883 */ 1884 if (ent->device == DID_ADL_N_SKU4) { 1885 edac_op_state = EDAC_OPSTATE_POLL; 1886 return; 1887 } 1888 1889 /* Set the mode according to the configuration data. */ 1890 if (cfg->machine_check) 1891 edac_op_state = EDAC_OPSTATE_INT; 1892 else 1893 edac_op_state = EDAC_OPSTATE_NMI; 1894 } 1895 1896 static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1897 { 1898 u64 mchbar; 1899 int rc; 1900 1901 edac_dbg(2, "\n"); 1902 1903 res_cfg = (struct res_config *)ent->driver_data; 1904 1905 igen6_pvt = igen6_pvt_setup(pdev); 1906 if (!igen6_pvt) 1907 return -ENOMEM; 1908 1909 rc = igen6_pci_setup(pdev, &mchbar); 1910 if (rc) 1911 goto fail; 1912 1913 opstate_set(res_cfg, ent); 1914 1915 rc = igen6_register_mcis(pdev, mchbar); 1916 if (rc) 1917 goto fail; 1918 1919 if (res_cfg->num_imc > 1) { 1920 rc = igen6_mem_slice_setup(mchbar); 1921 if (rc) 1922 goto fail2; 1923 } 1924 1925 ecclog_pool = ecclog_gen_pool_create(); 1926 if (!ecclog_pool) { 1927 rc = -ENOMEM; 1928 goto fail2; 1929 } 1930 1931 INIT_WORK(&ecclog_work, ecclog_work_cb); 1932 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); 1933 1934 rc = register_err_handler(); 1935 if (rc) 1936 goto fail3; 1937 1938 /* Enable error reporting */ 1939 rc = errcmd_enable_error_reporting(true); 1940 if (rc) { 1941 igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); 1942 goto fail4; 1943 } 1944 1945 /* Check if any pending errors before/during the registration of the error handler */ 1946 ecclog_handler(); 1947 1948 igen6_debug_setup(); 1949 return 0; 1950 fail4: 1951 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1952 fail3: 1953 gen_pool_destroy(ecclog_pool); 1954 fail2: 1955 igen6_unregister_mcis(); 1956 fail: 1957 igen6_pvt_release(igen6_pvt); 1958 return rc; 1959 } 1960 1961 static void igen6_remove(struct pci_dev *pdev) 1962 { 1963 edac_dbg(2, "\n"); 1964 1965 igen6_debug_teardown(); 1966 errcmd_enable_error_reporting(false); 1967 unregister_err_handler(); 1968 irq_work_sync(&ecclog_irq_work); 1969 flush_work(&ecclog_work); 1970 gen_pool_destroy(ecclog_pool); 1971 igen6_unregister_mcis(); 1972 igen6_pvt_release(igen6_pvt); 1973 } 1974 1975 static struct pci_driver igen6_driver = { 1976 .name = EDAC_MOD_STR, 1977 .probe = igen6_probe, 1978 .remove = igen6_remove, 1979 .id_table = igen6_pci_tbl, 1980 }; 1981 1982 static int __init igen6_init(void) 1983 { 1984 const char *owner; 1985 int rc; 1986 1987 edac_dbg(2, "\n"); 1988 1989 if (ghes_get_devices()) 1990 return -EBUSY; 1991 1992 owner = edac_get_owner(); 1993 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 1994 return -EBUSY; 1995 1996 rc = pci_register_driver(&igen6_driver); 1997 if (rc) 1998 return rc; 1999 2000 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); 2001 2002 return 0; 2003 } 2004 2005 static void __exit igen6_exit(void) 2006 { 2007 edac_dbg(2, "\n"); 2008 2009 pci_unregister_driver(&igen6_driver); 2010 } 2011 2012 module_init(igen6_init); 2013 module_exit(igen6_exit); 2014 2015 MODULE_LICENSE("GPL v2"); 2016 MODULE_AUTHOR("Qiuxu Zhuo"); 2017 MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); 2018