1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel client SoC with integrated memory controller using IBECC 4 * 5 * Copyright (C) 2020 Intel Corporation 6 * 7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific 8 * regions of the physical memory space. It's used for memory controllers 9 * that don't support the out-of-band ECC which often needs an additional 10 * storage device to each channel for storing ECC data. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 #include <linux/irq_work.h> 18 #include <linux/llist.h> 19 #include <linux/genalloc.h> 20 #include <linux/edac.h> 21 #include <linux/bits.h> 22 #include <linux/bitfield.h> 23 #include <linux/io.h> 24 #include <asm/mach_traps.h> 25 #include <asm/nmi.h> 26 #include <asm/mce.h> 27 28 #include "edac_mc.h" 29 #include "edac_module.h" 30 31 #define IGEN6_REVISION "v2.5.1" 32 33 #define EDAC_MOD_STR "igen6_edac" 34 #define IGEN6_NMI_NAME "igen6_ibecc" 35 36 /* Debug macros */ 37 #define igen6_printk(level, fmt, arg...) \ 38 edac_printk(level, "igen6", fmt, ##arg) 39 40 #define igen6_mc_printk(mci, level, fmt, arg...) \ 41 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) 42 43 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 44 45 #define NUM_IMC 2 /* Max memory controllers */ 46 #define NUM_CHANNELS 2 /* Max channels */ 47 #define NUM_DIMMS 2 /* Max DIMMs per channel */ 48 49 #define _4GB BIT_ULL(32) 50 51 /* Size of physical memory */ 52 #define TOM_OFFSET 0xa0 53 /* Top of low usable DRAM */ 54 #define TOLUD_OFFSET 0xbc 55 /* Capability register C */ 56 #define CAPID_C_OFFSET 0xec 57 #define CAPID_C_IBECC BIT(15) 58 59 /* Capability register E */ 60 #define CAPID_E_OFFSET 0xf0 61 #define CAPID_E_IBECC BIT(12) 62 #define CAPID_E_IBECC_BIT18 BIT(18) 63 64 /* Error Status */ 65 #define ERRSTS_OFFSET 0xc8 66 #define ERRSTS_CE BIT_ULL(6) 67 #define ERRSTS_UE BIT_ULL(7) 68 69 /* Error Command */ 70 #define ERRCMD_OFFSET 0xca 71 #define ERRCMD_CE BIT_ULL(6) 72 #define ERRCMD_UE BIT_ULL(7) 73 74 /* IBECC MMIO base address */ 75 #define IBECC_BASE (res_cfg->ibecc_base) 76 #define IBECC_ACTIVATE_OFFSET IBECC_BASE 77 #define IBECC_ACTIVATE_EN BIT(0) 78 79 /* IBECC error log */ 80 #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset) 81 #define ECC_ERROR_LOG_CE BIT_ULL(62) 82 #define ECC_ERROR_LOG_UE BIT_ULL(63) 83 #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) 84 85 /* Host MMIO base address */ 86 #define MCHBAR_OFFSET 0x48 87 #define MCHBAR_EN BIT_ULL(0) 88 #define MCHBAR_SIZE 0x10000 89 90 /* Parameters for the channel decode stage */ 91 #define IMC_BASE (res_cfg->imc_base) 92 #define MAD_INTER_CHANNEL_OFFSET IMC_BASE 93 #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) 94 #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) 95 #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) 96 #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) 97 98 /* Parameters for DRAM decode stage */ 99 #define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4) 100 #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) 101 102 /* DIMM characteristics */ 103 #define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc) 104 #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) 105 #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) 106 #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) 107 #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) 108 109 /* Hash for memory controller selection */ 110 #define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8) 111 #define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3) 112 113 /* Hash for channel selection */ 114 #define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24) 115 /* Hash for enhanced channel selection */ 116 #define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28) 117 #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 118 #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 119 #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) 120 121 /* Parameters for memory slice decode stage */ 122 #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 123 #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 124 125 struct igen6_imc { 126 int mc; 127 struct mem_ctl_info *mci; 128 struct pci_dev *pdev; 129 struct device dev; 130 void __iomem *window; 131 u64 size; 132 u64 ch_s_size; 133 int ch_l_map; 134 u64 dimm_s_size[NUM_CHANNELS]; 135 u64 dimm_l_size[NUM_CHANNELS]; 136 int dimm_l_map[NUM_CHANNELS]; 137 }; 138 139 static struct res_config { 140 bool machine_check; 141 /* The number of present memory controllers. */ 142 int num_imc; 143 /* Host MMIO configuration */ 144 u64 reg_mchbar_mask; 145 /* Top of memory */ 146 u64 reg_tom_mask; 147 /* Top of upper usable DRAM */ 148 u64 reg_touud_mask; 149 /* IBECC error log */ 150 u64 reg_eccerrlog_addr_mask; 151 /* MEMSS_PMA_CR registers. */ 152 u32 reg_mem_config_offset; 153 u32 reg_mem_config_ddr_type_mask; 154 u32 reg_mem_config_ibecc_en_mask; 155 u32 reg_capabilities_misc_offset; 156 u32 reg_capabilities_misc_ibecc_dis; 157 /* Memory controller registers. */ 158 u32 reg_mad_inter_size_mask[NUM_CHANNELS]; 159 u64 reg_mad_inter_size_granularity; 160 u32 reg_mad_intra_rank_mask[NUM_DIMMS]; 161 u32 reg_mad_intra_width_mask[NUM_DIMMS]; 162 u32 reg_mad_intra_density_mask[NUM_DIMMS]; 163 u32 imc_base; 164 u32 cmf_base; 165 u32 cmf_size; 166 u32 ms_hash_offset; 167 u32 ibecc_base; 168 u32 ibecc_error_log_offset; 169 /* Get memory type. */ 170 enum mem_type (*get_mem_type)(struct igen6_imc *imc); 171 /* Get DRAM chip type. */ 172 enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l); 173 /* Set imc->ch_{s_size,l_map}. */ 174 void (*set_chan_params)(struct igen6_imc *imc); 175 /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */ 176 void (*set_dimm_params)(struct igen6_imc *imc, int chan); 177 bool (*ibecc_available)(struct pci_dev *pdev); 178 /* Extract error address logged in IBECC */ 179 u64 (*err_addr)(u64 ecclog); 180 /* Convert error address logged in IBECC to system physical address */ 181 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); 182 /* Convert error address logged in IBECC to integrated memory controller address */ 183 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); 184 } *res_cfg; 185 186 static struct igen6_pvt { 187 struct igen6_imc imc[NUM_IMC]; 188 void __iomem *memss_pma_cr; 189 u64 ms_hash; 190 u64 ms_s_size; 191 int ms_l_map; 192 } *igen6_pvt; 193 194 /* The top of low usable DRAM */ 195 static u32 igen6_tolud; 196 /* The size of physical memory */ 197 static u64 igen6_tom; 198 199 struct decoded_addr { 200 int mc; 201 u64 imc_addr; 202 u64 sys_addr; 203 int channel_idx; 204 u64 channel_addr; 205 int sub_channel_idx; 206 u64 sub_channel_addr; 207 }; 208 209 struct ecclog_node { 210 struct llist_node llnode; 211 int mc; 212 u64 ecclog; 213 }; 214 215 /* 216 * In the NMI handler, the driver uses the lock-less memory allocator 217 * to allocate memory to store the IBECC error logs and links the logs 218 * to the lock-less list. Delay printk() and the work of error reporting 219 * to EDAC core in a worker. 220 */ 221 #define ECCLOG_POOL_SIZE PAGE_SIZE 222 static LLIST_HEAD(ecclog_llist); 223 static struct gen_pool *ecclog_pool; 224 static char ecclog_buf[ECCLOG_POOL_SIZE]; 225 static struct irq_work ecclog_irq_work; 226 static struct work_struct ecclog_work; 227 228 /* Compute die IDs for Elkhart Lake with IBECC */ 229 #define DID_EHL_SKU5 0x4514 230 #define DID_EHL_SKU6 0x4528 231 #define DID_EHL_SKU7 0x452a 232 #define DID_EHL_SKU8 0x4516 233 #define DID_EHL_SKU9 0x452c 234 #define DID_EHL_SKU10 0x452e 235 #define DID_EHL_SKU11 0x4532 236 #define DID_EHL_SKU12 0x4518 237 #define DID_EHL_SKU13 0x451a 238 #define DID_EHL_SKU14 0x4534 239 #define DID_EHL_SKU15 0x4536 240 241 /* Compute die IDs for ICL-NNPI with IBECC */ 242 #define DID_ICL_SKU8 0x4581 243 #define DID_ICL_SKU10 0x4585 244 #define DID_ICL_SKU11 0x4589 245 #define DID_ICL_SKU12 0x458d 246 247 /* Compute die IDs for Tiger Lake with IBECC */ 248 #define DID_TGL_SKU 0x9a14 249 250 /* Compute die IDs for Alder Lake with IBECC */ 251 #define DID_ADL_SKU1 0x4601 252 #define DID_ADL_SKU2 0x4602 253 #define DID_ADL_SKU3 0x4621 254 #define DID_ADL_SKU4 0x4641 255 256 /* Compute die IDs for Alder Lake-N with IBECC */ 257 #define DID_ADL_N_SKU1 0x4614 258 #define DID_ADL_N_SKU2 0x4617 259 #define DID_ADL_N_SKU3 0x461b 260 #define DID_ADL_N_SKU4 0x461c 261 #define DID_ADL_N_SKU5 0x4673 262 #define DID_ADL_N_SKU6 0x4674 263 #define DID_ADL_N_SKU7 0x4675 264 #define DID_ADL_N_SKU8 0x4677 265 #define DID_ADL_N_SKU9 0x4678 266 #define DID_ADL_N_SKU10 0x4679 267 #define DID_ADL_N_SKU11 0x467c 268 #define DID_ADL_N_SKU12 0x4632 269 270 /* Compute die IDs for Arizona Beach with IBECC */ 271 #define DID_AZB_SKU1 0x4676 272 273 /* Compute did IDs for Amston Lake with IBECC */ 274 #define DID_ASL_SKU1 0x464a 275 #define DID_ASL_SKU2 0x4646 276 #define DID_ASL_SKU3 0x4652 277 278 /* Compute die IDs for Raptor Lake-P with IBECC */ 279 #define DID_RPL_P_SKU1 0xa706 280 #define DID_RPL_P_SKU2 0xa707 281 #define DID_RPL_P_SKU3 0xa708 282 #define DID_RPL_P_SKU4 0xa716 283 #define DID_RPL_P_SKU5 0xa718 284 285 /* Compute die IDs for Meteor Lake-PS with IBECC */ 286 #define DID_MTL_PS_SKU1 0x7d21 287 #define DID_MTL_PS_SKU2 0x7d22 288 #define DID_MTL_PS_SKU3 0x7d23 289 #define DID_MTL_PS_SKU4 0x7d24 290 291 /* Compute die IDs for Meteor Lake-P with IBECC */ 292 #define DID_MTL_P_SKU1 0x7d01 293 #define DID_MTL_P_SKU2 0x7d02 294 #define DID_MTL_P_SKU3 0x7d14 295 296 /* Compute die IDs for Arrow Lake-UH with IBECC */ 297 #define DID_ARL_UH_SKU1 0x7d06 298 #define DID_ARL_UH_SKU2 0x7d20 299 #define DID_ARL_UH_SKU3 0x7d30 300 301 /* Compute die IDs for Panther Lake-H with IBECC */ 302 #define DID_PTL_H_SKU1 0xb000 303 #define DID_PTL_H_SKU2 0xb001 304 #define DID_PTL_H_SKU3 0xb002 305 #define DID_PTL_H_SKU4 0xb003 306 #define DID_PTL_H_SKU5 0xb004 307 #define DID_PTL_H_SKU6 0xb005 308 #define DID_PTL_H_SKU7 0xb008 309 #define DID_PTL_H_SKU8 0xb011 310 #define DID_PTL_H_SKU9 0xb014 311 #define DID_PTL_H_SKU10 0xb015 312 #define DID_PTL_H_SKU11 0xb028 313 #define DID_PTL_H_SKU12 0xb029 314 #define DID_PTL_H_SKU13 0xb02a 315 #define DID_PTL_H_SKU14 0xb00a 316 317 /* Compute die IDs for Wildcat Lake with IBECC */ 318 #define DID_WCL_SKU1 0xfd00 319 320 /* Compute die IDs for Nova Lake-H/HX with IBECC */ 321 #define DID_NVL_H_SKU1 0xd701 322 #define DID_NVL_H_SKU2 0xd702 323 #define DID_NVL_H_SKU3 0xd704 324 #define DID_NVL_H_SKU4 0xd705 325 326 static int get_mchbar(struct pci_dev *pdev, u64 *mchbar) 327 { 328 union { 329 u64 v; 330 struct { 331 u32 v_lo; 332 u32 v_hi; 333 }; 334 } u; 335 336 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { 337 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); 338 return -ENODEV; 339 } 340 341 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { 342 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); 343 return -ENODEV; 344 } 345 346 if (!(u.v & MCHBAR_EN)) { 347 igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); 348 return -ENODEV; 349 } 350 351 *mchbar = u.v & res_cfg->reg_mchbar_mask; 352 edac_dbg(2, "MCHBAR 0x%llx (reg 0x%llx)\n", *mchbar, u.v); 353 354 return 0; 355 } 356 357 static bool ehl_ibecc_available(struct pci_dev *pdev) 358 { 359 u32 v; 360 361 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 362 return false; 363 364 return !!(CAPID_C_IBECC & v); 365 } 366 367 static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc) 368 { 369 return eaddr; 370 } 371 372 static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc) 373 { 374 if (eaddr < igen6_tolud) 375 return eaddr; 376 377 if (igen6_tom <= _4GB) 378 return eaddr + igen6_tolud - _4GB; 379 380 if (eaddr >= igen6_tom) 381 return eaddr + igen6_tolud - igen6_tom; 382 383 return eaddr; 384 } 385 386 static bool icl_ibecc_available(struct pci_dev *pdev) 387 { 388 u32 v; 389 390 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 391 return false; 392 393 return !(CAPID_C_IBECC & v) && 394 (boot_cpu_data.x86_stepping >= 1); 395 } 396 397 static bool tgl_ibecc_available(struct pci_dev *pdev) 398 { 399 u32 v; 400 401 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) 402 return false; 403 404 return !(CAPID_E_IBECC & v); 405 } 406 407 static bool mtl_p_ibecc_available(struct pci_dev *pdev) 408 { 409 u32 v; 410 411 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) 412 return false; 413 414 return !(CAPID_E_IBECC_BIT18 & v); 415 } 416 417 static bool generic_ibecc_available(struct pci_dev *pdev) 418 { 419 void __iomem *base = igen6_pvt->memss_pma_cr; 420 bool present; 421 u32 val; 422 423 if (res_cfg->reg_capabilities_misc_offset) { 424 val = readl(base + res_cfg->reg_capabilities_misc_offset); 425 present = !(val & res_cfg->reg_capabilities_misc_ibecc_dis); 426 edac_dbg(2, "capabilities misc reg 0x%x\n", val); 427 } else if (res_cfg->reg_mem_config_offset) { 428 val = readl(base + res_cfg->reg_mem_config_offset); 429 present = !!(val & res_cfg->reg_mem_config_ibecc_en_mask); 430 edac_dbg(2, "mem config reg 0x%x\n", val); 431 } else { 432 igen6_printk(KERN_ERR, "No register for detecting IBECC presence.\n"); 433 present = false; 434 } 435 436 return present; 437 } 438 439 static u64 mem_addr_to_sys_addr(u64 maddr) 440 { 441 if (maddr < igen6_tolud) 442 return maddr; 443 444 if (igen6_tom <= _4GB) 445 return maddr - igen6_tolud + _4GB; 446 447 if (maddr < _4GB) 448 return maddr - igen6_tolud + igen6_tom; 449 450 return maddr; 451 } 452 453 static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit) 454 { 455 u64 hash_addr = addr & mask, hash = hash_init; 456 u64 intlv = (addr >> intlv_bit) & 1; 457 int i; 458 459 for (i = 6; i < 20; i++) 460 hash ^= (hash_addr >> i) & 1; 461 462 return hash ^ intlv; 463 } 464 465 static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc) 466 { 467 u64 maddr, hash, mask, ms_s_size; 468 int intlv_bit; 469 u32 ms_hash; 470 471 ms_s_size = igen6_pvt->ms_s_size; 472 if (eaddr >= ms_s_size) 473 return eaddr + ms_s_size; 474 475 ms_hash = igen6_pvt->ms_hash; 476 477 mask = MEM_SLICE_HASH_MASK(ms_hash); 478 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6; 479 480 maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) | 481 GET_BITFIELD(eaddr, 0, intlv_bit - 1); 482 483 hash = mem_slice_hash(maddr, mask, mc, intlv_bit); 484 485 return maddr | (hash << intlv_bit); 486 } 487 488 static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc) 489 { 490 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc); 491 492 return mem_addr_to_sys_addr(maddr); 493 } 494 495 static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc) 496 { 497 return eaddr; 498 } 499 500 static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc) 501 { 502 return mem_addr_to_sys_addr(eaddr); 503 } 504 505 static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc) 506 { 507 u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size; 508 struct igen6_imc *imc = &igen6_pvt->imc[mc]; 509 int intlv_bit; 510 u32 mc_hash; 511 512 if (eaddr >= 2 * ms_s_size) 513 return eaddr - ms_s_size; 514 515 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET); 516 517 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6; 518 519 imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit | 520 GET_BITFIELD(eaddr, 0, intlv_bit - 1); 521 522 return imc_addr; 523 } 524 525 static u64 rpl_p_err_addr(u64 ecclog) 526 { 527 return field_get(res_cfg->reg_eccerrlog_addr_mask, ecclog); 528 } 529 530 static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc) 531 { 532 u32 mtype, val; 533 534 val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset); 535 mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val); 536 537 edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val); 538 539 switch (mtype) { 540 case 1: 541 return MEM_DDR5; 542 case 2: 543 return MEM_LPDDR5; 544 case 3: 545 return MEM_LPDDR4; 546 default: 547 return MEM_UNKNOWN; 548 } 549 } 550 551 static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm) 552 { 553 u32 width, val; 554 555 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 556 width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val); 557 558 switch (width) { 559 case 1: 560 return DEV_X8; 561 default: 562 return DEV_X16; 563 } 564 } 565 566 static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan) 567 { 568 u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 569 570 return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) * 571 res_cfg->reg_mad_inter_size_granularity; 572 } 573 574 static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm) 575 { 576 u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 577 u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val); 578 /* DRAM device density in Gb */ 579 u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4; 580 581 enum mem_type mtype = ptl_h_get_mem_type(imc); 582 enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm); 583 u64 sub_ch_width, dev_num; 584 585 switch (mtype) { 586 case MEM_DDR5: 587 sub_ch_width = 32; 588 break; 589 case MEM_LPDDR5: 590 case MEM_LPDDR4: 591 sub_ch_width = 16; 592 break; 593 default: 594 sub_ch_width = 0; 595 } 596 597 switch (dtype) { 598 case DEV_X8: 599 dev_num = sub_ch_width / 8; 600 break; 601 case DEV_X16: 602 dev_num = sub_ch_width / 16; 603 break; 604 default: 605 dev_num = 0; 606 } 607 608 edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val); 609 610 return ((dev_num * density / 8) * ranks) << 30; 611 } 612 613 static void ptl_h_set_chan_params(struct igen6_imc *imc) 614 { 615 u64 ch0_size = ptl_h_get_chan_size(imc, 0); 616 u64 ch1_size = ptl_h_get_chan_size(imc, 1); 617 618 if (ch0_size <= ch1_size) { 619 imc->ch_s_size = ch0_size; 620 imc->ch_l_map = 1; 621 } else { 622 imc->ch_s_size = ch1_size; 623 imc->ch_l_map = 0; 624 } 625 } 626 627 static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan) 628 { 629 u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0); 630 u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1); 631 632 if (dimm0_size <= dimm1_size) { 633 imc->dimm_s_size[chan] = dimm0_size; 634 imc->dimm_l_size[chan] = dimm1_size; 635 imc->dimm_l_map[chan] = 1; 636 } else { 637 imc->dimm_s_size[chan] = dimm1_size; 638 imc->dimm_l_size[chan] = dimm0_size; 639 imc->dimm_l_map[chan] = 0; 640 } 641 } 642 643 static struct res_config ehl_cfg = { 644 .num_imc = 1, 645 .reg_mchbar_mask = GENMASK_ULL(38, 16), 646 .reg_tom_mask = GENMASK_ULL(38, 20), 647 .reg_touud_mask = GENMASK_ULL(38, 20), 648 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 649 .imc_base = 0x5000, 650 .ibecc_base = 0xdc00, 651 .ibecc_available = ehl_ibecc_available, 652 .ibecc_error_log_offset = 0x170, 653 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 654 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 655 }; 656 657 static struct res_config icl_cfg = { 658 .num_imc = 1, 659 .reg_mchbar_mask = GENMASK_ULL(38, 16), 660 .reg_tom_mask = GENMASK_ULL(38, 20), 661 .reg_touud_mask = GENMASK_ULL(38, 20), 662 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 663 .imc_base = 0x5000, 664 .ibecc_base = 0xd800, 665 .ibecc_error_log_offset = 0x170, 666 .ibecc_available = icl_ibecc_available, 667 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 668 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 669 }; 670 671 static struct res_config tgl_cfg = { 672 .machine_check = true, 673 .num_imc = 2, 674 .reg_mchbar_mask = GENMASK_ULL(38, 17), 675 .reg_tom_mask = GENMASK_ULL(38, 20), 676 .reg_touud_mask = GENMASK_ULL(38, 20), 677 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 678 .imc_base = 0x5000, 679 .cmf_base = 0x11000, 680 .cmf_size = 0x800, 681 .ms_hash_offset = 0xac, 682 .ibecc_base = 0xd400, 683 .ibecc_error_log_offset = 0x170, 684 .ibecc_available = tgl_ibecc_available, 685 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr, 686 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr, 687 }; 688 689 static struct res_config adl_cfg = { 690 .machine_check = true, 691 .num_imc = 2, 692 .reg_mchbar_mask = GENMASK_ULL(41, 17), 693 .reg_tom_mask = GENMASK_ULL(41, 20), 694 .reg_touud_mask = GENMASK_ULL(41, 20), 695 .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5), 696 .imc_base = 0xd800, 697 .ibecc_base = 0xd400, 698 .ibecc_error_log_offset = 0x68, 699 .ibecc_available = tgl_ibecc_available, 700 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 701 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 702 }; 703 704 static struct res_config adl_n_cfg = { 705 .machine_check = true, 706 .num_imc = 1, 707 .reg_mchbar_mask = GENMASK_ULL(41, 17), 708 .reg_tom_mask = GENMASK_ULL(41, 20), 709 .reg_touud_mask = GENMASK_ULL(41, 20), 710 .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5), 711 .imc_base = 0xd800, 712 .ibecc_base = 0xd400, 713 .ibecc_error_log_offset = 0x68, 714 .ibecc_available = tgl_ibecc_available, 715 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 716 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 717 }; 718 719 static struct res_config rpl_p_cfg = { 720 .machine_check = true, 721 .num_imc = 2, 722 .reg_mchbar_mask = GENMASK_ULL(41, 17), 723 .reg_tom_mask = GENMASK_ULL(41, 20), 724 .reg_touud_mask = GENMASK_ULL(41, 20), 725 .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5), 726 .imc_base = 0xd800, 727 .ibecc_base = 0xd400, 728 .ibecc_error_log_offset = 0x68, 729 .ibecc_available = tgl_ibecc_available, 730 .err_addr = rpl_p_err_addr, 731 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 732 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 733 }; 734 735 static struct res_config mtl_ps_cfg = { 736 .machine_check = true, 737 .num_imc = 2, 738 .reg_mchbar_mask = GENMASK_ULL(41, 17), 739 .reg_tom_mask = GENMASK_ULL(41, 20), 740 .reg_touud_mask = GENMASK_ULL(41, 20), 741 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 742 .reg_capabilities_misc_offset = 0x13c00, 743 .reg_capabilities_misc_ibecc_dis = BIT(6), 744 .imc_base = 0xd800, 745 .ibecc_base = 0xd400, 746 .ibecc_error_log_offset = 0x170, 747 .ibecc_available = generic_ibecc_available, 748 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 749 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 750 }; 751 752 static struct res_config mtl_p_cfg = { 753 .machine_check = true, 754 .num_imc = 2, 755 .reg_mchbar_mask = GENMASK_ULL(41, 17), 756 .reg_tom_mask = GENMASK_ULL(41, 20), 757 .reg_touud_mask = GENMASK_ULL(41, 20), 758 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 759 .imc_base = 0xd800, 760 .ibecc_base = 0xd400, 761 .ibecc_error_log_offset = 0x170, 762 .ibecc_available = mtl_p_ibecc_available, 763 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 764 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 765 }; 766 767 static struct res_config ptl_h_cfg = { 768 .machine_check = true, 769 .num_imc = 2, 770 .reg_mchbar_mask = GENMASK_ULL(41, 17), 771 .reg_tom_mask = GENMASK_ULL(41, 20), 772 .reg_touud_mask = GENMASK_ULL(41, 20), 773 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 774 .reg_mem_config_offset = 0x13d04, 775 .reg_mem_config_ddr_type_mask = GENMASK(8, 6), 776 .reg_mad_inter_size_mask[0] = GENMASK(15, 8), 777 .reg_mad_inter_size_mask[1] = GENMASK(23, 16), 778 .reg_mad_inter_size_granularity = BIT_ULL(29), 779 .reg_mad_intra_rank_mask[0] = BIT(7), 780 .reg_mad_intra_rank_mask[1] = BIT(15), 781 .reg_mad_intra_width_mask[0] = BIT(6), 782 .reg_mad_intra_width_mask[1] = BIT(14), 783 .reg_mad_intra_density_mask[0] = GENMASK(3, 0), 784 .reg_mad_intra_density_mask[1] = GENMASK(11, 8), 785 .imc_base = 0xd800, 786 .ibecc_base = 0xd400, 787 .ibecc_error_log_offset = 0x170, 788 .get_mem_type = ptl_h_get_mem_type, 789 .get_dev_type = ptl_h_get_dev_type, 790 .set_chan_params = ptl_h_set_chan_params, 791 .set_dimm_params = ptl_h_set_dimm_params, 792 .ibecc_available = mtl_p_ibecc_available, 793 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 794 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 795 }; 796 797 static struct res_config wcl_cfg = { 798 .machine_check = true, 799 .num_imc = 1, 800 .reg_mchbar_mask = GENMASK_ULL(41, 17), 801 .reg_tom_mask = GENMASK_ULL(41, 20), 802 .reg_touud_mask = GENMASK_ULL(41, 20), 803 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 804 .imc_base = 0xd800, 805 .ibecc_base = 0xd400, 806 .ibecc_error_log_offset = 0x170, 807 .ibecc_available = mtl_p_ibecc_available, 808 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 809 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 810 }; 811 812 static struct res_config nvl_h_cfg = { 813 .machine_check = true, 814 .num_imc = 2, 815 .reg_mchbar_mask = GENMASK_ULL(41, 17), 816 .reg_tom_mask = GENMASK_ULL(41, 20), 817 .reg_touud_mask = GENMASK_ULL(41, 20), 818 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), 819 .reg_mem_config_offset = 0x12904, 820 .reg_mem_config_ddr_type_mask = GENMASK(8, 6), 821 .reg_mem_config_ibecc_en_mask = GENMASK(3, 2), 822 .reg_mad_inter_size_mask[0] = GENMASK(15, 8), 823 .reg_mad_inter_size_mask[1] = GENMASK(23, 16), 824 .reg_mad_inter_size_granularity = BIT_ULL(29), 825 .reg_mad_intra_rank_mask[0] = BIT(7), 826 .reg_mad_intra_rank_mask[1] = BIT(15), 827 .reg_mad_intra_width_mask[0] = BIT(6), 828 .reg_mad_intra_width_mask[1] = BIT(14), 829 .reg_mad_intra_density_mask[0] = GENMASK(3, 0), 830 .reg_mad_intra_density_mask[1] = GENMASK(11, 8), 831 .imc_base = 0xd800, 832 .ibecc_base = 0xd400, 833 .ibecc_error_log_offset = 0x170, 834 .get_mem_type = ptl_h_get_mem_type, 835 .get_dev_type = ptl_h_get_dev_type, 836 .set_chan_params = ptl_h_set_chan_params, 837 .set_dimm_params = ptl_h_set_dimm_params, 838 .ibecc_available = generic_ibecc_available, 839 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 840 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 841 }; 842 843 static struct pci_device_id igen6_pci_tbl[] = { 844 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), .driver_data = (kernel_ulong_t)&ehl_cfg }, 845 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), .driver_data = (kernel_ulong_t)&ehl_cfg }, 846 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), .driver_data = (kernel_ulong_t)&ehl_cfg }, 847 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), .driver_data = (kernel_ulong_t)&ehl_cfg }, 848 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), .driver_data = (kernel_ulong_t)&ehl_cfg }, 849 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), .driver_data = (kernel_ulong_t)&ehl_cfg }, 850 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), .driver_data = (kernel_ulong_t)&ehl_cfg }, 851 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), .driver_data = (kernel_ulong_t)&ehl_cfg }, 852 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), .driver_data = (kernel_ulong_t)&ehl_cfg }, 853 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), .driver_data = (kernel_ulong_t)&ehl_cfg }, 854 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), .driver_data = (kernel_ulong_t)&ehl_cfg }, 855 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), .driver_data = (kernel_ulong_t)&icl_cfg }, 856 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), .driver_data = (kernel_ulong_t)&icl_cfg }, 857 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), .driver_data = (kernel_ulong_t)&icl_cfg }, 858 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), .driver_data = (kernel_ulong_t)&icl_cfg }, 859 { PCI_VDEVICE(INTEL, DID_TGL_SKU), .driver_data = (kernel_ulong_t)&tgl_cfg }, 860 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, 861 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, 862 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, 863 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, 864 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 865 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 866 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 867 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 868 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 869 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 870 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 871 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 872 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 873 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 874 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 875 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 876 { PCI_VDEVICE(INTEL, DID_AZB_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 877 { PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 878 { PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 879 { PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_n_cfg }, 880 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, 881 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, 882 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, 883 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, 884 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, 885 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 886 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 887 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 888 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, 889 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 890 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 891 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 892 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 893 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 894 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, 895 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 896 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 897 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 898 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 899 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 900 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 901 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 902 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 903 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 904 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 905 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 906 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 907 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 908 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, 909 { PCI_VDEVICE(INTEL, DID_WCL_SKU1), .driver_data = (kernel_ulong_t)&wcl_cfg }, 910 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU1), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 911 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU2), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 912 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU3), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 913 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU4), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, 914 { }, 915 }; 916 MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); 917 918 static enum mem_type get_mem_type(struct igen6_imc *imc) 919 { 920 u32 val; 921 922 if (res_cfg->get_mem_type) 923 return res_cfg->get_mem_type(imc); 924 925 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 926 927 switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) { 928 case 0: 929 return MEM_DDR4; 930 case 1: 931 return MEM_DDR3; 932 case 2: 933 return MEM_LPDDR3; 934 case 3: 935 return MEM_LPDDR4; 936 case 4: 937 return MEM_WIO2; 938 default: 939 return MEM_UNKNOWN; 940 } 941 } 942 943 static bool large_dimm(struct igen6_imc *imc, int chan, int dimm) 944 { 945 return dimm == imc->dimm_l_map[chan]; 946 } 947 948 static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm) 949 { 950 u32 width, val; 951 952 if (res_cfg->get_dev_type) 953 return res_cfg->get_dev_type(imc, chan, dimm); 954 955 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); 956 width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) : 957 MAD_DIMM_CH_DSW(val); 958 959 switch (width) { 960 case 0: 961 return DEV_X8; 962 case 1: 963 return DEV_X16; 964 case 2: 965 return DEV_X32; 966 default: 967 return DEV_UNKNOWN; 968 } 969 } 970 971 static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm) 972 { 973 if (large_dimm(imc, chan, dimm)) 974 return imc->dimm_l_size[chan]; 975 976 return imc->dimm_s_size[chan]; 977 } 978 979 static void set_chan_params(struct igen6_imc *imc) 980 { 981 u32 val; 982 983 if (res_cfg->set_chan_params) { 984 res_cfg->set_chan_params(imc); 985 return; 986 } 987 988 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 989 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val); 990 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val); 991 } 992 993 static void set_dimm_params(struct igen6_imc *imc, int chan) 994 { 995 u32 val; 996 997 if (res_cfg->set_dimm_params) { 998 res_cfg->set_dimm_params(imc, chan); 999 return; 1000 } 1001 1002 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); 1003 imc->dimm_l_map[chan] = MAD_INTRA_CH_DIMM_L_MAP(val); 1004 1005 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); 1006 imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val); 1007 imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val); 1008 } 1009 1010 static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) 1011 { 1012 u64 hash_addr = addr & mask, hash = 0; 1013 u64 intlv = (addr >> intlv_bit) & 1; 1014 int i; 1015 1016 for (i = 6; i < 20; i++) 1017 hash ^= (hash_addr >> i) & 1; 1018 1019 return (int)hash ^ intlv; 1020 } 1021 1022 static u64 decode_channel_addr(u64 addr, int intlv_bit) 1023 { 1024 u64 channel_addr; 1025 1026 /* Remove the interleave bit and shift upper part down to fill gap */ 1027 channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; 1028 channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); 1029 1030 return channel_addr; 1031 } 1032 1033 static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, 1034 int *idx, u64 *sub_addr) 1035 { 1036 int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; 1037 1038 if (addr > 2 * s_size) { 1039 *sub_addr = addr - s_size; 1040 *idx = l_map; 1041 return; 1042 } 1043 1044 if (CHANNEL_HASH_MODE(hash)) { 1045 *sub_addr = decode_channel_addr(addr, intlv_bit); 1046 *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); 1047 } else { 1048 *sub_addr = decode_channel_addr(addr, 6); 1049 *idx = GET_BITFIELD(addr, 6, 6); 1050 } 1051 } 1052 1053 static int igen6_decode(struct decoded_addr *res) 1054 { 1055 struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; 1056 u64 addr = res->imc_addr, sub_addr, s_size; 1057 int idx, l_map; 1058 u32 hash; 1059 1060 if (addr >= igen6_tom) { 1061 edac_dbg(0, "Address 0x%llx out of range\n", addr); 1062 return -EINVAL; 1063 } 1064 1065 /* Decode channel */ 1066 hash = readl(imc->window + CHANNEL_HASH_OFFSET); 1067 s_size = imc->ch_s_size; 1068 l_map = imc->ch_l_map; 1069 decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); 1070 res->channel_idx = idx; 1071 res->channel_addr = sub_addr; 1072 1073 /* Decode sub-channel/DIMM */ 1074 hash = readl(imc->window + CHANNEL_EHASH_OFFSET); 1075 s_size = imc->dimm_s_size[idx]; 1076 l_map = imc->dimm_l_map[idx]; 1077 decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); 1078 res->sub_channel_idx = idx; 1079 res->sub_channel_addr = sub_addr; 1080 1081 return 0; 1082 } 1083 1084 static void igen6_output_error(struct decoded_addr *res, 1085 struct mem_ctl_info *mci, u64 ecclog) 1086 { 1087 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? 1088 HW_EVENT_ERR_UNCORRECTED : 1089 HW_EVENT_ERR_CORRECTED; 1090 1091 edac_mc_handle_error(type, mci, 1, 1092 res->sys_addr >> PAGE_SHIFT, 1093 res->sys_addr & ~PAGE_MASK, 1094 ECC_ERROR_LOG_SYND(ecclog), 1095 res->channel_idx, res->sub_channel_idx, 1096 -1, "", ""); 1097 } 1098 1099 static struct gen_pool *ecclog_gen_pool_create(void) 1100 { 1101 struct gen_pool *pool; 1102 1103 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); 1104 if (!pool) 1105 return NULL; 1106 1107 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { 1108 gen_pool_destroy(pool); 1109 return NULL; 1110 } 1111 1112 return pool; 1113 } 1114 1115 static int ecclog_gen_pool_add(int mc, u64 ecclog) 1116 { 1117 struct ecclog_node *node; 1118 1119 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); 1120 if (!node) 1121 return -ENOMEM; 1122 1123 node->mc = mc; 1124 node->ecclog = ecclog; 1125 llist_add(&node->llnode, &ecclog_llist); 1126 1127 return 0; 1128 } 1129 1130 /* 1131 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI 1132 * configuration space status register ERRSTS can indicate whether a 1133 * correctable error or an uncorrectable error occurred. We only use the 1134 * ECC_ERROR_LOG register to check error type, but need to clear both 1135 * registers to enable future error events. 1136 */ 1137 static u64 ecclog_read_and_clear(struct igen6_imc *imc) 1138 { 1139 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); 1140 1141 /* 1142 * Quirk: The ECC_ERROR_LOG register of certain SoCs may contain 1143 * the invalid value ~0. This will result in a flood of invalid 1144 * error reports in polling mode. Skip it. 1145 */ 1146 if (ecclog == ~0) 1147 return 0; 1148 1149 /* Neither a CE nor a UE. Skip it.*/ 1150 if (!(ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE))) 1151 return 0; 1152 1153 /* Clear CE/UE bits by writing 1s */ 1154 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); 1155 1156 return ecclog; 1157 } 1158 1159 static void errsts_clear(struct igen6_imc *imc) 1160 { 1161 u16 errsts; 1162 1163 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { 1164 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); 1165 return; 1166 } 1167 1168 /* Clear CE/UE bits by writing 1s */ 1169 if (errsts & (ERRSTS_CE | ERRSTS_UE)) 1170 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); 1171 } 1172 1173 static int errcmd_enable_error_reporting(bool enable) 1174 { 1175 struct igen6_imc *imc = &igen6_pvt->imc[0]; 1176 u16 errcmd; 1177 int rc; 1178 1179 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); 1180 if (rc) 1181 return pcibios_err_to_errno(rc); 1182 1183 if (enable) 1184 errcmd |= ERRCMD_CE | ERRSTS_UE; 1185 else 1186 errcmd &= ~(ERRCMD_CE | ERRSTS_UE); 1187 1188 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); 1189 if (rc) 1190 return pcibios_err_to_errno(rc); 1191 1192 return 0; 1193 } 1194 1195 static int ecclog_handler(void) 1196 { 1197 struct igen6_imc *imc; 1198 int i, n = 0; 1199 u64 ecclog; 1200 1201 for (i = 0; i < res_cfg->num_imc; i++) { 1202 imc = &igen6_pvt->imc[i]; 1203 1204 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 1205 1206 ecclog = ecclog_read_and_clear(imc); 1207 if (!ecclog) 1208 continue; 1209 1210 if (!ecclog_gen_pool_add(i, ecclog)) 1211 irq_work_queue(&ecclog_irq_work); 1212 1213 n++; 1214 } 1215 1216 return n; 1217 } 1218 1219 static void ecclog_work_cb(struct work_struct *work) 1220 { 1221 struct ecclog_node *node, *tmp; 1222 struct mem_ctl_info *mci; 1223 struct llist_node *head; 1224 struct decoded_addr res; 1225 u64 eaddr; 1226 1227 head = llist_del_all(&ecclog_llist); 1228 if (!head) 1229 return; 1230 1231 llist_for_each_entry_safe(node, tmp, head, llnode) { 1232 memset(&res, 0, sizeof(res)); 1233 if (res_cfg->err_addr) 1234 eaddr = res_cfg->err_addr(node->ecclog); 1235 else 1236 eaddr = node->ecclog & res_cfg->reg_eccerrlog_addr_mask; 1237 1238 res.mc = node->mc; 1239 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc); 1240 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc); 1241 1242 mci = igen6_pvt->imc[res.mc].mci; 1243 1244 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); 1245 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); 1246 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); 1247 1248 if (!igen6_decode(&res)) 1249 igen6_output_error(&res, mci, node->ecclog); 1250 1251 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); 1252 } 1253 } 1254 1255 static void ecclog_irq_work_cb(struct irq_work *irq_work) 1256 { 1257 int i; 1258 1259 for (i = 0; i < res_cfg->num_imc; i++) 1260 errsts_clear(&igen6_pvt->imc[i]); 1261 1262 if (!llist_empty(&ecclog_llist)) 1263 schedule_work(&ecclog_work); 1264 } 1265 1266 static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1267 { 1268 unsigned char reason; 1269 1270 if (!ecclog_handler()) 1271 return NMI_DONE; 1272 1273 /* 1274 * Both In-Band ECC correctable error and uncorrectable error are 1275 * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) 1276 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to 1277 * re-enable the SERR# NMI after NMI handling. So clear this bit here 1278 * to re-enable SERR# NMI for receiving future In-Band ECC errors. 1279 */ 1280 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; 1281 reason |= NMI_REASON_CLEAR_SERR; 1282 outb(reason, NMI_REASON_PORT); 1283 reason &= ~NMI_REASON_CLEAR_SERR; 1284 outb(reason, NMI_REASON_PORT); 1285 1286 return NMI_HANDLED; 1287 } 1288 1289 static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val, 1290 void *data) 1291 { 1292 struct mce *mce = (struct mce *)data; 1293 char *type; 1294 1295 if (mce->kflags & MCE_HANDLED_CEC) 1296 return NOTIFY_DONE; 1297 1298 /* 1299 * Ignore unless this is a memory related error. 1300 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here, 1301 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3). 1302 */ 1303 if ((mce->status & 0xefff) >> 7 != 1) 1304 return NOTIFY_DONE; 1305 1306 if (mce->mcgstatus & MCG_STATUS_MCIP) 1307 type = "Exception"; 1308 else 1309 type = "Event"; 1310 1311 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n", 1312 mce->extcpu, type, mce->mcgstatus, 1313 mce->bank, mce->status); 1314 edac_dbg(0, "TSC 0x%llx\n", mce->tsc); 1315 edac_dbg(0, "ADDR 0x%llx\n", mce->addr); 1316 edac_dbg(0, "MISC 0x%llx\n", mce->misc); 1317 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n", 1318 mce->cpuvendor, mce->cpuid, mce->time, 1319 mce->socketid, mce->apicid); 1320 /* 1321 * We just use the Machine Check for the memory error notification. 1322 * Each memory controller is associated with an IBECC instance. 1323 * Directly read and clear the error information(error address and 1324 * error type) on all the IBECC instances so that we know on which 1325 * memory controller the memory error(s) occurred. 1326 */ 1327 if (!ecclog_handler()) 1328 return NOTIFY_DONE; 1329 1330 mce->kflags |= MCE_HANDLED_EDAC; 1331 1332 return NOTIFY_DONE; 1333 } 1334 1335 static struct notifier_block ecclog_mce_dec = { 1336 .notifier_call = ecclog_mce_handler, 1337 .priority = MCE_PRIO_EDAC, 1338 }; 1339 1340 static bool igen6_check_ecc(struct igen6_imc *imc) 1341 { 1342 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); 1343 1344 return !!(activate & IBECC_ACTIVATE_EN); 1345 } 1346 1347 static int igen6_get_dimm_config(struct mem_ctl_info *mci) 1348 { 1349 struct igen6_imc *imc = mci->pvt_info; 1350 int i, j, ndimms, mc = imc->mc; 1351 struct dimm_info *dimm; 1352 enum mem_type mtype; 1353 enum dev_type dtype; 1354 u64 dsize; 1355 bool ecc; 1356 1357 edac_dbg(2, "\n"); 1358 1359 mtype = get_mem_type(imc); 1360 ecc = igen6_check_ecc(imc); 1361 set_chan_params(imc); 1362 1363 for (i = 0; i < NUM_CHANNELS; i++) { 1364 set_dimm_params(imc, i); 1365 imc->size += imc->dimm_s_size[i]; 1366 imc->size += imc->dimm_l_size[i]; 1367 ndimms = 0; 1368 1369 for (j = 0; j < NUM_DIMMS; j++) { 1370 dimm = edac_get_dimm(mci, i, j, 0); 1371 dtype = get_dev_type(imc, i, j); 1372 dsize = get_dimm_size(imc, i, j); 1373 1374 if (!dsize) 1375 continue; 1376 1377 dimm->grain = 64; 1378 dimm->mtype = mtype; 1379 dimm->dtype = dtype; 1380 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); 1381 dimm->edac_mode = EDAC_SECDED; 1382 snprintf(dimm->label, sizeof(dimm->label), 1383 "MC#%d_Chan#%d_DIMM#%d", mc, i, j); 1384 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", 1385 mc, i, j, dsize >> 20, dimm->nr_pages); 1386 1387 ndimms++; 1388 } 1389 1390 if (ndimms && !ecc) { 1391 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); 1392 return -ENODEV; 1393 } 1394 } 1395 1396 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20); 1397 1398 return 0; 1399 } 1400 1401 #ifdef CONFIG_EDAC_DEBUG 1402 /* Top of upper usable DRAM */ 1403 static u64 igen6_touud; 1404 #define TOUUD_OFFSET 0xa8 1405 1406 static void igen6_reg_dump(struct igen6_imc *imc) 1407 { 1408 int i; 1409 1410 edac_dbg(2, "CHANNEL_HASH : 0x%x\n", 1411 readl(imc->window + CHANNEL_HASH_OFFSET)); 1412 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", 1413 readl(imc->window + CHANNEL_EHASH_OFFSET)); 1414 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", 1415 readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); 1416 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", 1417 readq(imc->window + ECC_ERROR_LOG_OFFSET)); 1418 1419 for (i = 0; i < NUM_CHANNELS; i++) { 1420 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, 1421 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); 1422 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, 1423 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); 1424 } 1425 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); 1426 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); 1427 edac_dbg(2, "TOM : 0x%llx", igen6_tom); 1428 } 1429 1430 static struct dentry *igen6_test; 1431 1432 static int debugfs_u64_set(void *data, u64 val) 1433 { 1434 u64 ecclog; 1435 1436 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { 1437 edac_dbg(0, "Address 0x%llx out of range\n", val); 1438 return 0; 1439 } 1440 1441 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); 1442 1443 ecclog = (val & res_cfg->reg_eccerrlog_addr_mask) | ECC_ERROR_LOG_CE; 1444 1445 if (!ecclog_gen_pool_add(0, ecclog)) 1446 irq_work_queue(&ecclog_irq_work); 1447 1448 return 0; 1449 } 1450 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 1451 1452 static void igen6_debug_setup(void) 1453 { 1454 igen6_test = edac_debugfs_create_dir("igen6_test"); 1455 if (!igen6_test) 1456 return; 1457 1458 if (!edac_debugfs_create_file("addr", 0200, igen6_test, 1459 NULL, &fops_u64_wo)) { 1460 debugfs_remove(igen6_test); 1461 igen6_test = NULL; 1462 } 1463 } 1464 1465 static void igen6_debug_teardown(void) 1466 { 1467 debugfs_remove_recursive(igen6_test); 1468 } 1469 #else 1470 static void igen6_reg_dump(struct igen6_imc *imc) {} 1471 static void igen6_debug_setup(void) {} 1472 static void igen6_debug_teardown(void) {} 1473 #endif 1474 1475 static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev) 1476 { 1477 void __iomem *memss_pma_cr; 1478 struct igen6_pvt *pvt; 1479 u64 mchbar; 1480 int rc; 1481 1482 pvt = kzalloc_obj(*igen6_pvt); 1483 if (!pvt) 1484 return NULL; 1485 1486 rc = get_mchbar(pdev, &mchbar); 1487 if (rc) { 1488 kfree(pvt); 1489 return NULL; 1490 } 1491 1492 memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2); 1493 if (!memss_pma_cr) { 1494 kfree(pvt); 1495 return NULL; 1496 } 1497 pvt->memss_pma_cr = memss_pma_cr; 1498 1499 return pvt; 1500 } 1501 1502 static void igen6_pvt_release(struct igen6_pvt *pvt) 1503 { 1504 iounmap(pvt->memss_pma_cr); 1505 kfree(pvt); 1506 } 1507 1508 static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) 1509 { 1510 union { 1511 u64 v; 1512 struct { 1513 u32 v_lo; 1514 u32 v_hi; 1515 }; 1516 } u; 1517 1518 edac_dbg(2, "\n"); 1519 1520 if (!res_cfg->ibecc_available(pdev)) { 1521 edac_dbg(2, "No In-Band ECC IP\n"); 1522 goto fail; 1523 } 1524 1525 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { 1526 igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); 1527 goto fail; 1528 } 1529 1530 igen6_tolud &= GENMASK(31, 20); 1531 1532 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { 1533 igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); 1534 goto fail; 1535 } 1536 1537 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { 1538 igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); 1539 goto fail; 1540 } 1541 1542 igen6_tom = u.v & res_cfg->reg_tom_mask; 1543 1544 if (get_mchbar(pdev, mchbar)) 1545 goto fail; 1546 1547 #ifdef CONFIG_EDAC_DEBUG 1548 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) 1549 edac_dbg(2, "Failed to read lower TOUUD\n"); 1550 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) 1551 edac_dbg(2, "Failed to read upper TOUUD\n"); 1552 else 1553 igen6_touud = u.v & res_cfg->reg_touud_mask; 1554 #endif 1555 1556 return 0; 1557 fail: 1558 return -ENODEV; 1559 } 1560 1561 static void igen6_check(struct mem_ctl_info *mci) 1562 { 1563 struct igen6_imc *imc = mci->pvt_info; 1564 u64 ecclog; 1565 1566 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 1567 ecclog = ecclog_read_and_clear(imc); 1568 if (!ecclog) 1569 return; 1570 1571 if (!ecclog_gen_pool_add(imc->mc, ecclog)) 1572 irq_work_queue(&ecclog_irq_work); 1573 } 1574 1575 /* Check whether the memory controller is absent. */ 1576 static bool igen6_imc_absent(void __iomem *window) 1577 { 1578 return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0; 1579 } 1580 1581 static void imc_release(struct device *dev) 1582 { 1583 /* Nothing to do, the 'imc' owns the 'dev' and will also release it. */ 1584 } 1585 1586 static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev) 1587 { 1588 struct edac_mc_layer layers[2]; 1589 struct mem_ctl_info *mci; 1590 struct igen6_imc *imc; 1591 int rc; 1592 1593 edac_dbg(2, "\n"); 1594 1595 layers[0].type = EDAC_MC_LAYER_CHANNEL; 1596 layers[0].size = NUM_CHANNELS; 1597 layers[0].is_virt_csrow = false; 1598 layers[1].type = EDAC_MC_LAYER_SLOT; 1599 layers[1].size = NUM_DIMMS; 1600 layers[1].is_virt_csrow = true; 1601 1602 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); 1603 if (!mci) { 1604 rc = -ENOMEM; 1605 goto fail; 1606 } 1607 1608 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); 1609 if (!mci->ctl_name) { 1610 rc = -ENOMEM; 1611 goto fail2; 1612 } 1613 1614 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; 1615 mci->edac_ctl_cap = EDAC_FLAG_SECDED; 1616 mci->edac_cap = EDAC_FLAG_SECDED; 1617 mci->mod_name = EDAC_MOD_STR; 1618 mci->dev_name = pci_name(pdev); 1619 if (edac_op_state == EDAC_OPSTATE_POLL) 1620 mci->edac_check = igen6_check; 1621 mci->pvt_info = &igen6_pvt->imc[mc]; 1622 1623 imc = mci->pvt_info; 1624 imc->dev.release = imc_release; 1625 device_initialize(&imc->dev); 1626 /* 1627 * EDAC core uses mci->pdev(pointer of structure device) as 1628 * memory controller ID. The client SoCs attach one or more 1629 * memory controllers to single pci_dev (single pci_dev->dev 1630 * can be for multiple memory controllers). 1631 * 1632 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev 1633 * for the first memory controller and assign a unique imc->dev 1634 * to mci->pdev for each non-first memory controller. 1635 */ 1636 mci->pdev = mc ? &imc->dev : &pdev->dev; 1637 imc->mc = mc; 1638 imc->pdev = pdev; 1639 imc->window = window; 1640 1641 igen6_reg_dump(imc); 1642 1643 rc = igen6_get_dimm_config(mci); 1644 if (rc) 1645 goto fail3; 1646 1647 rc = edac_mc_add_mc(mci); 1648 if (rc) { 1649 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); 1650 goto fail3; 1651 } 1652 1653 imc->mci = mci; 1654 return 0; 1655 fail3: 1656 put_device(&imc->dev); 1657 mci->pvt_info = NULL; 1658 kfree(mci->ctl_name); 1659 fail2: 1660 edac_mc_free(mci); 1661 fail: 1662 return rc; 1663 } 1664 1665 static void igen6_unregister_mcis(void) 1666 { 1667 struct mem_ctl_info *mci; 1668 struct igen6_imc *imc; 1669 int i; 1670 1671 edac_dbg(2, "\n"); 1672 1673 for (i = 0; i < res_cfg->num_imc; i++) { 1674 imc = &igen6_pvt->imc[i]; 1675 mci = imc->mci; 1676 if (!mci) 1677 continue; 1678 1679 edac_mc_del_mc(mci->pdev); 1680 kfree(mci->ctl_name); 1681 mci->pvt_info = NULL; 1682 edac_mc_free(mci); 1683 put_device(&imc->dev); 1684 iounmap(imc->window); 1685 } 1686 } 1687 1688 static int igen6_register_mcis(struct pci_dev *pdev, u64 mchbar) 1689 { 1690 void __iomem *window; 1691 int lmc, pmc, rc; 1692 u64 base; 1693 1694 for (lmc = 0, pmc = 0; pmc < NUM_IMC; pmc++) { 1695 base = mchbar + pmc * MCHBAR_SIZE; 1696 window = ioremap(base, MCHBAR_SIZE); 1697 if (!window) { 1698 igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx for mc%d\n", base, pmc); 1699 rc = -ENOMEM; 1700 goto out_unregister_mcis; 1701 } 1702 1703 if (igen6_imc_absent(window)) { 1704 iounmap(window); 1705 edac_dbg(2, "Skip absent mc%d\n", pmc); 1706 continue; 1707 } 1708 1709 rc = igen6_register_mci(lmc, window, pdev); 1710 if (rc) 1711 goto out_iounmap; 1712 1713 /* Done, if all present MCs are detected and registered. */ 1714 if (++lmc >= res_cfg->num_imc) 1715 break; 1716 } 1717 1718 if (!lmc) { 1719 igen6_printk(KERN_ERR, "No mc found.\n"); 1720 return -ENODEV; 1721 } 1722 1723 if (lmc < res_cfg->num_imc) { 1724 igen6_printk(KERN_DEBUG, "Expected %d mcs, but only %d detected.", 1725 res_cfg->num_imc, lmc); 1726 res_cfg->num_imc = lmc; 1727 } 1728 1729 return 0; 1730 1731 out_iounmap: 1732 iounmap(window); 1733 1734 out_unregister_mcis: 1735 igen6_unregister_mcis(); 1736 1737 return rc; 1738 } 1739 1740 static int igen6_mem_slice_setup(u64 mchbar) 1741 { 1742 struct igen6_imc *imc = &igen6_pvt->imc[0]; 1743 u64 base = mchbar + res_cfg->cmf_base; 1744 u32 offset = res_cfg->ms_hash_offset; 1745 u32 size = res_cfg->cmf_size; 1746 u64 ms_s_size, ms_hash; 1747 void __iomem *cmf; 1748 int ms_l_map; 1749 1750 edac_dbg(2, "\n"); 1751 1752 if (imc[0].size < imc[1].size) { 1753 ms_s_size = imc[0].size; 1754 ms_l_map = 1; 1755 } else { 1756 ms_s_size = imc[1].size; 1757 ms_l_map = 0; 1758 } 1759 1760 igen6_pvt->ms_s_size = ms_s_size; 1761 igen6_pvt->ms_l_map = ms_l_map; 1762 1763 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n", 1764 ms_s_size >> 20, ms_l_map); 1765 1766 if (!size) 1767 return 0; 1768 1769 cmf = ioremap(base, size); 1770 if (!cmf) { 1771 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base); 1772 return -ENODEV; 1773 } 1774 1775 ms_hash = readq(cmf + offset); 1776 igen6_pvt->ms_hash = ms_hash; 1777 1778 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash); 1779 1780 iounmap(cmf); 1781 1782 return 0; 1783 } 1784 1785 static int register_err_handler(void) 1786 { 1787 int rc; 1788 1789 if (res_cfg->machine_check) { 1790 mce_register_decode_chain(&ecclog_mce_dec); 1791 return 0; 1792 } 1793 1794 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, 1795 0, IGEN6_NMI_NAME); 1796 if (rc) { 1797 igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); 1798 return rc; 1799 } 1800 1801 return 0; 1802 } 1803 1804 static void unregister_err_handler(void) 1805 { 1806 if (res_cfg->machine_check) { 1807 mce_unregister_decode_chain(&ecclog_mce_dec); 1808 return; 1809 } 1810 1811 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1812 } 1813 1814 static void opstate_set(const struct res_config *cfg, const struct pci_device_id *ent) 1815 { 1816 /* 1817 * Quirk: Certain SoCs' error reporting interrupts don't work. 1818 * Force polling mode for them to ensure that memory error 1819 * events can be handled. 1820 */ 1821 if (ent->device == DID_ADL_N_SKU4) { 1822 edac_op_state = EDAC_OPSTATE_POLL; 1823 return; 1824 } 1825 1826 /* Set the mode according to the configuration data. */ 1827 if (cfg->machine_check) 1828 edac_op_state = EDAC_OPSTATE_INT; 1829 else 1830 edac_op_state = EDAC_OPSTATE_NMI; 1831 } 1832 1833 static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1834 { 1835 u64 mchbar; 1836 int rc; 1837 1838 edac_dbg(2, "\n"); 1839 1840 res_cfg = (struct res_config *)ent->driver_data; 1841 1842 igen6_pvt = igen6_pvt_setup(pdev); 1843 if (!igen6_pvt) 1844 return -ENOMEM; 1845 1846 rc = igen6_pci_setup(pdev, &mchbar); 1847 if (rc) 1848 goto fail; 1849 1850 opstate_set(res_cfg, ent); 1851 1852 rc = igen6_register_mcis(pdev, mchbar); 1853 if (rc) 1854 goto fail; 1855 1856 if (res_cfg->num_imc > 1) { 1857 rc = igen6_mem_slice_setup(mchbar); 1858 if (rc) 1859 goto fail2; 1860 } 1861 1862 ecclog_pool = ecclog_gen_pool_create(); 1863 if (!ecclog_pool) { 1864 rc = -ENOMEM; 1865 goto fail2; 1866 } 1867 1868 INIT_WORK(&ecclog_work, ecclog_work_cb); 1869 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); 1870 1871 rc = register_err_handler(); 1872 if (rc) 1873 goto fail3; 1874 1875 /* Enable error reporting */ 1876 rc = errcmd_enable_error_reporting(true); 1877 if (rc) { 1878 igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); 1879 goto fail4; 1880 } 1881 1882 /* Check if any pending errors before/during the registration of the error handler */ 1883 ecclog_handler(); 1884 1885 igen6_debug_setup(); 1886 return 0; 1887 fail4: 1888 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1889 fail3: 1890 gen_pool_destroy(ecclog_pool); 1891 fail2: 1892 igen6_unregister_mcis(); 1893 fail: 1894 igen6_pvt_release(igen6_pvt); 1895 return rc; 1896 } 1897 1898 static void igen6_remove(struct pci_dev *pdev) 1899 { 1900 edac_dbg(2, "\n"); 1901 1902 igen6_debug_teardown(); 1903 errcmd_enable_error_reporting(false); 1904 unregister_err_handler(); 1905 irq_work_sync(&ecclog_irq_work); 1906 flush_work(&ecclog_work); 1907 gen_pool_destroy(ecclog_pool); 1908 igen6_unregister_mcis(); 1909 igen6_pvt_release(igen6_pvt); 1910 } 1911 1912 static struct pci_driver igen6_driver = { 1913 .name = EDAC_MOD_STR, 1914 .probe = igen6_probe, 1915 .remove = igen6_remove, 1916 .id_table = igen6_pci_tbl, 1917 }; 1918 1919 static int __init igen6_init(void) 1920 { 1921 const char *owner; 1922 int rc; 1923 1924 edac_dbg(2, "\n"); 1925 1926 if (ghes_get_devices()) 1927 return -EBUSY; 1928 1929 owner = edac_get_owner(); 1930 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 1931 return -EBUSY; 1932 1933 rc = pci_register_driver(&igen6_driver); 1934 if (rc) 1935 return rc; 1936 1937 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); 1938 1939 return 0; 1940 } 1941 1942 static void __exit igen6_exit(void) 1943 { 1944 edac_dbg(2, "\n"); 1945 1946 pci_unregister_driver(&igen6_driver); 1947 } 1948 1949 module_init(igen6_init); 1950 module_exit(igen6_exit); 1951 1952 MODULE_LICENSE("GPL v2"); 1953 MODULE_AUTHOR("Qiuxu Zhuo"); 1954 MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); 1955