1 /* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong 2 because MTRRs can span upto 40 bits (36bits on most modern x86) */ 3 #include <linux/init.h> 4 #include <linux/slab.h> 5 #include <linux/mm.h> 6 #include <linux/module.h> 7 #include <asm/io.h> 8 #include <asm/mtrr.h> 9 #include <asm/msr.h> 10 #include <asm/system.h> 11 #include <asm/cpufeature.h> 12 #include <asm/processor-flags.h> 13 #include <asm/tlbflush.h> 14 #include "mtrr.h" 15 16 struct mtrr_state { 17 struct mtrr_var_range var_ranges[MAX_VAR_RANGES]; 18 mtrr_type fixed_ranges[NUM_FIXED_RANGES]; 19 unsigned char enabled; 20 unsigned char have_fixed; 21 mtrr_type def_type; 22 }; 23 24 struct fixed_range_block { 25 int base_msr; /* start address of an MTRR block */ 26 int ranges; /* number of MTRRs in this block */ 27 }; 28 29 static struct fixed_range_block fixed_range_blocks[] = { 30 { MTRRfix64K_00000_MSR, 1 }, /* one 64k MTRR */ 31 { MTRRfix16K_80000_MSR, 2 }, /* two 16k MTRRs */ 32 { MTRRfix4K_C0000_MSR, 8 }, /* eight 4k MTRRs */ 33 {} 34 }; 35 36 static unsigned long smp_changes_mask; 37 static struct mtrr_state mtrr_state = {}; 38 39 #undef MODULE_PARAM_PREFIX 40 #define MODULE_PARAM_PREFIX "mtrr." 41 42 static int mtrr_show; 43 module_param_named(show, mtrr_show, bool, 0); 44 45 /* Get the MSR pair relating to a var range */ 46 static void 47 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) 48 { 49 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 50 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 51 } 52 53 static void 54 get_fixed_ranges(mtrr_type * frs) 55 { 56 unsigned int *p = (unsigned int *) frs; 57 int i; 58 59 rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]); 60 61 for (i = 0; i < 2; i++) 62 rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]); 63 for (i = 0; i < 8; i++) 64 rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]); 65 } 66 67 void mtrr_save_fixed_ranges(void *info) 68 { 69 if (cpu_has_mtrr) 70 get_fixed_ranges(mtrr_state.fixed_ranges); 71 } 72 73 static void print_fixed(unsigned base, unsigned step, const mtrr_type*types) 74 { 75 unsigned i; 76 77 for (i = 0; i < 8; ++i, ++types, base += step) 78 printk(KERN_INFO "MTRR %05X-%05X %s\n", 79 base, base + step - 1, mtrr_attrib_to_str(*types)); 80 } 81 82 /* Grab all of the MTRR state for this CPU into *state */ 83 void __init get_mtrr_state(void) 84 { 85 unsigned int i; 86 struct mtrr_var_range *vrs; 87 unsigned lo, dummy; 88 89 vrs = mtrr_state.var_ranges; 90 91 rdmsr(MTRRcap_MSR, lo, dummy); 92 mtrr_state.have_fixed = (lo >> 8) & 1; 93 94 for (i = 0; i < num_var_ranges; i++) 95 get_mtrr_var_range(i, &vrs[i]); 96 if (mtrr_state.have_fixed) 97 get_fixed_ranges(mtrr_state.fixed_ranges); 98 99 rdmsr(MTRRdefType_MSR, lo, dummy); 100 mtrr_state.def_type = (lo & 0xff); 101 mtrr_state.enabled = (lo & 0xc00) >> 10; 102 103 if (mtrr_show) { 104 int high_width; 105 106 printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type)); 107 if (mtrr_state.have_fixed) { 108 printk(KERN_INFO "MTRR fixed ranges %sabled:\n", 109 mtrr_state.enabled & 1 ? "en" : "dis"); 110 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0); 111 for (i = 0; i < 2; ++i) 112 print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8); 113 for (i = 0; i < 8; ++i) 114 print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8); 115 } 116 printk(KERN_INFO "MTRR variable ranges %sabled:\n", 117 mtrr_state.enabled & 2 ? "en" : "dis"); 118 high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4; 119 for (i = 0; i < num_var_ranges; ++i) { 120 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11)) 121 printk(KERN_INFO "MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n", 122 i, 123 high_width, 124 mtrr_state.var_ranges[i].base_hi, 125 mtrr_state.var_ranges[i].base_lo >> 12, 126 high_width, 127 mtrr_state.var_ranges[i].mask_hi, 128 mtrr_state.var_ranges[i].mask_lo >> 12, 129 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff)); 130 else 131 printk(KERN_INFO "MTRR %u disabled\n", i); 132 } 133 } 134 } 135 136 /* Some BIOS's are fucked and don't set all MTRRs the same! */ 137 void __init mtrr_state_warn(void) 138 { 139 unsigned long mask = smp_changes_mask; 140 141 if (!mask) 142 return; 143 if (mask & MTRR_CHANGE_MASK_FIXED) 144 printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n"); 145 if (mask & MTRR_CHANGE_MASK_VARIABLE) 146 printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n"); 147 if (mask & MTRR_CHANGE_MASK_DEFTYPE) 148 printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n"); 149 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n"); 150 printk(KERN_INFO "mtrr: corrected configuration.\n"); 151 } 152 153 /* Doesn't attempt to pass an error out to MTRR users 154 because it's quite complicated in some cases and probably not 155 worth it because the best error handling is to ignore it. */ 156 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) 157 { 158 if (wrmsr_safe(msr, a, b) < 0) 159 printk(KERN_ERR 160 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", 161 smp_processor_id(), msr, a, b); 162 } 163 164 /** 165 * Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs 166 * see AMD publication no. 24593, chapter 3.2.1 for more information 167 */ 168 static inline void k8_enable_fixed_iorrs(void) 169 { 170 unsigned lo, hi; 171 172 rdmsr(MSR_K8_SYSCFG, lo, hi); 173 mtrr_wrmsr(MSR_K8_SYSCFG, lo 174 | K8_MTRRFIXRANGE_DRAM_ENABLE 175 | K8_MTRRFIXRANGE_DRAM_MODIFY, hi); 176 } 177 178 /** 179 * Checks and updates an fixed-range MTRR if it differs from the value it 180 * should have. If K8 extentions are wanted, update the K8 SYSCFG MSR also. 181 * see AMD publication no. 24593, chapter 7.8.1, page 233 for more information 182 * \param msr MSR address of the MTTR which should be checked and updated 183 * \param changed pointer which indicates whether the MTRR needed to be changed 184 * \param msrwords pointer to the MSR values which the MSR should have 185 */ 186 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) 187 { 188 unsigned lo, hi; 189 190 rdmsr(msr, lo, hi); 191 192 if (lo != msrwords[0] || hi != msrwords[1]) { 193 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 194 boot_cpu_data.x86 == 15 && 195 ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK)) 196 k8_enable_fixed_iorrs(); 197 mtrr_wrmsr(msr, msrwords[0], msrwords[1]); 198 *changed = true; 199 } 200 } 201 202 int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) 203 /* [SUMMARY] Get a free MTRR. 204 <base> The starting (base) address of the region. 205 <size> The size (in bytes) of the region. 206 [RETURNS] The index of the region on success, else -1 on error. 207 */ 208 { 209 int i, max; 210 mtrr_type ltype; 211 unsigned long lbase, lsize; 212 213 max = num_var_ranges; 214 if (replace_reg >= 0 && replace_reg < max) 215 return replace_reg; 216 for (i = 0; i < max; ++i) { 217 mtrr_if->get(i, &lbase, &lsize, <ype); 218 if (lsize == 0) 219 return i; 220 } 221 return -ENOSPC; 222 } 223 224 static void generic_get_mtrr(unsigned int reg, unsigned long *base, 225 unsigned long *size, mtrr_type *type) 226 { 227 unsigned int mask_lo, mask_hi, base_lo, base_hi; 228 229 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); 230 if ((mask_lo & 0x800) == 0) { 231 /* Invalid (i.e. free) range */ 232 *base = 0; 233 *size = 0; 234 *type = 0; 235 return; 236 } 237 238 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); 239 240 /* Work out the shifted address mask. */ 241 mask_lo = size_or_mask | mask_hi << (32 - PAGE_SHIFT) 242 | mask_lo >> PAGE_SHIFT; 243 244 /* This works correctly if size is a power of two, i.e. a 245 contiguous range. */ 246 *size = -mask_lo; 247 *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; 248 *type = base_lo & 0xff; 249 } 250 251 /** 252 * Checks and updates the fixed-range MTRRs if they differ from the saved set 253 * \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges() 254 */ 255 static int set_fixed_ranges(mtrr_type * frs) 256 { 257 unsigned long long *saved = (unsigned long long *) frs; 258 bool changed = false; 259 int block=-1, range; 260 261 while (fixed_range_blocks[++block].ranges) 262 for (range=0; range < fixed_range_blocks[block].ranges; range++) 263 set_fixed_range(fixed_range_blocks[block].base_msr + range, 264 &changed, (unsigned int *) saved++); 265 266 return changed; 267 } 268 269 /* Set the MSR pair relating to a var range. Returns TRUE if 270 changes are made */ 271 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) 272 { 273 unsigned int lo, hi; 274 bool changed = false; 275 276 rdmsr(MTRRphysBase_MSR(index), lo, hi); 277 if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL) 278 || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != 279 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { 280 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 281 changed = true; 282 } 283 284 rdmsr(MTRRphysMask_MSR(index), lo, hi); 285 286 if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL) 287 || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != 288 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { 289 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 290 changed = true; 291 } 292 return changed; 293 } 294 295 static u32 deftype_lo, deftype_hi; 296 297 static unsigned long set_mtrr_state(void) 298 /* [SUMMARY] Set the MTRR state for this CPU. 299 <state> The MTRR state information to read. 300 <ctxt> Some relevant CPU context. 301 [NOTE] The CPU must already be in a safe state for MTRR changes. 302 [RETURNS] 0 if no changes made, else a mask indication what was changed. 303 */ 304 { 305 unsigned int i; 306 unsigned long change_mask = 0; 307 308 for (i = 0; i < num_var_ranges; i++) 309 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i])) 310 change_mask |= MTRR_CHANGE_MASK_VARIABLE; 311 312 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges)) 313 change_mask |= MTRR_CHANGE_MASK_FIXED; 314 315 /* Set_mtrr_restore restores the old value of MTRRdefType, 316 so to set it we fiddle with the saved value */ 317 if ((deftype_lo & 0xff) != mtrr_state.def_type 318 || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) { 319 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10); 320 change_mask |= MTRR_CHANGE_MASK_DEFTYPE; 321 } 322 323 return change_mask; 324 } 325 326 327 static unsigned long cr4 = 0; 328 static DEFINE_SPINLOCK(set_atomicity_lock); 329 330 /* 331 * Since we are disabling the cache don't allow any interrupts - they 332 * would run extremely slow and would only increase the pain. The caller must 333 * ensure that local interrupts are disabled and are reenabled after post_set() 334 * has been called. 335 */ 336 337 static void prepare_set(void) __acquires(set_atomicity_lock) 338 { 339 unsigned long cr0; 340 341 /* Note that this is not ideal, since the cache is only flushed/disabled 342 for this CPU while the MTRRs are changed, but changing this requires 343 more invasive changes to the way the kernel boots */ 344 345 spin_lock(&set_atomicity_lock); 346 347 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */ 348 cr0 = read_cr0() | X86_CR0_CD; 349 write_cr0(cr0); 350 wbinvd(); 351 352 /* Save value of CR4 and clear Page Global Enable (bit 7) */ 353 if ( cpu_has_pge ) { 354 cr4 = read_cr4(); 355 write_cr4(cr4 & ~X86_CR4_PGE); 356 } 357 358 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ 359 __flush_tlb(); 360 361 /* Save MTRR state */ 362 rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); 363 364 /* Disable MTRRs, and set the default type to uncached */ 365 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi); 366 } 367 368 static void post_set(void) __releases(set_atomicity_lock) 369 { 370 /* Flush TLBs (no need to flush caches - they are disabled) */ 371 __flush_tlb(); 372 373 /* Intel (P6) standard MTRRs */ 374 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); 375 376 /* Enable caches */ 377 write_cr0(read_cr0() & 0xbfffffff); 378 379 /* Restore value of CR4 */ 380 if ( cpu_has_pge ) 381 write_cr4(cr4); 382 spin_unlock(&set_atomicity_lock); 383 } 384 385 static void generic_set_all(void) 386 { 387 unsigned long mask, count; 388 unsigned long flags; 389 390 local_irq_save(flags); 391 prepare_set(); 392 393 /* Actually set the state */ 394 mask = set_mtrr_state(); 395 396 post_set(); 397 local_irq_restore(flags); 398 399 /* Use the atomic bitops to update the global mask */ 400 for (count = 0; count < sizeof mask * 8; ++count) { 401 if (mask & 0x01) 402 set_bit(count, &smp_changes_mask); 403 mask >>= 1; 404 } 405 406 } 407 408 static void generic_set_mtrr(unsigned int reg, unsigned long base, 409 unsigned long size, mtrr_type type) 410 /* [SUMMARY] Set variable MTRR register on the local CPU. 411 <reg> The register to set. 412 <base> The base address of the region. 413 <size> The size of the region. If this is 0 the region is disabled. 414 <type> The type of the region. 415 [RETURNS] Nothing. 416 */ 417 { 418 unsigned long flags; 419 struct mtrr_var_range *vr; 420 421 vr = &mtrr_state.var_ranges[reg]; 422 423 local_irq_save(flags); 424 prepare_set(); 425 426 if (size == 0) { 427 /* The invalid bit is kept in the mask, so we simply clear the 428 relevant mask register to disable a range. */ 429 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0); 430 memset(vr, 0, sizeof(struct mtrr_var_range)); 431 } else { 432 vr->base_lo = base << PAGE_SHIFT | type; 433 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT); 434 vr->mask_lo = -size << PAGE_SHIFT | 0x800; 435 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT); 436 437 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi); 438 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi); 439 } 440 441 post_set(); 442 local_irq_restore(flags); 443 } 444 445 int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type) 446 { 447 unsigned long lbase, last; 448 449 /* For Intel PPro stepping <= 7, must be 4 MiB aligned 450 and not touch 0x70000000->0x7003FFFF */ 451 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 && 452 boot_cpu_data.x86_model == 1 && 453 boot_cpu_data.x86_mask <= 7) { 454 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) { 455 printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base); 456 return -EINVAL; 457 } 458 if (!(base + size < 0x70000 || base > 0x7003F) && 459 (type == MTRR_TYPE_WRCOMB 460 || type == MTRR_TYPE_WRBACK)) { 461 printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n"); 462 return -EINVAL; 463 } 464 } 465 466 /* Check upper bits of base and last are equal and lower bits are 0 467 for base and 1 for last */ 468 last = base + size - 1; 469 for (lbase = base; !(lbase & 1) && (last & 1); 470 lbase = lbase >> 1, last = last >> 1) ; 471 if (lbase != last) { 472 printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", 473 base, size); 474 return -EINVAL; 475 } 476 return 0; 477 } 478 479 480 static int generic_have_wrcomb(void) 481 { 482 unsigned long config, dummy; 483 rdmsr(MTRRcap_MSR, config, dummy); 484 return (config & (1 << 10)); 485 } 486 487 int positive_have_wrcomb(void) 488 { 489 return 1; 490 } 491 492 /* generic structure... 493 */ 494 struct mtrr_ops generic_mtrr_ops = { 495 .use_intel_if = 1, 496 .set_all = generic_set_all, 497 .get = generic_get_mtrr, 498 .get_free_region = generic_get_free_region, 499 .set = generic_set_mtrr, 500 .validate_add_page = generic_validate_add_page, 501 .have_wrcomb = generic_have_wrcomb, 502 }; 503