1 // SPDX-License-Identifier: LGPL-2.0+ 2 /* 3 * MTRR (Memory Type Range Register) cleanup 4 * 5 * Copyright (C) 2009 Yinghai Lu 6 */ 7 #include <linux/init.h> 8 #include <linux/pci.h> 9 #include <linux/smp.h> 10 #include <linux/cpu.h> 11 #include <linux/mutex.h> 12 #include <linux/uaccess.h> 13 #include <linux/kvm_para.h> 14 #include <linux/range.h> 15 16 #include <asm/processor.h> 17 #include <asm/e820/api.h> 18 #include <asm/mtrr.h> 19 #include <asm/msr.h> 20 21 #include "mtrr.h" 22 23 struct var_mtrr_range_state { 24 unsigned long base_pfn; 25 unsigned long size_pfn; 26 mtrr_type type; 27 }; 28 29 struct var_mtrr_state { 30 unsigned long range_startk; 31 unsigned long range_sizek; 32 unsigned long chunk_sizek; 33 unsigned long gran_sizek; 34 unsigned int reg; 35 }; 36 37 /* Should be related to MTRR_VAR_RANGES nums */ 38 #define RANGE_NUM 256 39 40 static struct range __initdata range[RANGE_NUM]; 41 static int __initdata nr_range; 42 43 static struct var_mtrr_range_state __initdata range_state[RANGE_NUM]; 44 45 #define BIOS_BUG_MSG \ 46 "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n" 47 48 static int __init 49 x86_get_mtrr_mem_range(struct range *range, int nr_range, 50 unsigned long extra_remove_base, 51 unsigned long extra_remove_size) 52 { 53 unsigned long base, size; 54 mtrr_type type; 55 int i; 56 57 for (i = 0; i < num_var_ranges; i++) { 58 type = range_state[i].type; 59 if (type != MTRR_TYPE_WRBACK) 60 continue; 61 base = range_state[i].base_pfn; 62 size = range_state[i].size_pfn; 63 nr_range = add_range_with_merge(range, RANGE_NUM, nr_range, 64 base, base + size); 65 } 66 67 Dprintk("After WB checking\n"); 68 for (i = 0; i < nr_range; i++) 69 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 70 range[i].start, range[i].end); 71 72 /* Take out UC ranges: */ 73 for (i = 0; i < num_var_ranges; i++) { 74 type = range_state[i].type; 75 if (type != MTRR_TYPE_UNCACHABLE && 76 type != MTRR_TYPE_WRPROT) 77 continue; 78 size = range_state[i].size_pfn; 79 if (!size) 80 continue; 81 base = range_state[i].base_pfn; 82 if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed && 83 (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) && 84 (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) { 85 /* Var MTRR contains UC entry below 1M? Skip it: */ 86 pr_warn(BIOS_BUG_MSG, i); 87 if (base + size <= (1<<(20-PAGE_SHIFT))) 88 continue; 89 size -= (1<<(20-PAGE_SHIFT)) - base; 90 base = 1<<(20-PAGE_SHIFT); 91 } 92 subtract_range(range, RANGE_NUM, base, base + size); 93 } 94 if (extra_remove_size) 95 subtract_range(range, RANGE_NUM, extra_remove_base, 96 extra_remove_base + extra_remove_size); 97 98 Dprintk("After UC checking\n"); 99 for (i = 0; i < RANGE_NUM; i++) { 100 if (!range[i].end) 101 continue; 102 103 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 104 range[i].start, range[i].end); 105 } 106 107 /* sort the ranges */ 108 nr_range = clean_sort_range(range, RANGE_NUM); 109 110 Dprintk("After sorting\n"); 111 for (i = 0; i < nr_range; i++) 112 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 113 range[i].start, range[i].end); 114 115 return nr_range; 116 } 117 118 #ifdef CONFIG_MTRR_SANITIZER 119 120 static unsigned long __init sum_ranges(struct range *range, int nr_range) 121 { 122 unsigned long sum = 0; 123 int i; 124 125 for (i = 0; i < nr_range; i++) 126 sum += range[i].end - range[i].start; 127 128 return sum; 129 } 130 131 static int enable_mtrr_cleanup __initdata = 132 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT; 133 134 static int __init disable_mtrr_cleanup_setup(char *str) 135 { 136 enable_mtrr_cleanup = 0; 137 return 0; 138 } 139 early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup); 140 141 static int __init enable_mtrr_cleanup_setup(char *str) 142 { 143 enable_mtrr_cleanup = 1; 144 return 0; 145 } 146 early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup); 147 148 static void __init 149 set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 150 unsigned char type) 151 { 152 u32 base_lo, base_hi, mask_lo, mask_hi; 153 u64 base, mask; 154 155 if (!sizek) { 156 fill_mtrr_var_range(reg, 0, 0, 0, 0); 157 return; 158 } 159 160 mask = (1ULL << boot_cpu_data.x86_phys_bits) - 1; 161 mask &= ~((((u64)sizek) << 10) - 1); 162 163 base = ((u64)basek) << 10; 164 165 base |= type; 166 mask |= 0x800; 167 168 base_lo = base & ((1ULL<<32) - 1); 169 base_hi = base >> 32; 170 171 mask_lo = mask & ((1ULL<<32) - 1); 172 mask_hi = mask >> 32; 173 174 fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi); 175 } 176 177 static void __init 178 save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 179 unsigned char type) 180 { 181 range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10); 182 range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10); 183 range_state[reg].type = type; 184 } 185 186 static void __init set_var_mtrr_all(void) 187 { 188 unsigned long basek, sizek; 189 unsigned char type; 190 unsigned int reg; 191 192 for (reg = 0; reg < num_var_ranges; reg++) { 193 basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10); 194 sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10); 195 type = range_state[reg].type; 196 197 set_var_mtrr(reg, basek, sizek, type); 198 } 199 } 200 201 static unsigned long to_size_factor(unsigned long sizek, char *factorp) 202 { 203 unsigned long base = sizek; 204 char factor; 205 206 if (base & ((1<<10) - 1)) { 207 /* Not MB-aligned: */ 208 factor = 'K'; 209 } else if (base & ((1<<20) - 1)) { 210 factor = 'M'; 211 base >>= 10; 212 } else { 213 factor = 'G'; 214 base >>= 20; 215 } 216 217 *factorp = factor; 218 219 return base; 220 } 221 222 static unsigned int __init 223 range_to_mtrr(unsigned int reg, unsigned long range_startk, 224 unsigned long range_sizek, unsigned char type) 225 { 226 if (!range_sizek || (reg >= num_var_ranges)) 227 return reg; 228 229 while (range_sizek) { 230 unsigned long max_align, align; 231 unsigned long sizek; 232 233 /* Compute the maximum size with which we can make a range: */ 234 if (range_startk) 235 max_align = __ffs(range_startk); 236 else 237 max_align = BITS_PER_LONG - 1; 238 239 align = __fls(range_sizek); 240 if (align > max_align) 241 align = max_align; 242 243 sizek = 1UL << align; 244 if (mtrr_debug) { 245 char start_factor = 'K', size_factor = 'K'; 246 unsigned long start_base, size_base; 247 248 start_base = to_size_factor(range_startk, &start_factor); 249 size_base = to_size_factor(sizek, &size_factor); 250 251 Dprintk("Setting variable MTRR %d, " 252 "base: %ld%cB, range: %ld%cB, type %s\n", 253 reg, start_base, start_factor, 254 size_base, size_factor, 255 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 256 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other") 257 ); 258 } 259 save_var_mtrr(reg++, range_startk, sizek, type); 260 range_startk += sizek; 261 range_sizek -= sizek; 262 if (reg >= num_var_ranges) 263 break; 264 } 265 return reg; 266 } 267 268 static unsigned __init 269 range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek, 270 unsigned long sizek) 271 { 272 unsigned long hole_basek, hole_sizek; 273 unsigned long second_sizek; 274 unsigned long range0_basek, range0_sizek; 275 unsigned long range_basek, range_sizek; 276 unsigned long chunk_sizek; 277 unsigned long gran_sizek; 278 279 hole_basek = 0; 280 hole_sizek = 0; 281 second_sizek = 0; 282 chunk_sizek = state->chunk_sizek; 283 gran_sizek = state->gran_sizek; 284 285 /* Align with gran size, prevent small block used up MTRRs: */ 286 range_basek = ALIGN(state->range_startk, gran_sizek); 287 if ((range_basek > basek) && basek) 288 return second_sizek; 289 290 state->range_sizek -= (range_basek - state->range_startk); 291 range_sizek = ALIGN(state->range_sizek, gran_sizek); 292 293 while (range_sizek > state->range_sizek) { 294 range_sizek -= gran_sizek; 295 if (!range_sizek) 296 return 0; 297 } 298 state->range_sizek = range_sizek; 299 300 /* Try to append some small hole: */ 301 range0_basek = state->range_startk; 302 range0_sizek = ALIGN(state->range_sizek, chunk_sizek); 303 304 /* No increase: */ 305 if (range0_sizek == state->range_sizek) { 306 Dprintk("rangeX: %016lx - %016lx\n", 307 range0_basek<<10, 308 (range0_basek + state->range_sizek)<<10); 309 state->reg = range_to_mtrr(state->reg, range0_basek, 310 state->range_sizek, MTRR_TYPE_WRBACK); 311 return 0; 312 } 313 314 /* Only cut back when it is not the last: */ 315 if (sizek) { 316 while (range0_basek + range0_sizek > (basek + sizek)) { 317 if (range0_sizek >= chunk_sizek) 318 range0_sizek -= chunk_sizek; 319 else 320 range0_sizek = 0; 321 322 if (!range0_sizek) 323 break; 324 } 325 } 326 327 second_try: 328 range_basek = range0_basek + range0_sizek; 329 330 /* One hole in the middle: */ 331 if (range_basek > basek && range_basek <= (basek + sizek)) 332 second_sizek = range_basek - basek; 333 334 if (range0_sizek > state->range_sizek) { 335 336 /* One hole in middle or at the end: */ 337 hole_sizek = range0_sizek - state->range_sizek - second_sizek; 338 339 /* Hole size should be less than half of range0 size: */ 340 if (hole_sizek >= (range0_sizek >> 1) && 341 range0_sizek >= chunk_sizek) { 342 range0_sizek -= chunk_sizek; 343 second_sizek = 0; 344 hole_sizek = 0; 345 346 goto second_try; 347 } 348 } 349 350 if (range0_sizek) { 351 Dprintk("range0: %016lx - %016lx\n", 352 range0_basek<<10, 353 (range0_basek + range0_sizek)<<10); 354 state->reg = range_to_mtrr(state->reg, range0_basek, 355 range0_sizek, MTRR_TYPE_WRBACK); 356 } 357 358 if (range0_sizek < state->range_sizek) { 359 /* Need to handle left over range: */ 360 range_sizek = state->range_sizek - range0_sizek; 361 362 Dprintk("range: %016lx - %016lx\n", 363 range_basek<<10, 364 (range_basek + range_sizek)<<10); 365 366 state->reg = range_to_mtrr(state->reg, range_basek, 367 range_sizek, MTRR_TYPE_WRBACK); 368 } 369 370 if (hole_sizek) { 371 hole_basek = range_basek - hole_sizek - second_sizek; 372 Dprintk("hole: %016lx - %016lx\n", 373 hole_basek<<10, 374 (hole_basek + hole_sizek)<<10); 375 state->reg = range_to_mtrr(state->reg, hole_basek, 376 hole_sizek, MTRR_TYPE_UNCACHABLE); 377 } 378 379 return second_sizek; 380 } 381 382 static void __init 383 set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn, 384 unsigned long size_pfn) 385 { 386 unsigned long basek, sizek; 387 unsigned long second_sizek = 0; 388 389 if (state->reg >= num_var_ranges) 390 return; 391 392 basek = base_pfn << (PAGE_SHIFT - 10); 393 sizek = size_pfn << (PAGE_SHIFT - 10); 394 395 /* See if I can merge with the last range: */ 396 if ((basek <= 1024) || 397 (state->range_startk + state->range_sizek == basek)) { 398 unsigned long endk = basek + sizek; 399 state->range_sizek = endk - state->range_startk; 400 return; 401 } 402 /* Write the range mtrrs: */ 403 if (state->range_sizek != 0) 404 second_sizek = range_to_mtrr_with_hole(state, basek, sizek); 405 406 /* Allocate an msr: */ 407 state->range_startk = basek + second_sizek; 408 state->range_sizek = sizek - second_sizek; 409 } 410 411 /* Minimum size of mtrr block that can take hole: */ 412 static u64 mtrr_chunk_size __initdata = (256ULL<<20); 413 414 static int __init parse_mtrr_chunk_size_opt(char *p) 415 { 416 if (!p) 417 return -EINVAL; 418 mtrr_chunk_size = memparse(p, &p); 419 return 0; 420 } 421 early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt); 422 423 /* Granularity of mtrr of block: */ 424 static u64 mtrr_gran_size __initdata; 425 426 static int __init parse_mtrr_gran_size_opt(char *p) 427 { 428 if (!p) 429 return -EINVAL; 430 mtrr_gran_size = memparse(p, &p); 431 return 0; 432 } 433 early_param("mtrr_gran_size", parse_mtrr_gran_size_opt); 434 435 static unsigned long nr_mtrr_spare_reg __initdata = 436 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT; 437 438 static int __init parse_mtrr_spare_reg(char *arg) 439 { 440 if (arg) 441 nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0); 442 return 0; 443 } 444 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg); 445 446 static int __init 447 x86_setup_var_mtrrs(struct range *range, int nr_range, 448 u64 chunk_size, u64 gran_size) 449 { 450 struct var_mtrr_state var_state; 451 int num_reg; 452 int i; 453 454 var_state.range_startk = 0; 455 var_state.range_sizek = 0; 456 var_state.reg = 0; 457 var_state.chunk_sizek = chunk_size >> 10; 458 var_state.gran_sizek = gran_size >> 10; 459 460 memset(range_state, 0, sizeof(range_state)); 461 462 /* Write the range: */ 463 for (i = 0; i < nr_range; i++) { 464 set_var_mtrr_range(&var_state, range[i].start, 465 range[i].end - range[i].start); 466 } 467 468 /* Write the last range: */ 469 if (var_state.range_sizek != 0) 470 range_to_mtrr_with_hole(&var_state, 0, 0); 471 472 num_reg = var_state.reg; 473 /* Clear out the extra MTRR's: */ 474 while (var_state.reg < num_var_ranges) { 475 save_var_mtrr(var_state.reg, 0, 0, 0); 476 var_state.reg++; 477 } 478 479 return num_reg; 480 } 481 482 struct mtrr_cleanup_result { 483 unsigned long gran_sizek; 484 unsigned long chunk_sizek; 485 unsigned long lose_cover_sizek; 486 unsigned int num_reg; 487 int bad; 488 }; 489 490 /* 491 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G 492 * chunk size: gran_size, ..., 2G 493 * so we need (1+16)*8 494 */ 495 #define NUM_RESULT 136 496 #define PSHIFT (PAGE_SHIFT - 10) 497 498 static struct mtrr_cleanup_result __initdata result[NUM_RESULT]; 499 static unsigned long __initdata min_loss_pfn[RANGE_NUM]; 500 501 static void __init print_out_mtrr_range_state(void) 502 { 503 char start_factor = 'K', size_factor = 'K'; 504 unsigned long start_base, size_base; 505 mtrr_type type; 506 int i; 507 508 for (i = 0; i < num_var_ranges; i++) { 509 510 size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10); 511 if (!size_base) 512 continue; 513 514 size_base = to_size_factor(size_base, &size_factor); 515 start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10); 516 start_base = to_size_factor(start_base, &start_factor); 517 type = range_state[i].type; 518 519 Dprintk("reg %d, base: %ld%cB, range: %ld%cB, type %s\n", 520 i, start_base, start_factor, 521 size_base, size_factor, 522 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 523 ((type == MTRR_TYPE_WRPROT) ? "WP" : 524 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")) 525 ); 526 } 527 } 528 529 static int __init mtrr_need_cleanup(void) 530 { 531 int i; 532 mtrr_type type; 533 unsigned long size; 534 /* Extra one for all 0: */ 535 int num[MTRR_NUM_TYPES + 1]; 536 537 /* Check entries number: */ 538 memset(num, 0, sizeof(num)); 539 for (i = 0; i < num_var_ranges; i++) { 540 type = range_state[i].type; 541 size = range_state[i].size_pfn; 542 if (type >= MTRR_NUM_TYPES) 543 continue; 544 if (!size) 545 type = MTRR_NUM_TYPES; 546 num[type]++; 547 } 548 549 /* Check if we got UC entries: */ 550 if (!num[MTRR_TYPE_UNCACHABLE]) 551 return 0; 552 553 /* Check if we only had WB and UC */ 554 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 555 num_var_ranges - num[MTRR_NUM_TYPES]) 556 return 0; 557 558 return 1; 559 } 560 561 static unsigned long __initdata range_sums; 562 563 static void __init 564 mtrr_calc_range_state(u64 chunk_size, u64 gran_size, 565 unsigned long x_remove_base, 566 unsigned long x_remove_size, int i) 567 { 568 /* 569 * range_new should really be an automatic variable, but 570 * putting 4096 bytes on the stack is frowned upon, to put it 571 * mildly. It is safe to make it a static __initdata variable, 572 * since mtrr_calc_range_state is only called during init and 573 * there's no way it will call itself recursively. 574 */ 575 static struct range range_new[RANGE_NUM] __initdata; 576 unsigned long range_sums_new; 577 int nr_range_new; 578 int num_reg; 579 580 /* Convert ranges to var ranges state: */ 581 num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 582 583 /* We got new setting in range_state, check it: */ 584 memset(range_new, 0, sizeof(range_new)); 585 nr_range_new = x86_get_mtrr_mem_range(range_new, 0, 586 x_remove_base, x_remove_size); 587 range_sums_new = sum_ranges(range_new, nr_range_new); 588 589 result[i].chunk_sizek = chunk_size >> 10; 590 result[i].gran_sizek = gran_size >> 10; 591 result[i].num_reg = num_reg; 592 593 if (range_sums < range_sums_new) { 594 result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT; 595 result[i].bad = 1; 596 } else { 597 result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT; 598 } 599 600 /* Double check it: */ 601 if (!result[i].bad && !result[i].lose_cover_sizek) { 602 if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range))) 603 result[i].bad = 1; 604 } 605 606 if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg])) 607 min_loss_pfn[num_reg] = range_sums - range_sums_new; 608 } 609 610 static void __init mtrr_print_out_one_result(int i) 611 { 612 unsigned long gran_base, chunk_base, lose_base; 613 char gran_factor, chunk_factor, lose_factor; 614 615 gran_base = to_size_factor(result[i].gran_sizek, &gran_factor); 616 chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor); 617 lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor); 618 619 pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t", 620 result[i].bad ? "*BAD*" : " ", 621 gran_base, gran_factor, chunk_base, chunk_factor); 622 pr_cont("num_reg: %d \tlose cover RAM: %s%ld%c\n", 623 result[i].num_reg, result[i].bad ? "-" : "", 624 lose_base, lose_factor); 625 } 626 627 static int __init mtrr_search_optimal_index(void) 628 { 629 int num_reg_good; 630 int index_good; 631 int i; 632 633 if (nr_mtrr_spare_reg >= num_var_ranges) 634 nr_mtrr_spare_reg = num_var_ranges - 1; 635 636 num_reg_good = -1; 637 for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) { 638 if (!min_loss_pfn[i]) 639 num_reg_good = i; 640 } 641 642 index_good = -1; 643 if (num_reg_good != -1) { 644 for (i = 0; i < NUM_RESULT; i++) { 645 if (!result[i].bad && 646 result[i].num_reg == num_reg_good && 647 !result[i].lose_cover_sizek) { 648 index_good = i; 649 break; 650 } 651 } 652 } 653 654 return index_good; 655 } 656 657 int __init mtrr_cleanup(void) 658 { 659 unsigned long x_remove_base, x_remove_size; 660 unsigned long base, size, def, dummy; 661 u64 chunk_size, gran_size; 662 mtrr_type type; 663 int index_good; 664 int i; 665 666 if (!mtrr_enabled()) 667 return 0; 668 669 if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1) 670 return 0; 671 672 rdmsr(MSR_MTRRdefType, def, dummy); 673 def &= 0xff; 674 if (def != MTRR_TYPE_UNCACHABLE) 675 return 0; 676 677 /* Get it and store it aside: */ 678 memset(range_state, 0, sizeof(range_state)); 679 for (i = 0; i < num_var_ranges; i++) { 680 mtrr_if->get(i, &base, &size, &type); 681 range_state[i].base_pfn = base; 682 range_state[i].size_pfn = size; 683 range_state[i].type = type; 684 } 685 686 /* Check if we need handle it and can handle it: */ 687 if (!mtrr_need_cleanup()) 688 return 0; 689 690 /* Print original var MTRRs at first, for debugging: */ 691 Dprintk("original variable MTRRs\n"); 692 print_out_mtrr_range_state(); 693 694 memset(range, 0, sizeof(range)); 695 x_remove_size = 0; 696 x_remove_base = 1 << (32 - PAGE_SHIFT); 697 if (mtrr_tom2) 698 x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base; 699 700 /* 701 * [0, 1M) should always be covered by var mtrr with WB 702 * and fixed mtrrs should take effect before var mtrr for it: 703 */ 704 nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0, 705 1ULL<<(20 - PAGE_SHIFT)); 706 /* add from var mtrr at last */ 707 nr_range = x86_get_mtrr_mem_range(range, nr_range, 708 x_remove_base, x_remove_size); 709 710 range_sums = sum_ranges(range, nr_range); 711 pr_info("total RAM covered: %ldM\n", 712 range_sums >> (20 - PAGE_SHIFT)); 713 714 if (mtrr_chunk_size && mtrr_gran_size) { 715 i = 0; 716 mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size, 717 x_remove_base, x_remove_size, i); 718 719 mtrr_print_out_one_result(i); 720 721 if (!result[i].bad) { 722 set_var_mtrr_all(); 723 Dprintk("New variable MTRRs\n"); 724 print_out_mtrr_range_state(); 725 return 1; 726 } 727 pr_info("invalid mtrr_gran_size or mtrr_chunk_size, will find optimal one\n"); 728 } 729 730 i = 0; 731 memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn)); 732 memset(result, 0, sizeof(result)); 733 for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) { 734 735 for (chunk_size = gran_size; chunk_size < (1ULL<<32); 736 chunk_size <<= 1) { 737 738 if (i >= NUM_RESULT) 739 continue; 740 741 mtrr_calc_range_state(chunk_size, gran_size, 742 x_remove_base, x_remove_size, i); 743 if (mtrr_debug) { 744 mtrr_print_out_one_result(i); 745 pr_info("\n"); 746 } 747 748 i++; 749 } 750 } 751 752 /* Try to find the optimal index: */ 753 index_good = mtrr_search_optimal_index(); 754 755 if (index_good != -1) { 756 pr_info("Found optimal setting for mtrr clean up\n"); 757 i = index_good; 758 mtrr_print_out_one_result(i); 759 760 /* Convert ranges to var ranges state: */ 761 chunk_size = result[i].chunk_sizek; 762 chunk_size <<= 10; 763 gran_size = result[i].gran_sizek; 764 gran_size <<= 10; 765 x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 766 set_var_mtrr_all(); 767 Dprintk("New variable MTRRs\n"); 768 print_out_mtrr_range_state(); 769 return 1; 770 } else { 771 /* print out all */ 772 for (i = 0; i < NUM_RESULT; i++) 773 mtrr_print_out_one_result(i); 774 } 775 776 pr_info("mtrr_cleanup: can not find optimal value\n"); 777 pr_info("please specify mtrr_gran_size/mtrr_chunk_size\n"); 778 779 return 0; 780 } 781 #else 782 int __init mtrr_cleanup(void) 783 { 784 return 0; 785 } 786 #endif 787 788 static int disable_mtrr_trim; 789 790 static int __init disable_mtrr_trim_setup(char *str) 791 { 792 disable_mtrr_trim = 1; 793 return 0; 794 } 795 early_param("disable_mtrr_trim", disable_mtrr_trim_setup); 796 797 /* 798 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB 799 * for memory >4GB. Check for that here. 800 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't 801 * apply to are wrong, but so far we don't know of any such case in the wild. 802 */ 803 #define Tom2Enabled (1U << 21) 804 #define Tom2ForceMemTypeWB (1U << 22) 805 806 int __init amd_special_default_mtrr(void) 807 { 808 u32 l, h; 809 810 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 811 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 812 return 0; 813 if (boot_cpu_data.x86 < 0xf) 814 return 0; 815 /* In case some hypervisor doesn't pass SYSCFG through: */ 816 if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0) 817 return 0; 818 /* 819 * Memory between 4GB and top of mem is forced WB by this magic bit. 820 * Reserved before K8RevF, but should be zero there. 821 */ 822 if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == 823 (Tom2Enabled | Tom2ForceMemTypeWB)) 824 return 1; 825 return 0; 826 } 827 828 static u64 __init 829 real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) 830 { 831 u64 trim_start, trim_size; 832 833 trim_start = start_pfn; 834 trim_start <<= PAGE_SHIFT; 835 836 trim_size = limit_pfn; 837 trim_size <<= PAGE_SHIFT; 838 trim_size -= trim_start; 839 840 return e820__range_update(trim_start, trim_size, E820_TYPE_RAM, E820_TYPE_RESERVED); 841 } 842 843 /** 844 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs 845 * @end_pfn: ending page frame number 846 * 847 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain 848 * memory configurations. This routine checks that the highest MTRR matches 849 * the end of memory, to make sure the MTRRs having a write back type cover 850 * all of the memory the kernel is intending to use. If not, it'll trim any 851 * memory off the end by adjusting end_pfn, removing it from the kernel's 852 * allocation pools, warning the user with an obnoxious message. 853 */ 854 int __init mtrr_trim_uncached_memory(unsigned long end_pfn) 855 { 856 unsigned long i, base, size, highest_pfn = 0, def, dummy; 857 mtrr_type type; 858 u64 total_trim_size; 859 /* extra one for all 0 */ 860 int num[MTRR_NUM_TYPES + 1]; 861 862 if (!mtrr_enabled()) 863 return 0; 864 865 /* 866 * Make sure we only trim uncachable memory on machines that 867 * support the Intel MTRR architecture: 868 */ 869 if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim) 870 return 0; 871 872 rdmsr(MSR_MTRRdefType, def, dummy); 873 def &= MTRR_DEF_TYPE_TYPE; 874 if (def != MTRR_TYPE_UNCACHABLE) 875 return 0; 876 877 /* Get it and store it aside: */ 878 memset(range_state, 0, sizeof(range_state)); 879 for (i = 0; i < num_var_ranges; i++) { 880 mtrr_if->get(i, &base, &size, &type); 881 range_state[i].base_pfn = base; 882 range_state[i].size_pfn = size; 883 range_state[i].type = type; 884 } 885 886 /* Find highest cached pfn: */ 887 for (i = 0; i < num_var_ranges; i++) { 888 type = range_state[i].type; 889 if (type != MTRR_TYPE_WRBACK) 890 continue; 891 base = range_state[i].base_pfn; 892 size = range_state[i].size_pfn; 893 if (highest_pfn < base + size) 894 highest_pfn = base + size; 895 } 896 897 /* kvm/qemu doesn't have mtrr set right, don't trim them all: */ 898 if (!highest_pfn) { 899 pr_info("CPU MTRRs all blank - virtualized system.\n"); 900 return 0; 901 } 902 903 /* Check entries number: */ 904 memset(num, 0, sizeof(num)); 905 for (i = 0; i < num_var_ranges; i++) { 906 type = range_state[i].type; 907 if (type >= MTRR_NUM_TYPES) 908 continue; 909 size = range_state[i].size_pfn; 910 if (!size) 911 type = MTRR_NUM_TYPES; 912 num[type]++; 913 } 914 915 /* No entry for WB? */ 916 if (!num[MTRR_TYPE_WRBACK]) 917 return 0; 918 919 /* Check if we only had WB and UC: */ 920 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 921 num_var_ranges - num[MTRR_NUM_TYPES]) 922 return 0; 923 924 memset(range, 0, sizeof(range)); 925 nr_range = 0; 926 if (mtrr_tom2) { 927 range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT)); 928 range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT; 929 if (highest_pfn < range[nr_range].end) 930 highest_pfn = range[nr_range].end; 931 nr_range++; 932 } 933 nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0); 934 935 /* Check the head: */ 936 total_trim_size = 0; 937 if (range[0].start) 938 total_trim_size += real_trim_memory(0, range[0].start); 939 940 /* Check the holes: */ 941 for (i = 0; i < nr_range - 1; i++) { 942 if (range[i].end < range[i+1].start) 943 total_trim_size += real_trim_memory(range[i].end, 944 range[i+1].start); 945 } 946 947 /* Check the top: */ 948 i = nr_range - 1; 949 if (range[i].end < end_pfn) 950 total_trim_size += real_trim_memory(range[i].end, 951 end_pfn); 952 953 if (total_trim_size) { 954 pr_warn("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", 955 total_trim_size >> 20); 956 957 if (!changed_by_mtrr_cleanup) 958 WARN_ON(1); 959 960 pr_info("update e820 for mtrr\n"); 961 e820__update_table_print(); 962 963 return 1; 964 } 965 966 return 0; 967 } 968