1 // SPDX-License-Identifier: LGPL-2.0+ 2 /* 3 * MTRR (Memory Type Range Register) cleanup 4 * 5 * Copyright (C) 2009 Yinghai Lu 6 */ 7 #include <linux/init.h> 8 #include <linux/pci.h> 9 #include <linux/smp.h> 10 #include <linux/cpu.h> 11 #include <linux/mutex.h> 12 #include <linux/uaccess.h> 13 #include <linux/kvm_para.h> 14 #include <linux/range.h> 15 16 #include <asm/processor.h> 17 #include <asm/e820/api.h> 18 #include <asm/mtrr.h> 19 #include <asm/msr.h> 20 21 #include "mtrr.h" 22 23 struct var_mtrr_range_state { 24 unsigned long base_pfn; 25 unsigned long size_pfn; 26 mtrr_type type; 27 }; 28 29 struct var_mtrr_state { 30 unsigned long range_startk; 31 unsigned long range_sizek; 32 unsigned long chunk_sizek; 33 unsigned long gran_sizek; 34 unsigned int reg; 35 }; 36 37 /* Should be related to MTRR_VAR_RANGES nums */ 38 #define RANGE_NUM 256 39 40 static struct range __initdata range[RANGE_NUM]; 41 static int __initdata nr_range; 42 43 static struct var_mtrr_range_state __initdata range_state[RANGE_NUM]; 44 45 #define BIOS_BUG_MSG \ 46 "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n" 47 48 static int __init 49 x86_get_mtrr_mem_range(struct range *range, int nr_range, 50 unsigned long extra_remove_base, 51 unsigned long extra_remove_size) 52 { 53 unsigned long base, size; 54 mtrr_type type; 55 int i; 56 57 for (i = 0; i < num_var_ranges; i++) { 58 type = range_state[i].type; 59 if (type != MTRR_TYPE_WRBACK) 60 continue; 61 base = range_state[i].base_pfn; 62 size = range_state[i].size_pfn; 63 nr_range = add_range_with_merge(range, RANGE_NUM, nr_range, 64 base, base + size); 65 } 66 67 Dprintk("After WB checking\n"); 68 for (i = 0; i < nr_range; i++) 69 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 70 range[i].start, range[i].end); 71 72 /* Take out UC ranges: */ 73 for (i = 0; i < num_var_ranges; i++) { 74 type = range_state[i].type; 75 if (type != MTRR_TYPE_UNCACHABLE && 76 type != MTRR_TYPE_WRPROT) 77 continue; 78 size = range_state[i].size_pfn; 79 if (!size) 80 continue; 81 base = range_state[i].base_pfn; 82 if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed && 83 (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) && 84 (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) { 85 /* Var MTRR contains UC entry below 1M? Skip it: */ 86 pr_warn(BIOS_BUG_MSG, i); 87 if (base + size <= (1<<(20-PAGE_SHIFT))) 88 continue; 89 size -= (1<<(20-PAGE_SHIFT)) - base; 90 base = 1<<(20-PAGE_SHIFT); 91 } 92 subtract_range(range, RANGE_NUM, base, base + size); 93 } 94 if (extra_remove_size) 95 subtract_range(range, RANGE_NUM, extra_remove_base, 96 extra_remove_base + extra_remove_size); 97 98 Dprintk("After UC checking\n"); 99 for (i = 0; i < RANGE_NUM; i++) { 100 if (!range[i].end) 101 continue; 102 103 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 104 range[i].start, range[i].end); 105 } 106 107 /* sort the ranges */ 108 nr_range = clean_sort_range(range, RANGE_NUM); 109 110 Dprintk("After sorting\n"); 111 for (i = 0; i < nr_range; i++) 112 Dprintk("MTRR MAP PFN: %016llx - %016llx\n", 113 range[i].start, range[i].end); 114 115 return nr_range; 116 } 117 118 #ifdef CONFIG_MTRR_SANITIZER 119 120 static unsigned long __init sum_ranges(struct range *range, int nr_range) 121 { 122 unsigned long sum = 0; 123 int i; 124 125 for (i = 0; i < nr_range; i++) 126 sum += range[i].end - range[i].start; 127 128 return sum; 129 } 130 131 static int enable_mtrr_cleanup __initdata = 132 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT; 133 134 static int __init disable_mtrr_cleanup_setup(char *str) 135 { 136 enable_mtrr_cleanup = 0; 137 return 0; 138 } 139 early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup); 140 141 static int __init enable_mtrr_cleanup_setup(char *str) 142 { 143 enable_mtrr_cleanup = 1; 144 return 0; 145 } 146 early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup); 147 148 static void __init 149 set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 150 unsigned char type) 151 { 152 u32 base_lo, base_hi, mask_lo, mask_hi; 153 u64 base, mask; 154 155 if (!sizek) { 156 fill_mtrr_var_range(reg, 0, 0, 0, 0); 157 return; 158 } 159 160 mask = (1ULL << boot_cpu_data.x86_phys_bits) - 1; 161 mask &= ~((((u64)sizek) << 10) - 1); 162 163 base = ((u64)basek) << 10; 164 165 base |= type; 166 mask |= 0x800; 167 168 base_lo = base & ((1ULL<<32) - 1); 169 base_hi = base >> 32; 170 171 mask_lo = mask & ((1ULL<<32) - 1); 172 mask_hi = mask >> 32; 173 174 fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi); 175 } 176 177 static void __init 178 save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 179 unsigned char type) 180 { 181 range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10); 182 range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10); 183 range_state[reg].type = type; 184 } 185 186 static void __init set_var_mtrr_all(void) 187 { 188 unsigned long basek, sizek; 189 unsigned char type; 190 unsigned int reg; 191 192 for (reg = 0; reg < num_var_ranges; reg++) { 193 basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10); 194 sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10); 195 type = range_state[reg].type; 196 197 set_var_mtrr(reg, basek, sizek, type); 198 } 199 } 200 201 static unsigned long to_size_factor(unsigned long sizek, char *factorp) 202 { 203 unsigned long base = sizek; 204 char factor; 205 206 if (base & ((1<<10) - 1)) { 207 /* Not MB-aligned: */ 208 factor = 'K'; 209 } else if (base & ((1<<20) - 1)) { 210 factor = 'M'; 211 base >>= 10; 212 } else { 213 factor = 'G'; 214 base >>= 20; 215 } 216 217 *factorp = factor; 218 219 return base; 220 } 221 222 static unsigned int __init 223 range_to_mtrr(unsigned int reg, unsigned long range_startk, 224 unsigned long range_sizek, unsigned char type) 225 { 226 if (!range_sizek || (reg >= num_var_ranges)) 227 return reg; 228 229 while (range_sizek) { 230 unsigned long max_align, align; 231 unsigned long sizek; 232 233 /* Compute the maximum size with which we can make a range: */ 234 if (range_startk) 235 max_align = __ffs(range_startk); 236 else 237 max_align = BITS_PER_LONG - 1; 238 239 align = __fls(range_sizek); 240 if (align > max_align) 241 align = max_align; 242 243 sizek = 1UL << align; 244 if (mtrr_debug) { 245 char start_factor = 'K', size_factor = 'K'; 246 unsigned long start_base, size_base; 247 248 start_base = to_size_factor(range_startk, &start_factor); 249 size_base = to_size_factor(sizek, &size_factor); 250 251 Dprintk("Setting variable MTRR %d, " 252 "base: %ld%cB, range: %ld%cB, type %s\n", 253 reg, start_base, start_factor, 254 size_base, size_factor, 255 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 256 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other") 257 ); 258 } 259 save_var_mtrr(reg++, range_startk, sizek, type); 260 range_startk += sizek; 261 range_sizek -= sizek; 262 if (reg >= num_var_ranges) 263 break; 264 } 265 return reg; 266 } 267 268 static unsigned __init 269 range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek, 270 unsigned long sizek) 271 { 272 unsigned long hole_basek, hole_sizek; 273 unsigned long second_sizek; 274 unsigned long range0_basek, range0_sizek; 275 unsigned long range_basek, range_sizek; 276 unsigned long chunk_sizek; 277 unsigned long gran_sizek; 278 279 hole_basek = 0; 280 hole_sizek = 0; 281 second_sizek = 0; 282 chunk_sizek = state->chunk_sizek; 283 gran_sizek = state->gran_sizek; 284 285 /* Align with gran size, prevent small block used up MTRRs: */ 286 range_basek = ALIGN(state->range_startk, gran_sizek); 287 if ((range_basek > basek) && basek) 288 return second_sizek; 289 290 state->range_sizek -= (range_basek - state->range_startk); 291 range_sizek = ALIGN(state->range_sizek, gran_sizek); 292 293 while (range_sizek > state->range_sizek) { 294 range_sizek -= gran_sizek; 295 if (!range_sizek) 296 return 0; 297 } 298 state->range_sizek = range_sizek; 299 300 /* Try to append some small hole: */ 301 range0_basek = state->range_startk; 302 range0_sizek = ALIGN(state->range_sizek, chunk_sizek); 303 304 /* No increase: */ 305 if (range0_sizek == state->range_sizek) { 306 Dprintk("rangeX: %016lx - %016lx\n", 307 range0_basek<<10, 308 (range0_basek + state->range_sizek)<<10); 309 state->reg = range_to_mtrr(state->reg, range0_basek, 310 state->range_sizek, MTRR_TYPE_WRBACK); 311 return 0; 312 } 313 314 /* Only cut back when it is not the last: */ 315 if (sizek) { 316 while (range0_basek + range0_sizek > (basek + sizek)) { 317 if (range0_sizek >= chunk_sizek) 318 range0_sizek -= chunk_sizek; 319 else 320 range0_sizek = 0; 321 322 if (!range0_sizek) 323 break; 324 } 325 } 326 327 second_try: 328 range_basek = range0_basek + range0_sizek; 329 330 /* One hole in the middle: */ 331 if (range_basek > basek && range_basek <= (basek + sizek)) 332 second_sizek = range_basek - basek; 333 334 if (range0_sizek > state->range_sizek) { 335 336 /* One hole in middle or at the end: */ 337 hole_sizek = range0_sizek - state->range_sizek - second_sizek; 338 339 /* Hole size should be less than half of range0 size: */ 340 if (hole_sizek >= (range0_sizek >> 1) && 341 range0_sizek >= chunk_sizek) { 342 range0_sizek -= chunk_sizek; 343 second_sizek = 0; 344 hole_sizek = 0; 345 346 goto second_try; 347 } 348 } 349 350 if (range0_sizek) { 351 Dprintk("range0: %016lx - %016lx\n", 352 range0_basek<<10, 353 (range0_basek + range0_sizek)<<10); 354 state->reg = range_to_mtrr(state->reg, range0_basek, 355 range0_sizek, MTRR_TYPE_WRBACK); 356 } 357 358 if (range0_sizek < state->range_sizek) { 359 /* Need to handle left over range: */ 360 range_sizek = state->range_sizek - range0_sizek; 361 362 Dprintk("range: %016lx - %016lx\n", 363 range_basek<<10, 364 (range_basek + range_sizek)<<10); 365 366 state->reg = range_to_mtrr(state->reg, range_basek, 367 range_sizek, MTRR_TYPE_WRBACK); 368 } 369 370 if (hole_sizek) { 371 hole_basek = range_basek - hole_sizek - second_sizek; 372 Dprintk("hole: %016lx - %016lx\n", 373 hole_basek<<10, 374 (hole_basek + hole_sizek)<<10); 375 state->reg = range_to_mtrr(state->reg, hole_basek, 376 hole_sizek, MTRR_TYPE_UNCACHABLE); 377 } 378 379 return second_sizek; 380 } 381 382 static void __init 383 set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn, 384 unsigned long size_pfn) 385 { 386 unsigned long basek, sizek; 387 unsigned long second_sizek = 0; 388 389 if (state->reg >= num_var_ranges) 390 return; 391 392 basek = base_pfn << (PAGE_SHIFT - 10); 393 sizek = size_pfn << (PAGE_SHIFT - 10); 394 395 /* See if I can merge with the last range: */ 396 if ((basek <= 1024) || 397 (state->range_startk + state->range_sizek == basek)) { 398 unsigned long endk = basek + sizek; 399 state->range_sizek = endk - state->range_startk; 400 return; 401 } 402 /* Write the range mtrrs: */ 403 if (state->range_sizek != 0) 404 second_sizek = range_to_mtrr_with_hole(state, basek, sizek); 405 406 /* Allocate an msr: */ 407 state->range_startk = basek + second_sizek; 408 state->range_sizek = sizek - second_sizek; 409 } 410 411 /* Minimum size of mtrr block that can take hole: */ 412 static u64 mtrr_chunk_size __initdata = (256ULL<<20); 413 414 static int __init parse_mtrr_chunk_size_opt(char *p) 415 { 416 if (!p) 417 return -EINVAL; 418 mtrr_chunk_size = memparse(p, &p); 419 return 0; 420 } 421 early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt); 422 423 /* Granularity of mtrr of block: */ 424 static u64 mtrr_gran_size __initdata; 425 426 static int __init parse_mtrr_gran_size_opt(char *p) 427 { 428 if (!p) 429 return -EINVAL; 430 mtrr_gran_size = memparse(p, &p); 431 return 0; 432 } 433 early_param("mtrr_gran_size", parse_mtrr_gran_size_opt); 434 435 static unsigned long nr_mtrr_spare_reg __initdata = 436 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT; 437 438 static int __init parse_mtrr_spare_reg(char *arg) 439 { 440 if (!arg) 441 return -EINVAL; 442 443 return kstrtoul(arg, 0, &nr_mtrr_spare_reg); 444 } 445 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg); 446 447 static int __init 448 x86_setup_var_mtrrs(struct range *range, int nr_range, 449 u64 chunk_size, u64 gran_size) 450 { 451 struct var_mtrr_state var_state; 452 int num_reg; 453 int i; 454 455 var_state.range_startk = 0; 456 var_state.range_sizek = 0; 457 var_state.reg = 0; 458 var_state.chunk_sizek = chunk_size >> 10; 459 var_state.gran_sizek = gran_size >> 10; 460 461 memset(range_state, 0, sizeof(range_state)); 462 463 /* Write the range: */ 464 for (i = 0; i < nr_range; i++) { 465 set_var_mtrr_range(&var_state, range[i].start, 466 range[i].end - range[i].start); 467 } 468 469 /* Write the last range: */ 470 if (var_state.range_sizek != 0) 471 range_to_mtrr_with_hole(&var_state, 0, 0); 472 473 num_reg = var_state.reg; 474 /* Clear out the extra MTRR's: */ 475 while (var_state.reg < num_var_ranges) { 476 save_var_mtrr(var_state.reg, 0, 0, 0); 477 var_state.reg++; 478 } 479 480 return num_reg; 481 } 482 483 struct mtrr_cleanup_result { 484 unsigned long gran_sizek; 485 unsigned long chunk_sizek; 486 unsigned long lose_cover_sizek; 487 unsigned int num_reg; 488 int bad; 489 }; 490 491 /* 492 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G 493 * chunk size: gran_size, ..., 2G 494 * so we need (1+16)*8 495 */ 496 #define NUM_RESULT 136 497 #define PSHIFT (PAGE_SHIFT - 10) 498 499 static struct mtrr_cleanup_result __initdata result[NUM_RESULT]; 500 static unsigned long __initdata min_loss_pfn[RANGE_NUM]; 501 502 static void __init print_out_mtrr_range_state(void) 503 { 504 char start_factor = 'K', size_factor = 'K'; 505 unsigned long start_base, size_base; 506 mtrr_type type; 507 int i; 508 509 for (i = 0; i < num_var_ranges; i++) { 510 511 size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10); 512 if (!size_base) 513 continue; 514 515 size_base = to_size_factor(size_base, &size_factor); 516 start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10); 517 start_base = to_size_factor(start_base, &start_factor); 518 type = range_state[i].type; 519 520 Dprintk("reg %d, base: %ld%cB, range: %ld%cB, type %s\n", 521 i, start_base, start_factor, 522 size_base, size_factor, 523 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 524 ((type == MTRR_TYPE_WRPROT) ? "WP" : 525 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")) 526 ); 527 } 528 } 529 530 static int __init mtrr_need_cleanup(void) 531 { 532 int i; 533 mtrr_type type; 534 unsigned long size; 535 /* Extra one for all 0: */ 536 int num[MTRR_NUM_TYPES + 1]; 537 538 /* Check entries number: */ 539 memset(num, 0, sizeof(num)); 540 for (i = 0; i < num_var_ranges; i++) { 541 type = range_state[i].type; 542 size = range_state[i].size_pfn; 543 if (type >= MTRR_NUM_TYPES) 544 continue; 545 if (!size) 546 type = MTRR_NUM_TYPES; 547 num[type]++; 548 } 549 550 /* Check if we got UC entries: */ 551 if (!num[MTRR_TYPE_UNCACHABLE]) 552 return 0; 553 554 /* Check if we only had WB and UC */ 555 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 556 num_var_ranges - num[MTRR_NUM_TYPES]) 557 return 0; 558 559 return 1; 560 } 561 562 static unsigned long __initdata range_sums; 563 564 static void __init 565 mtrr_calc_range_state(u64 chunk_size, u64 gran_size, 566 unsigned long x_remove_base, 567 unsigned long x_remove_size, int i) 568 { 569 /* 570 * range_new should really be an automatic variable, but 571 * putting 4096 bytes on the stack is frowned upon, to put it 572 * mildly. It is safe to make it a static __initdata variable, 573 * since mtrr_calc_range_state is only called during init and 574 * there's no way it will call itself recursively. 575 */ 576 static struct range range_new[RANGE_NUM] __initdata; 577 unsigned long range_sums_new; 578 int nr_range_new; 579 int num_reg; 580 581 /* Convert ranges to var ranges state: */ 582 num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 583 584 /* We got new setting in range_state, check it: */ 585 memset(range_new, 0, sizeof(range_new)); 586 nr_range_new = x86_get_mtrr_mem_range(range_new, 0, 587 x_remove_base, x_remove_size); 588 range_sums_new = sum_ranges(range_new, nr_range_new); 589 590 result[i].chunk_sizek = chunk_size >> 10; 591 result[i].gran_sizek = gran_size >> 10; 592 result[i].num_reg = num_reg; 593 594 if (range_sums < range_sums_new) { 595 result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT; 596 result[i].bad = 1; 597 } else { 598 result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT; 599 } 600 601 /* Double check it: */ 602 if (!result[i].bad && !result[i].lose_cover_sizek) { 603 if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range))) 604 result[i].bad = 1; 605 } 606 607 if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg])) 608 min_loss_pfn[num_reg] = range_sums - range_sums_new; 609 } 610 611 static void __init mtrr_print_out_one_result(int i) 612 { 613 unsigned long gran_base, chunk_base, lose_base; 614 char gran_factor, chunk_factor, lose_factor; 615 616 gran_base = to_size_factor(result[i].gran_sizek, &gran_factor); 617 chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor); 618 lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor); 619 620 pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t", 621 result[i].bad ? "*BAD*" : " ", 622 gran_base, gran_factor, chunk_base, chunk_factor); 623 pr_cont("num_reg: %d \tlose cover RAM: %s%ld%c\n", 624 result[i].num_reg, result[i].bad ? "-" : "", 625 lose_base, lose_factor); 626 } 627 628 static int __init mtrr_search_optimal_index(void) 629 { 630 int num_reg_good; 631 int index_good; 632 int i; 633 634 if (nr_mtrr_spare_reg >= num_var_ranges) 635 nr_mtrr_spare_reg = num_var_ranges - 1; 636 637 num_reg_good = -1; 638 for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) { 639 if (!min_loss_pfn[i]) 640 num_reg_good = i; 641 } 642 643 index_good = -1; 644 if (num_reg_good != -1) { 645 for (i = 0; i < NUM_RESULT; i++) { 646 if (!result[i].bad && 647 result[i].num_reg == num_reg_good && 648 !result[i].lose_cover_sizek) { 649 index_good = i; 650 break; 651 } 652 } 653 } 654 655 return index_good; 656 } 657 658 int __init mtrr_cleanup(void) 659 { 660 unsigned long x_remove_base, x_remove_size; 661 unsigned long base, size, def, dummy; 662 u64 chunk_size, gran_size; 663 mtrr_type type; 664 int index_good; 665 int i; 666 667 if (!mtrr_enabled()) 668 return 0; 669 670 if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1) 671 return 0; 672 673 rdmsr(MSR_MTRRdefType, def, dummy); 674 def &= 0xff; 675 if (def != MTRR_TYPE_UNCACHABLE) 676 return 0; 677 678 /* Get it and store it aside: */ 679 memset(range_state, 0, sizeof(range_state)); 680 for (i = 0; i < num_var_ranges; i++) { 681 mtrr_if->get(i, &base, &size, &type); 682 range_state[i].base_pfn = base; 683 range_state[i].size_pfn = size; 684 range_state[i].type = type; 685 } 686 687 /* Check if we need handle it and can handle it: */ 688 if (!mtrr_need_cleanup()) 689 return 0; 690 691 /* Print original var MTRRs at first, for debugging: */ 692 Dprintk("original variable MTRRs\n"); 693 print_out_mtrr_range_state(); 694 695 memset(range, 0, sizeof(range)); 696 x_remove_size = 0; 697 x_remove_base = 1 << (32 - PAGE_SHIFT); 698 if (mtrr_tom2) 699 x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base; 700 701 /* 702 * [0, 1M) should always be covered by var mtrr with WB 703 * and fixed mtrrs should take effect before var mtrr for it: 704 */ 705 nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0, 706 1ULL<<(20 - PAGE_SHIFT)); 707 /* add from var mtrr at last */ 708 nr_range = x86_get_mtrr_mem_range(range, nr_range, 709 x_remove_base, x_remove_size); 710 711 range_sums = sum_ranges(range, nr_range); 712 pr_info("total RAM covered: %ldM\n", 713 range_sums >> (20 - PAGE_SHIFT)); 714 715 if (mtrr_chunk_size && mtrr_gran_size) { 716 i = 0; 717 mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size, 718 x_remove_base, x_remove_size, i); 719 720 mtrr_print_out_one_result(i); 721 722 if (!result[i].bad) { 723 set_var_mtrr_all(); 724 Dprintk("New variable MTRRs\n"); 725 print_out_mtrr_range_state(); 726 return 1; 727 } 728 pr_info("invalid mtrr_gran_size or mtrr_chunk_size, will find optimal one\n"); 729 } 730 731 i = 0; 732 memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn)); 733 memset(result, 0, sizeof(result)); 734 for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) { 735 736 for (chunk_size = gran_size; chunk_size < (1ULL<<32); 737 chunk_size <<= 1) { 738 739 if (i >= NUM_RESULT) 740 continue; 741 742 mtrr_calc_range_state(chunk_size, gran_size, 743 x_remove_base, x_remove_size, i); 744 if (mtrr_debug) { 745 mtrr_print_out_one_result(i); 746 pr_info("\n"); 747 } 748 749 i++; 750 } 751 } 752 753 /* Try to find the optimal index: */ 754 index_good = mtrr_search_optimal_index(); 755 756 if (index_good != -1) { 757 pr_info("Found optimal setting for mtrr clean up\n"); 758 i = index_good; 759 mtrr_print_out_one_result(i); 760 761 /* Convert ranges to var ranges state: */ 762 chunk_size = result[i].chunk_sizek; 763 chunk_size <<= 10; 764 gran_size = result[i].gran_sizek; 765 gran_size <<= 10; 766 x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 767 set_var_mtrr_all(); 768 Dprintk("New variable MTRRs\n"); 769 print_out_mtrr_range_state(); 770 return 1; 771 } else { 772 /* print out all */ 773 for (i = 0; i < NUM_RESULT; i++) 774 mtrr_print_out_one_result(i); 775 } 776 777 pr_info("mtrr_cleanup: can not find optimal value\n"); 778 pr_info("please specify mtrr_gran_size/mtrr_chunk_size\n"); 779 780 return 0; 781 } 782 #else 783 int __init mtrr_cleanup(void) 784 { 785 return 0; 786 } 787 #endif 788 789 static int disable_mtrr_trim; 790 791 static int __init disable_mtrr_trim_setup(char *str) 792 { 793 disable_mtrr_trim = 1; 794 return 0; 795 } 796 early_param("disable_mtrr_trim", disable_mtrr_trim_setup); 797 798 /* 799 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB 800 * for memory >4GB. Check for that here. 801 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't 802 * apply to are wrong, but so far we don't know of any such case in the wild. 803 */ 804 #define Tom2Enabled (1U << 21) 805 #define Tom2ForceMemTypeWB (1U << 22) 806 807 int __init amd_special_default_mtrr(void) 808 { 809 u32 l, h; 810 811 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 812 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 813 return 0; 814 if (boot_cpu_data.x86 < 0xf) 815 return 0; 816 /* In case some hypervisor doesn't pass SYSCFG through: */ 817 if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0) 818 return 0; 819 /* 820 * Memory between 4GB and top of mem is forced WB by this magic bit. 821 * Reserved before K8RevF, but should be zero there. 822 */ 823 if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == 824 (Tom2Enabled | Tom2ForceMemTypeWB)) 825 return 1; 826 return 0; 827 } 828 829 static u64 __init 830 real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) 831 { 832 u64 trim_start, trim_size; 833 834 trim_start = start_pfn; 835 trim_start <<= PAGE_SHIFT; 836 837 trim_size = limit_pfn; 838 trim_size <<= PAGE_SHIFT; 839 trim_size -= trim_start; 840 841 return e820__range_update(trim_start, trim_size, E820_TYPE_RAM, E820_TYPE_RESERVED); 842 } 843 844 /** 845 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs 846 * @end_pfn: ending page frame number 847 * 848 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain 849 * memory configurations. This routine checks that the highest MTRR matches 850 * the end of memory, to make sure the MTRRs having a write back type cover 851 * all of the memory the kernel is intending to use. If not, it'll trim any 852 * memory off the end by adjusting end_pfn, removing it from the kernel's 853 * allocation pools, warning the user with an obnoxious message. 854 */ 855 int __init mtrr_trim_uncached_memory(unsigned long end_pfn) 856 { 857 unsigned long i, base, size, highest_pfn = 0, def, dummy; 858 mtrr_type type; 859 u64 total_trim_size; 860 /* extra one for all 0 */ 861 int num[MTRR_NUM_TYPES + 1]; 862 863 if (!mtrr_enabled()) 864 return 0; 865 866 /* 867 * Make sure we only trim uncachable memory on machines that 868 * support the Intel MTRR architecture: 869 */ 870 if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim) 871 return 0; 872 873 rdmsr(MSR_MTRRdefType, def, dummy); 874 def &= MTRR_DEF_TYPE_TYPE; 875 if (def != MTRR_TYPE_UNCACHABLE) 876 return 0; 877 878 /* Get it and store it aside: */ 879 memset(range_state, 0, sizeof(range_state)); 880 for (i = 0; i < num_var_ranges; i++) { 881 mtrr_if->get(i, &base, &size, &type); 882 range_state[i].base_pfn = base; 883 range_state[i].size_pfn = size; 884 range_state[i].type = type; 885 } 886 887 /* Find highest cached pfn: */ 888 for (i = 0; i < num_var_ranges; i++) { 889 type = range_state[i].type; 890 if (type != MTRR_TYPE_WRBACK) 891 continue; 892 base = range_state[i].base_pfn; 893 size = range_state[i].size_pfn; 894 if (highest_pfn < base + size) 895 highest_pfn = base + size; 896 } 897 898 /* kvm/qemu doesn't have mtrr set right, don't trim them all: */ 899 if (!highest_pfn) { 900 pr_info("CPU MTRRs all blank - virtualized system.\n"); 901 return 0; 902 } 903 904 /* Check entries number: */ 905 memset(num, 0, sizeof(num)); 906 for (i = 0; i < num_var_ranges; i++) { 907 type = range_state[i].type; 908 if (type >= MTRR_NUM_TYPES) 909 continue; 910 size = range_state[i].size_pfn; 911 if (!size) 912 type = MTRR_NUM_TYPES; 913 num[type]++; 914 } 915 916 /* No entry for WB? */ 917 if (!num[MTRR_TYPE_WRBACK]) 918 return 0; 919 920 /* Check if we only had WB and UC: */ 921 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 922 num_var_ranges - num[MTRR_NUM_TYPES]) 923 return 0; 924 925 memset(range, 0, sizeof(range)); 926 nr_range = 0; 927 if (mtrr_tom2) { 928 range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT)); 929 range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT; 930 if (highest_pfn < range[nr_range].end) 931 highest_pfn = range[nr_range].end; 932 nr_range++; 933 } 934 nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0); 935 936 /* Check the head: */ 937 total_trim_size = 0; 938 if (range[0].start) 939 total_trim_size += real_trim_memory(0, range[0].start); 940 941 /* Check the holes: */ 942 for (i = 0; i < nr_range - 1; i++) { 943 if (range[i].end < range[i+1].start) 944 total_trim_size += real_trim_memory(range[i].end, 945 range[i+1].start); 946 } 947 948 /* Check the top: */ 949 i = nr_range - 1; 950 if (range[i].end < end_pfn) 951 total_trim_size += real_trim_memory(range[i].end, 952 end_pfn); 953 954 if (total_trim_size) { 955 pr_warn("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", 956 total_trim_size >> 20); 957 958 if (!changed_by_mtrr_cleanup) 959 WARN_ON(1); 960 961 pr_info("update e820 for mtrr\n"); 962 e820__update_table_print(); 963 964 return 1; 965 } 966 967 return 0; 968 } 969