1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Firmware replacement code. 4 * 5 * Work around broken BIOSes that don't set an aperture, only set the 6 * aperture in the AGP bridge, or set too small aperture. 7 * 8 * If all fails map the aperture over some low memory. This is cheaper than 9 * doing bounce buffering. The memory is lost. This is done at early boot 10 * because only the bootmem allocator can allocate 32+MB. 11 * 12 * Copyright 2002 Andi Kleen, SuSE Labs. 13 */ 14 #define pr_fmt(fmt) "AGP: " fmt 15 16 #include <linux/kernel.h> 17 #include <linux/kcore.h> 18 #include <linux/types.h> 19 #include <linux/init.h> 20 #include <linux/memblock.h> 21 #include <linux/mmzone.h> 22 #include <linux/pci_ids.h> 23 #include <linux/pci.h> 24 #include <linux/bitops.h> 25 #include <linux/suspend.h> 26 #include <asm/e820/api.h> 27 #include <asm/io.h> 28 #include <asm/iommu.h> 29 #include <asm/gart.h> 30 #include <asm/pci-direct.h> 31 #include <asm/dma.h> 32 #include <asm/amd_nb.h> 33 #include <asm/x86_init.h> 34 #include <linux/crash_dump.h> 35 36 /* 37 * Using 512M as goal, in case kexec will load kernel_big 38 * that will do the on-position decompress, and could overlap with 39 * with the gart aperture that is used. 40 * Sequence: 41 * kernel_small 42 * ==> kexec (with kdump trigger path or gart still enabled) 43 * ==> kernel_small (gart area become e820_reserved) 44 * ==> kexec (with kdump trigger path or gart still enabled) 45 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 46 * So don't use 512M below as gart iommu, leave the space for kernel 47 * code for safe. 48 */ 49 #define GART_MIN_ADDR (512ULL << 20) 50 #define GART_MAX_ADDR (1ULL << 32) 51 52 int gart_iommu_aperture; 53 int gart_iommu_aperture_disabled __initdata; 54 int gart_iommu_aperture_allowed __initdata; 55 56 int fallback_aper_order __initdata = 1; /* 64MB */ 57 int fallback_aper_force __initdata; 58 59 int fix_aperture __initdata = 1; 60 61 #if defined(CONFIG_PROC_VMCORE) || defined(CONFIG_PROC_KCORE) 62 /* 63 * If the first kernel maps the aperture over e820 RAM, the kdump kernel will 64 * use the same range because it will remain configured in the northbridge. 65 * Trying to dump this area via /proc/vmcore may crash the machine, so exclude 66 * it from vmcore. 67 */ 68 static unsigned long aperture_pfn_start, aperture_page_count; 69 70 static int gart_mem_pfn_is_ram(unsigned long pfn) 71 { 72 return likely((pfn < aperture_pfn_start) || 73 (pfn >= aperture_pfn_start + aperture_page_count)); 74 } 75 76 static void __init exclude_from_core(u64 aper_base, u32 aper_order) 77 { 78 aperture_pfn_start = aper_base >> PAGE_SHIFT; 79 aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; 80 #ifdef CONFIG_PROC_VMCORE 81 WARN_ON(register_oldmem_pfn_is_ram(&gart_mem_pfn_is_ram)); 82 #endif 83 #ifdef CONFIG_PROC_KCORE 84 WARN_ON(register_mem_pfn_is_ram(&gart_mem_pfn_is_ram)); 85 #endif 86 } 87 #else 88 static void exclude_from_core(u64 aper_base, u32 aper_order) 89 { 90 } 91 #endif 92 93 /* This code runs before the PCI subsystem is initialized, so just 94 access the northbridge directly. */ 95 96 static u32 __init allocate_aperture(void) 97 { 98 u32 aper_size; 99 unsigned long addr; 100 101 /* aper_size should <= 1G */ 102 if (fallback_aper_order > 5) 103 fallback_aper_order = 5; 104 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 105 106 /* 107 * Aperture has to be naturally aligned. This means a 2GB aperture 108 * won't have much chance of finding a place in the lower 4GB of 109 * memory. Unfortunately we cannot move it up because that would 110 * make the IOMMU useless. 111 */ 112 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 113 aper_size, aper_size); 114 if (!addr) { 115 pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n", 116 addr, addr + aper_size - 1, aper_size >> 10); 117 return 0; 118 } 119 memblock_reserve(addr, aper_size); 120 pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n", 121 addr, addr + aper_size - 1, aper_size >> 10); 122 register_nosave_region(addr >> PAGE_SHIFT, 123 (addr+aper_size) >> PAGE_SHIFT); 124 125 return (u32)addr; 126 } 127 128 129 /* Find a PCI capability */ 130 static u32 __init find_cap(int bus, int slot, int func, int cap) 131 { 132 int bytes; 133 u8 pos; 134 135 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 136 PCI_STATUS_CAP_LIST)) 137 return 0; 138 139 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 140 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 141 u8 id; 142 143 pos &= ~3; 144 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 145 if (id == 0xff) 146 break; 147 if (id == cap) 148 return pos; 149 pos = read_pci_config_byte(bus, slot, func, 150 pos+PCI_CAP_LIST_NEXT); 151 } 152 return 0; 153 } 154 155 /* Read a standard AGPv3 bridge header */ 156 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 157 { 158 u32 apsize; 159 u32 apsizereg; 160 int nbits; 161 u32 aper_low, aper_hi; 162 u64 aper; 163 u32 old_order; 164 165 pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n", bus, slot, func); 166 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 167 if (apsizereg == 0xffffffff) { 168 pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n", 169 bus, slot, func); 170 return 0; 171 } 172 173 /* old_order could be the value from NB gart setting */ 174 old_order = *order; 175 176 apsize = apsizereg & 0xfff; 177 /* Some BIOS use weird encodings not in the AGPv3 table. */ 178 if (apsize & 0xff) 179 apsize |= 0xf00; 180 nbits = hweight16(apsize); 181 *order = 7 - nbits; 182 if ((int)*order < 0) /* < 32MB */ 183 *order = 0; 184 185 aper_low = read_pci_config(bus, slot, func, 0x10); 186 aper_hi = read_pci_config(bus, slot, func, 0x14); 187 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 188 189 /* 190 * On some sick chips, APSIZE is 0. It means it wants 4G 191 * so let double check that order, and lets trust AMD NB settings: 192 */ 193 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n", 194 bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, 195 32 << old_order); 196 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 197 pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n", 198 bus, slot, func, 32 << *order, apsizereg); 199 *order = old_order; 200 } 201 202 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n", 203 bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, 204 32 << *order, apsizereg); 205 206 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 207 return 0; 208 return (u32)aper; 209 } 210 211 /* 212 * Look for an AGP bridge. Windows only expects the aperture in the 213 * AGP bridge and some BIOS forget to initialize the Northbridge too. 214 * Work around this here. 215 * 216 * Do an PCI bus scan by hand because we're running before the PCI 217 * subsystem. 218 * 219 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 220 * generically. It's probably overkill to always scan all slots because 221 * the AGP bridges should be always an own bus on the HT hierarchy, 222 * but do it here for future safety. 223 */ 224 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 225 { 226 int bus, slot, func; 227 228 /* Poor man's PCI discovery */ 229 for (bus = 0; bus < 256; bus++) { 230 for (slot = 0; slot < 32; slot++) { 231 for (func = 0; func < 8; func++) { 232 u32 class, cap; 233 u8 type; 234 class = read_pci_config(bus, slot, func, 235 PCI_CLASS_REVISION); 236 if (class == 0xffffffff) 237 break; 238 239 switch (class >> 16) { 240 case PCI_CLASS_BRIDGE_HOST: 241 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 242 /* AGP bridge? */ 243 cap = find_cap(bus, slot, func, 244 PCI_CAP_ID_AGP); 245 if (!cap) 246 break; 247 *valid_agp = 1; 248 return read_agp(bus, slot, func, cap, 249 order); 250 } 251 252 /* No multi-function device? */ 253 type = read_pci_config_byte(bus, slot, func, 254 PCI_HEADER_TYPE); 255 if (!(type & 0x80)) 256 break; 257 } 258 } 259 } 260 pr_info("No AGP bridge found\n"); 261 262 return 0; 263 } 264 265 static bool gart_fix_e820 __initdata = true; 266 267 static int __init parse_gart_mem(char *p) 268 { 269 return kstrtobool(p, &gart_fix_e820); 270 } 271 early_param("gart_fix_e820", parse_gart_mem); 272 273 /* 274 * With kexec/kdump, if the first kernel doesn't shut down the GART and the 275 * second kernel allocates a different GART region, there might be two 276 * overlapping GART regions present: 277 * 278 * - the first still used by the GART initialized in the first kernel. 279 * - (sub-)set of it used as normal RAM by the second kernel. 280 * 281 * which leads to memory corruptions and a kernel panic eventually. 282 * 283 * This can also happen if the BIOS has forgotten to mark the GART region 284 * as reserved. 285 * 286 * Try to update the e820 map to mark that new region as reserved. 287 */ 288 void __init early_gart_iommu_check(void) 289 { 290 u32 agp_aper_order = 0; 291 int i, fix, slot, valid_agp = 0; 292 u32 ctl; 293 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 294 u64 aper_base = 0, last_aper_base = 0; 295 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 296 297 if (!amd_gart_present()) 298 return; 299 300 if (!early_pci_allowed()) 301 return; 302 303 /* This is mostly duplicate of iommu_hole_init */ 304 search_agp_bridge(&agp_aper_order, &valid_agp); 305 306 fix = 0; 307 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 308 int bus; 309 int dev_base, dev_limit; 310 311 bus = amd_nb_bus_dev_ranges[i].bus; 312 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 313 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 314 315 for (slot = dev_base; slot < dev_limit; slot++) { 316 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 317 continue; 318 319 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 320 aper_enabled = ctl & GARTEN; 321 aper_order = (ctl >> 1) & 7; 322 aper_size = (32 * 1024 * 1024) << aper_order; 323 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 324 aper_base <<= 25; 325 326 if (last_valid) { 327 if ((aper_order != last_aper_order) || 328 (aper_base != last_aper_base) || 329 (aper_enabled != last_aper_enabled)) { 330 fix = 1; 331 break; 332 } 333 } 334 335 last_aper_order = aper_order; 336 last_aper_base = aper_base; 337 last_aper_enabled = aper_enabled; 338 last_valid = 1; 339 } 340 } 341 342 if (!fix && !aper_enabled) 343 return; 344 345 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 346 fix = 1; 347 348 if (gart_fix_e820 && !fix && aper_enabled) { 349 if (e820__mapped_any(aper_base, aper_base + aper_size, 350 E820_TYPE_RAM)) { 351 /* reserve it, so we can reuse it in second kernel */ 352 pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n", 353 aper_base, aper_base + aper_size - 1); 354 e820__range_add(aper_base, aper_size, E820_TYPE_RESERVED); 355 e820__update_table_print(); 356 } 357 } 358 359 if (valid_agp) 360 return; 361 362 /* disable them all at first */ 363 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 364 int bus; 365 int dev_base, dev_limit; 366 367 bus = amd_nb_bus_dev_ranges[i].bus; 368 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 369 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 370 371 for (slot = dev_base; slot < dev_limit; slot++) { 372 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 373 continue; 374 375 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 376 ctl &= ~GARTEN; 377 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 378 } 379 } 380 381 } 382 383 static int __initdata printed_gart_size_msg; 384 385 int __init gart_iommu_hole_init(void) 386 { 387 u32 agp_aper_base = 0, agp_aper_order = 0; 388 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 389 u64 aper_base, last_aper_base = 0; 390 int fix, slot, valid_agp = 0; 391 int i, node; 392 393 if (!amd_gart_present()) 394 return -ENODEV; 395 396 if (gart_iommu_aperture_disabled || !fix_aperture || 397 !early_pci_allowed()) 398 return -ENODEV; 399 400 pr_info("Checking aperture...\n"); 401 402 if (!fallback_aper_force) 403 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 404 405 fix = 0; 406 node = 0; 407 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 408 int bus; 409 int dev_base, dev_limit; 410 u32 ctl; 411 412 bus = amd_nb_bus_dev_ranges[i].bus; 413 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 414 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 415 416 for (slot = dev_base; slot < dev_limit; slot++) { 417 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 418 continue; 419 420 iommu_detected = 1; 421 gart_iommu_aperture = 1; 422 x86_init.iommu.iommu_init = gart_iommu_init; 423 424 ctl = read_pci_config(bus, slot, 3, 425 AMD64_GARTAPERTURECTL); 426 427 /* 428 * Before we do anything else disable the GART. It may 429 * still be enabled if we boot into a crash-kernel here. 430 * Reconfiguring the GART while it is enabled could have 431 * unknown side-effects. 432 */ 433 ctl &= ~GARTEN; 434 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 435 436 aper_order = (ctl >> 1) & 7; 437 aper_size = (32 * 1024 * 1024) << aper_order; 438 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 439 aper_base <<= 25; 440 441 pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n", 442 node, aper_base, aper_base + aper_size - 1, 443 aper_size >> 20); 444 node++; 445 446 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 447 if (valid_agp && agp_aper_base && 448 agp_aper_base == aper_base && 449 agp_aper_order == aper_order) { 450 /* the same between two setting from NB and agp */ 451 if (!no_iommu && 452 max_pfn > MAX_DMA32_PFN && 453 !printed_gart_size_msg) { 454 pr_err("you are using iommu with agp, but GART size is less than 64MB\n"); 455 pr_err("please increase GART size in your BIOS setup\n"); 456 pr_err("if BIOS doesn't have that option, contact your HW vendor!\n"); 457 printed_gart_size_msg = 1; 458 } 459 } else { 460 fix = 1; 461 goto out; 462 } 463 } 464 465 if ((last_aper_order && aper_order != last_aper_order) || 466 (last_aper_base && aper_base != last_aper_base)) { 467 fix = 1; 468 goto out; 469 } 470 last_aper_order = aper_order; 471 last_aper_base = aper_base; 472 } 473 } 474 475 out: 476 if (!fix && !fallback_aper_force) { 477 if (last_aper_base) { 478 /* 479 * If this is the kdump kernel, the first kernel 480 * may have allocated the range over its e820 RAM 481 * and fixed up the northbridge 482 */ 483 exclude_from_core(last_aper_base, last_aper_order); 484 485 return 1; 486 } 487 return 0; 488 } 489 490 if (!fallback_aper_force) { 491 aper_alloc = agp_aper_base; 492 aper_order = agp_aper_order; 493 } 494 495 if (aper_alloc) { 496 /* Got the aperture from the AGP bridge */ 497 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 498 force_iommu || 499 valid_agp || 500 fallback_aper_force) { 501 pr_info("Your BIOS doesn't leave an aperture memory hole\n"); 502 pr_info("Please enable the IOMMU option in the BIOS setup\n"); 503 pr_info("This costs you %dMB of RAM\n", 504 32 << fallback_aper_order); 505 506 aper_order = fallback_aper_order; 507 aper_alloc = allocate_aperture(); 508 if (!aper_alloc) { 509 /* 510 * Could disable AGP and IOMMU here, but it's 511 * probably not worth it. But the later users 512 * cannot deal with bad apertures and turning 513 * on the aperture over memory causes very 514 * strange problems, so it's better to panic 515 * early. 516 */ 517 panic("Not enough memory for aperture"); 518 } 519 } else { 520 return 0; 521 } 522 523 /* 524 * If this is the kdump kernel _and_ the first kernel did not 525 * configure the aperture in the northbridge, this range may 526 * overlap with the first kernel's memory. We can't access the 527 * range through vmcore even though it should be part of the dump. 528 */ 529 exclude_from_core(aper_alloc, aper_order); 530 531 /* Fix up the north bridges */ 532 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 533 int bus, dev_base, dev_limit; 534 535 /* 536 * Don't enable translation yet but enable GART IO and CPU 537 * accesses and set DISTLBWALKPRB since GART table memory is UC. 538 */ 539 u32 ctl = aper_order << 1; 540 541 bus = amd_nb_bus_dev_ranges[i].bus; 542 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 543 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 544 for (slot = dev_base; slot < dev_limit; slot++) { 545 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 546 continue; 547 548 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 549 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 550 } 551 } 552 553 set_up_gart_resume(aper_order, aper_alloc); 554 555 return 1; 556 } 557