1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Firmware replacement code. 4 * 5 * Work around broken BIOSes that don't set an aperture, only set the 6 * aperture in the AGP bridge, or set too small aperture. 7 * 8 * If all fails map the aperture over some low memory. This is cheaper than 9 * doing bounce buffering. The memory is lost. This is done at early boot 10 * because only the bootmem allocator can allocate 32+MB. 11 * 12 * Copyright 2002 Andi Kleen, SuSE Labs. 13 */ 14 #define pr_fmt(fmt) "AGP: " fmt 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/init.h> 19 #include <linux/memblock.h> 20 #include <linux/mmzone.h> 21 #include <linux/pci_ids.h> 22 #include <linux/pci.h> 23 #include <linux/bitops.h> 24 #include <linux/suspend.h> 25 #include <asm/e820/api.h> 26 #include <asm/io.h> 27 #include <asm/iommu.h> 28 #include <asm/gart.h> 29 #include <asm/pci-direct.h> 30 #include <asm/dma.h> 31 #include <asm/amd_nb.h> 32 #include <asm/x86_init.h> 33 #include <linux/crash_dump.h> 34 35 /* 36 * Using 512M as goal, in case kexec will load kernel_big 37 * that will do the on-position decompress, and could overlap with 38 * with the gart aperture that is used. 39 * Sequence: 40 * kernel_small 41 * ==> kexec (with kdump trigger path or gart still enabled) 42 * ==> kernel_small (gart area become e820_reserved) 43 * ==> kexec (with kdump trigger path or gart still enabled) 44 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 45 * So don't use 512M below as gart iommu, leave the space for kernel 46 * code for safe. 47 */ 48 #define GART_MIN_ADDR (512ULL << 20) 49 #define GART_MAX_ADDR (1ULL << 32) 50 51 int gart_iommu_aperture; 52 int gart_iommu_aperture_disabled __initdata; 53 int gart_iommu_aperture_allowed __initdata; 54 55 int fallback_aper_order __initdata = 1; /* 64MB */ 56 int fallback_aper_force __initdata; 57 58 int fix_aperture __initdata = 1; 59 60 #ifdef CONFIG_PROC_VMCORE 61 /* 62 * If the first kernel maps the aperture over e820 RAM, the kdump kernel will 63 * use the same range because it will remain configured in the northbridge. 64 * Trying to dump this area via /proc/vmcore may crash the machine, so exclude 65 * it from vmcore. 66 */ 67 static unsigned long aperture_pfn_start, aperture_page_count; 68 69 static int gart_oldmem_pfn_is_ram(unsigned long pfn) 70 { 71 return likely((pfn < aperture_pfn_start) || 72 (pfn >= aperture_pfn_start + aperture_page_count)); 73 } 74 75 static void exclude_from_vmcore(u64 aper_base, u32 aper_order) 76 { 77 aperture_pfn_start = aper_base >> PAGE_SHIFT; 78 aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; 79 WARN_ON(register_oldmem_pfn_is_ram(&gart_oldmem_pfn_is_ram)); 80 } 81 #else 82 static void exclude_from_vmcore(u64 aper_base, u32 aper_order) 83 { 84 } 85 #endif 86 87 /* This code runs before the PCI subsystem is initialized, so just 88 access the northbridge directly. */ 89 90 static u32 __init allocate_aperture(void) 91 { 92 u32 aper_size; 93 unsigned long addr; 94 95 /* aper_size should <= 1G */ 96 if (fallback_aper_order > 5) 97 fallback_aper_order = 5; 98 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 99 100 /* 101 * Aperture has to be naturally aligned. This means a 2GB aperture 102 * won't have much chance of finding a place in the lower 4GB of 103 * memory. Unfortunately we cannot move it up because that would 104 * make the IOMMU useless. 105 */ 106 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 107 aper_size, aper_size); 108 if (!addr) { 109 pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n", 110 addr, addr + aper_size - 1, aper_size >> 10); 111 return 0; 112 } 113 memblock_reserve(addr, aper_size); 114 pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n", 115 addr, addr + aper_size - 1, aper_size >> 10); 116 register_nosave_region(addr >> PAGE_SHIFT, 117 (addr+aper_size) >> PAGE_SHIFT); 118 119 return (u32)addr; 120 } 121 122 123 /* Find a PCI capability */ 124 static u32 __init find_cap(int bus, int slot, int func, int cap) 125 { 126 int bytes; 127 u8 pos; 128 129 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 130 PCI_STATUS_CAP_LIST)) 131 return 0; 132 133 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 134 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 135 u8 id; 136 137 pos &= ~3; 138 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 139 if (id == 0xff) 140 break; 141 if (id == cap) 142 return pos; 143 pos = read_pci_config_byte(bus, slot, func, 144 pos+PCI_CAP_LIST_NEXT); 145 } 146 return 0; 147 } 148 149 /* Read a standard AGPv3 bridge header */ 150 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 151 { 152 u32 apsize; 153 u32 apsizereg; 154 int nbits; 155 u32 aper_low, aper_hi; 156 u64 aper; 157 u32 old_order; 158 159 pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n", bus, slot, func); 160 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 161 if (apsizereg == 0xffffffff) { 162 pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n", 163 bus, slot, func); 164 return 0; 165 } 166 167 /* old_order could be the value from NB gart setting */ 168 old_order = *order; 169 170 apsize = apsizereg & 0xfff; 171 /* Some BIOS use weird encodings not in the AGPv3 table. */ 172 if (apsize & 0xff) 173 apsize |= 0xf00; 174 nbits = hweight16(apsize); 175 *order = 7 - nbits; 176 if ((int)*order < 0) /* < 32MB */ 177 *order = 0; 178 179 aper_low = read_pci_config(bus, slot, func, 0x10); 180 aper_hi = read_pci_config(bus, slot, func, 0x14); 181 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 182 183 /* 184 * On some sick chips, APSIZE is 0. It means it wants 4G 185 * so let double check that order, and lets trust AMD NB settings: 186 */ 187 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n", 188 bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, 189 32 << old_order); 190 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 191 pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n", 192 bus, slot, func, 32 << *order, apsizereg); 193 *order = old_order; 194 } 195 196 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n", 197 bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, 198 32 << *order, apsizereg); 199 200 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 201 return 0; 202 return (u32)aper; 203 } 204 205 /* 206 * Look for an AGP bridge. Windows only expects the aperture in the 207 * AGP bridge and some BIOS forget to initialize the Northbridge too. 208 * Work around this here. 209 * 210 * Do an PCI bus scan by hand because we're running before the PCI 211 * subsystem. 212 * 213 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 214 * generically. It's probably overkill to always scan all slots because 215 * the AGP bridges should be always an own bus on the HT hierarchy, 216 * but do it here for future safety. 217 */ 218 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 219 { 220 int bus, slot, func; 221 222 /* Poor man's PCI discovery */ 223 for (bus = 0; bus < 256; bus++) { 224 for (slot = 0; slot < 32; slot++) { 225 for (func = 0; func < 8; func++) { 226 u32 class, cap; 227 u8 type; 228 class = read_pci_config(bus, slot, func, 229 PCI_CLASS_REVISION); 230 if (class == 0xffffffff) 231 break; 232 233 switch (class >> 16) { 234 case PCI_CLASS_BRIDGE_HOST: 235 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 236 /* AGP bridge? */ 237 cap = find_cap(bus, slot, func, 238 PCI_CAP_ID_AGP); 239 if (!cap) 240 break; 241 *valid_agp = 1; 242 return read_agp(bus, slot, func, cap, 243 order); 244 } 245 246 /* No multi-function device? */ 247 type = read_pci_config_byte(bus, slot, func, 248 PCI_HEADER_TYPE); 249 if (!(type & 0x80)) 250 break; 251 } 252 } 253 } 254 pr_info("No AGP bridge found\n"); 255 256 return 0; 257 } 258 259 static bool gart_fix_e820 __initdata = true; 260 261 static int __init parse_gart_mem(char *p) 262 { 263 return kstrtobool(p, &gart_fix_e820); 264 } 265 early_param("gart_fix_e820", parse_gart_mem); 266 267 void __init early_gart_iommu_check(void) 268 { 269 /* 270 * in case it is enabled before, esp for kexec/kdump, 271 * previous kernel already enable that. memset called 272 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 273 * or second kernel have different position for GART hole. and new 274 * kernel could use hole as RAM that is still used by GART set by 275 * first kernel 276 * or BIOS forget to put that in reserved. 277 * try to update e820 to make that region as reserved. 278 */ 279 u32 agp_aper_order = 0; 280 int i, fix, slot, valid_agp = 0; 281 u32 ctl; 282 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 283 u64 aper_base = 0, last_aper_base = 0; 284 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 285 286 if (!amd_gart_present()) 287 return; 288 289 if (!early_pci_allowed()) 290 return; 291 292 /* This is mostly duplicate of iommu_hole_init */ 293 search_agp_bridge(&agp_aper_order, &valid_agp); 294 295 fix = 0; 296 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 297 int bus; 298 int dev_base, dev_limit; 299 300 bus = amd_nb_bus_dev_ranges[i].bus; 301 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 302 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 303 304 for (slot = dev_base; slot < dev_limit; slot++) { 305 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 306 continue; 307 308 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 309 aper_enabled = ctl & GARTEN; 310 aper_order = (ctl >> 1) & 7; 311 aper_size = (32 * 1024 * 1024) << aper_order; 312 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 313 aper_base <<= 25; 314 315 if (last_valid) { 316 if ((aper_order != last_aper_order) || 317 (aper_base != last_aper_base) || 318 (aper_enabled != last_aper_enabled)) { 319 fix = 1; 320 break; 321 } 322 } 323 324 last_aper_order = aper_order; 325 last_aper_base = aper_base; 326 last_aper_enabled = aper_enabled; 327 last_valid = 1; 328 } 329 } 330 331 if (!fix && !aper_enabled) 332 return; 333 334 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 335 fix = 1; 336 337 if (gart_fix_e820 && !fix && aper_enabled) { 338 if (e820__mapped_any(aper_base, aper_base + aper_size, 339 E820_TYPE_RAM)) { 340 /* reserve it, so we can reuse it in second kernel */ 341 pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n", 342 aper_base, aper_base + aper_size - 1); 343 e820__range_add(aper_base, aper_size, E820_TYPE_RESERVED); 344 e820__update_table_print(); 345 } 346 } 347 348 if (valid_agp) 349 return; 350 351 /* disable them all at first */ 352 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 353 int bus; 354 int dev_base, dev_limit; 355 356 bus = amd_nb_bus_dev_ranges[i].bus; 357 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 358 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 359 360 for (slot = dev_base; slot < dev_limit; slot++) { 361 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 362 continue; 363 364 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 365 ctl &= ~GARTEN; 366 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 367 } 368 } 369 370 } 371 372 static int __initdata printed_gart_size_msg; 373 374 int __init gart_iommu_hole_init(void) 375 { 376 u32 agp_aper_base = 0, agp_aper_order = 0; 377 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 378 u64 aper_base, last_aper_base = 0; 379 int fix, slot, valid_agp = 0; 380 int i, node; 381 382 if (!amd_gart_present()) 383 return -ENODEV; 384 385 if (gart_iommu_aperture_disabled || !fix_aperture || 386 !early_pci_allowed()) 387 return -ENODEV; 388 389 pr_info("Checking aperture...\n"); 390 391 if (!fallback_aper_force) 392 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 393 394 fix = 0; 395 node = 0; 396 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 397 int bus; 398 int dev_base, dev_limit; 399 u32 ctl; 400 401 bus = amd_nb_bus_dev_ranges[i].bus; 402 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 403 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 404 405 for (slot = dev_base; slot < dev_limit; slot++) { 406 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 407 continue; 408 409 iommu_detected = 1; 410 gart_iommu_aperture = 1; 411 x86_init.iommu.iommu_init = gart_iommu_init; 412 413 ctl = read_pci_config(bus, slot, 3, 414 AMD64_GARTAPERTURECTL); 415 416 /* 417 * Before we do anything else disable the GART. It may 418 * still be enabled if we boot into a crash-kernel here. 419 * Reconfiguring the GART while it is enabled could have 420 * unknown side-effects. 421 */ 422 ctl &= ~GARTEN; 423 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 424 425 aper_order = (ctl >> 1) & 7; 426 aper_size = (32 * 1024 * 1024) << aper_order; 427 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 428 aper_base <<= 25; 429 430 pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n", 431 node, aper_base, aper_base + aper_size - 1, 432 aper_size >> 20); 433 node++; 434 435 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 436 if (valid_agp && agp_aper_base && 437 agp_aper_base == aper_base && 438 agp_aper_order == aper_order) { 439 /* the same between two setting from NB and agp */ 440 if (!no_iommu && 441 max_pfn > MAX_DMA32_PFN && 442 !printed_gart_size_msg) { 443 pr_err("you are using iommu with agp, but GART size is less than 64MB\n"); 444 pr_err("please increase GART size in your BIOS setup\n"); 445 pr_err("if BIOS doesn't have that option, contact your HW vendor!\n"); 446 printed_gart_size_msg = 1; 447 } 448 } else { 449 fix = 1; 450 goto out; 451 } 452 } 453 454 if ((last_aper_order && aper_order != last_aper_order) || 455 (last_aper_base && aper_base != last_aper_base)) { 456 fix = 1; 457 goto out; 458 } 459 last_aper_order = aper_order; 460 last_aper_base = aper_base; 461 } 462 } 463 464 out: 465 if (!fix && !fallback_aper_force) { 466 if (last_aper_base) { 467 /* 468 * If this is the kdump kernel, the first kernel 469 * may have allocated the range over its e820 RAM 470 * and fixed up the northbridge 471 */ 472 exclude_from_vmcore(last_aper_base, last_aper_order); 473 474 return 1; 475 } 476 return 0; 477 } 478 479 if (!fallback_aper_force) { 480 aper_alloc = agp_aper_base; 481 aper_order = agp_aper_order; 482 } 483 484 if (aper_alloc) { 485 /* Got the aperture from the AGP bridge */ 486 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 487 force_iommu || 488 valid_agp || 489 fallback_aper_force) { 490 pr_info("Your BIOS doesn't leave an aperture memory hole\n"); 491 pr_info("Please enable the IOMMU option in the BIOS setup\n"); 492 pr_info("This costs you %dMB of RAM\n", 493 32 << fallback_aper_order); 494 495 aper_order = fallback_aper_order; 496 aper_alloc = allocate_aperture(); 497 if (!aper_alloc) { 498 /* 499 * Could disable AGP and IOMMU here, but it's 500 * probably not worth it. But the later users 501 * cannot deal with bad apertures and turning 502 * on the aperture over memory causes very 503 * strange problems, so it's better to panic 504 * early. 505 */ 506 panic("Not enough memory for aperture"); 507 } 508 } else { 509 return 0; 510 } 511 512 /* 513 * If this is the kdump kernel _and_ the first kernel did not 514 * configure the aperture in the northbridge, this range may 515 * overlap with the first kernel's memory. We can't access the 516 * range through vmcore even though it should be part of the dump. 517 */ 518 exclude_from_vmcore(aper_alloc, aper_order); 519 520 /* Fix up the north bridges */ 521 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 522 int bus, dev_base, dev_limit; 523 524 /* 525 * Don't enable translation yet but enable GART IO and CPU 526 * accesses and set DISTLBWALKPRB since GART table memory is UC. 527 */ 528 u32 ctl = aper_order << 1; 529 530 bus = amd_nb_bus_dev_ranges[i].bus; 531 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 532 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 533 for (slot = dev_base; slot < dev_limit; slot++) { 534 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 535 continue; 536 537 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 538 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 539 } 540 } 541 542 set_up_gart_resume(aper_order, aper_alloc); 543 544 return 1; 545 } 546