1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #define pr_fmt(fmt) "AGP: " fmt 14 15 #include <linux/kernel.h> 16 #include <linux/types.h> 17 #include <linux/init.h> 18 #include <linux/memblock.h> 19 #include <linux/mmzone.h> 20 #include <linux/pci_ids.h> 21 #include <linux/pci.h> 22 #include <linux/bitops.h> 23 #include <linux/suspend.h> 24 #include <asm/e820.h> 25 #include <asm/io.h> 26 #include <asm/iommu.h> 27 #include <asm/gart.h> 28 #include <asm/pci-direct.h> 29 #include <asm/dma.h> 30 #include <asm/amd_nb.h> 31 #include <asm/x86_init.h> 32 33 /* 34 * Using 512M as goal, in case kexec will load kernel_big 35 * that will do the on-position decompress, and could overlap with 36 * with the gart aperture that is used. 37 * Sequence: 38 * kernel_small 39 * ==> kexec (with kdump trigger path or gart still enabled) 40 * ==> kernel_small (gart area become e820_reserved) 41 * ==> kexec (with kdump trigger path or gart still enabled) 42 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 43 * So don't use 512M below as gart iommu, leave the space for kernel 44 * code for safe. 45 */ 46 #define GART_MIN_ADDR (512ULL << 20) 47 #define GART_MAX_ADDR (1ULL << 32) 48 49 int gart_iommu_aperture; 50 int gart_iommu_aperture_disabled __initdata; 51 int gart_iommu_aperture_allowed __initdata; 52 53 int fallback_aper_order __initdata = 1; /* 64MB */ 54 int fallback_aper_force __initdata; 55 56 int fix_aperture __initdata = 1; 57 58 /* This code runs before the PCI subsystem is initialized, so just 59 access the northbridge directly. */ 60 61 static u32 __init allocate_aperture(void) 62 { 63 u32 aper_size; 64 unsigned long addr; 65 66 /* aper_size should <= 1G */ 67 if (fallback_aper_order > 5) 68 fallback_aper_order = 5; 69 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 70 71 /* 72 * Aperture has to be naturally aligned. This means a 2GB aperture 73 * won't have much chance of finding a place in the lower 4GB of 74 * memory. Unfortunately we cannot move it up because that would 75 * make the IOMMU useless. 76 */ 77 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 78 aper_size, aper_size); 79 if (!addr) { 80 pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n", 81 addr, addr + aper_size - 1, aper_size >> 10); 82 return 0; 83 } 84 memblock_reserve(addr, aper_size); 85 pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n", 86 addr, addr + aper_size - 1, aper_size >> 10); 87 register_nosave_region(addr >> PAGE_SHIFT, 88 (addr+aper_size) >> PAGE_SHIFT); 89 90 return (u32)addr; 91 } 92 93 94 /* Find a PCI capability */ 95 static u32 __init find_cap(int bus, int slot, int func, int cap) 96 { 97 int bytes; 98 u8 pos; 99 100 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 101 PCI_STATUS_CAP_LIST)) 102 return 0; 103 104 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 105 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 106 u8 id; 107 108 pos &= ~3; 109 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 110 if (id == 0xff) 111 break; 112 if (id == cap) 113 return pos; 114 pos = read_pci_config_byte(bus, slot, func, 115 pos+PCI_CAP_LIST_NEXT); 116 } 117 return 0; 118 } 119 120 /* Read a standard AGPv3 bridge header */ 121 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 122 { 123 u32 apsize; 124 u32 apsizereg; 125 int nbits; 126 u32 aper_low, aper_hi; 127 u64 aper; 128 u32 old_order; 129 130 pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n", bus, slot, func); 131 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 132 if (apsizereg == 0xffffffff) { 133 pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n", 134 bus, slot, func); 135 return 0; 136 } 137 138 /* old_order could be the value from NB gart setting */ 139 old_order = *order; 140 141 apsize = apsizereg & 0xfff; 142 /* Some BIOS use weird encodings not in the AGPv3 table. */ 143 if (apsize & 0xff) 144 apsize |= 0xf00; 145 nbits = hweight16(apsize); 146 *order = 7 - nbits; 147 if ((int)*order < 0) /* < 32MB */ 148 *order = 0; 149 150 aper_low = read_pci_config(bus, slot, func, 0x10); 151 aper_hi = read_pci_config(bus, slot, func, 0x14); 152 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 153 154 /* 155 * On some sick chips, APSIZE is 0. It means it wants 4G 156 * so let double check that order, and lets trust AMD NB settings: 157 */ 158 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n", 159 bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, 160 32 << old_order); 161 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 162 pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n", 163 bus, slot, func, 32 << *order, apsizereg); 164 *order = old_order; 165 } 166 167 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n", 168 bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, 169 32 << *order, apsizereg); 170 171 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 172 return 0; 173 return (u32)aper; 174 } 175 176 /* 177 * Look for an AGP bridge. Windows only expects the aperture in the 178 * AGP bridge and some BIOS forget to initialize the Northbridge too. 179 * Work around this here. 180 * 181 * Do an PCI bus scan by hand because we're running before the PCI 182 * subsystem. 183 * 184 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 185 * generically. It's probably overkill to always scan all slots because 186 * the AGP bridges should be always an own bus on the HT hierarchy, 187 * but do it here for future safety. 188 */ 189 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 190 { 191 int bus, slot, func; 192 193 /* Poor man's PCI discovery */ 194 for (bus = 0; bus < 256; bus++) { 195 for (slot = 0; slot < 32; slot++) { 196 for (func = 0; func < 8; func++) { 197 u32 class, cap; 198 u8 type; 199 class = read_pci_config(bus, slot, func, 200 PCI_CLASS_REVISION); 201 if (class == 0xffffffff) 202 break; 203 204 switch (class >> 16) { 205 case PCI_CLASS_BRIDGE_HOST: 206 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 207 /* AGP bridge? */ 208 cap = find_cap(bus, slot, func, 209 PCI_CAP_ID_AGP); 210 if (!cap) 211 break; 212 *valid_agp = 1; 213 return read_agp(bus, slot, func, cap, 214 order); 215 } 216 217 /* No multi-function device? */ 218 type = read_pci_config_byte(bus, slot, func, 219 PCI_HEADER_TYPE); 220 if (!(type & 0x80)) 221 break; 222 } 223 } 224 } 225 pr_info("No AGP bridge found\n"); 226 227 return 0; 228 } 229 230 static int gart_fix_e820 __initdata = 1; 231 232 static int __init parse_gart_mem(char *p) 233 { 234 if (!p) 235 return -EINVAL; 236 237 if (!strncmp(p, "off", 3)) 238 gart_fix_e820 = 0; 239 else if (!strncmp(p, "on", 2)) 240 gart_fix_e820 = 1; 241 242 return 0; 243 } 244 early_param("gart_fix_e820", parse_gart_mem); 245 246 void __init early_gart_iommu_check(void) 247 { 248 /* 249 * in case it is enabled before, esp for kexec/kdump, 250 * previous kernel already enable that. memset called 251 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 252 * or second kernel have different position for GART hole. and new 253 * kernel could use hole as RAM that is still used by GART set by 254 * first kernel 255 * or BIOS forget to put that in reserved. 256 * try to update e820 to make that region as reserved. 257 */ 258 u32 agp_aper_order = 0; 259 int i, fix, slot, valid_agp = 0; 260 u32 ctl; 261 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 262 u64 aper_base = 0, last_aper_base = 0; 263 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 264 265 if (!amd_gart_present()) 266 return; 267 268 if (!early_pci_allowed()) 269 return; 270 271 /* This is mostly duplicate of iommu_hole_init */ 272 search_agp_bridge(&agp_aper_order, &valid_agp); 273 274 fix = 0; 275 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 276 int bus; 277 int dev_base, dev_limit; 278 279 bus = amd_nb_bus_dev_ranges[i].bus; 280 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 281 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 282 283 for (slot = dev_base; slot < dev_limit; slot++) { 284 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 285 continue; 286 287 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 288 aper_enabled = ctl & GARTEN; 289 aper_order = (ctl >> 1) & 7; 290 aper_size = (32 * 1024 * 1024) << aper_order; 291 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 292 aper_base <<= 25; 293 294 if (last_valid) { 295 if ((aper_order != last_aper_order) || 296 (aper_base != last_aper_base) || 297 (aper_enabled != last_aper_enabled)) { 298 fix = 1; 299 break; 300 } 301 } 302 303 last_aper_order = aper_order; 304 last_aper_base = aper_base; 305 last_aper_enabled = aper_enabled; 306 last_valid = 1; 307 } 308 } 309 310 if (!fix && !aper_enabled) 311 return; 312 313 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 314 fix = 1; 315 316 if (gart_fix_e820 && !fix && aper_enabled) { 317 if (e820_any_mapped(aper_base, aper_base + aper_size, 318 E820_RAM)) { 319 /* reserve it, so we can reuse it in second kernel */ 320 pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n", 321 aper_base, aper_base + aper_size - 1); 322 e820_add_region(aper_base, aper_size, E820_RESERVED); 323 update_e820(); 324 } 325 } 326 327 if (valid_agp) 328 return; 329 330 /* disable them all at first */ 331 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 332 int bus; 333 int dev_base, dev_limit; 334 335 bus = amd_nb_bus_dev_ranges[i].bus; 336 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 337 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 338 339 for (slot = dev_base; slot < dev_limit; slot++) { 340 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 341 continue; 342 343 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 344 ctl &= ~GARTEN; 345 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 346 } 347 } 348 349 } 350 351 static int __initdata printed_gart_size_msg; 352 353 int __init gart_iommu_hole_init(void) 354 { 355 u32 agp_aper_base = 0, agp_aper_order = 0; 356 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 357 u64 aper_base, last_aper_base = 0; 358 int fix, slot, valid_agp = 0; 359 int i, node; 360 361 if (!amd_gart_present()) 362 return -ENODEV; 363 364 if (gart_iommu_aperture_disabled || !fix_aperture || 365 !early_pci_allowed()) 366 return -ENODEV; 367 368 pr_info("Checking aperture...\n"); 369 370 if (!fallback_aper_force) 371 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 372 373 fix = 0; 374 node = 0; 375 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 376 int bus; 377 int dev_base, dev_limit; 378 u32 ctl; 379 380 bus = amd_nb_bus_dev_ranges[i].bus; 381 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 382 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 383 384 for (slot = dev_base; slot < dev_limit; slot++) { 385 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 386 continue; 387 388 iommu_detected = 1; 389 gart_iommu_aperture = 1; 390 x86_init.iommu.iommu_init = gart_iommu_init; 391 392 ctl = read_pci_config(bus, slot, 3, 393 AMD64_GARTAPERTURECTL); 394 395 /* 396 * Before we do anything else disable the GART. It may 397 * still be enabled if we boot into a crash-kernel here. 398 * Reconfiguring the GART while it is enabled could have 399 * unknown side-effects. 400 */ 401 ctl &= ~GARTEN; 402 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 403 404 aper_order = (ctl >> 1) & 7; 405 aper_size = (32 * 1024 * 1024) << aper_order; 406 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 407 aper_base <<= 25; 408 409 pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n", 410 node, aper_base, aper_base + aper_size - 1, 411 aper_size >> 20); 412 node++; 413 414 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 415 if (valid_agp && agp_aper_base && 416 agp_aper_base == aper_base && 417 agp_aper_order == aper_order) { 418 /* the same between two setting from NB and agp */ 419 if (!no_iommu && 420 max_pfn > MAX_DMA32_PFN && 421 !printed_gart_size_msg) { 422 pr_err("you are using iommu with agp, but GART size is less than 64MB\n"); 423 pr_err("please increase GART size in your BIOS setup\n"); 424 pr_err("if BIOS doesn't have that option, contact your HW vendor!\n"); 425 printed_gart_size_msg = 1; 426 } 427 } else { 428 fix = 1; 429 goto out; 430 } 431 } 432 433 if ((last_aper_order && aper_order != last_aper_order) || 434 (last_aper_base && aper_base != last_aper_base)) { 435 fix = 1; 436 goto out; 437 } 438 last_aper_order = aper_order; 439 last_aper_base = aper_base; 440 } 441 } 442 443 out: 444 if (!fix && !fallback_aper_force) { 445 if (last_aper_base) 446 return 1; 447 return 0; 448 } 449 450 if (!fallback_aper_force) { 451 aper_alloc = agp_aper_base; 452 aper_order = agp_aper_order; 453 } 454 455 if (aper_alloc) { 456 /* Got the aperture from the AGP bridge */ 457 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 458 force_iommu || 459 valid_agp || 460 fallback_aper_force) { 461 pr_info("Your BIOS doesn't leave an aperture memory hole\n"); 462 pr_info("Please enable the IOMMU option in the BIOS setup\n"); 463 pr_info("This costs you %dMB of RAM\n", 464 32 << fallback_aper_order); 465 466 aper_order = fallback_aper_order; 467 aper_alloc = allocate_aperture(); 468 if (!aper_alloc) { 469 /* 470 * Could disable AGP and IOMMU here, but it's 471 * probably not worth it. But the later users 472 * cannot deal with bad apertures and turning 473 * on the aperture over memory causes very 474 * strange problems, so it's better to panic 475 * early. 476 */ 477 panic("Not enough memory for aperture"); 478 } 479 } else { 480 return 0; 481 } 482 483 /* Fix up the north bridges */ 484 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 485 int bus, dev_base, dev_limit; 486 487 /* 488 * Don't enable translation yet but enable GART IO and CPU 489 * accesses and set DISTLBWALKPRB since GART table memory is UC. 490 */ 491 u32 ctl = aper_order << 1; 492 493 bus = amd_nb_bus_dev_ranges[i].bus; 494 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 495 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 496 for (slot = dev_base; slot < dev_limit; slot++) { 497 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 498 continue; 499 500 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 501 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 502 } 503 } 504 505 set_up_gart_resume(aper_order, aper_alloc); 506 507 return 1; 508 } 509