1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/memblock.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/ioport.h> 22 #include <linux/suspend.h> 23 #include <linux/kmemleak.h> 24 #include <asm/e820.h> 25 #include <asm/io.h> 26 #include <asm/iommu.h> 27 #include <asm/gart.h> 28 #include <asm/pci-direct.h> 29 #include <asm/dma.h> 30 #include <asm/amd_nb.h> 31 #include <asm/x86_init.h> 32 33 int gart_iommu_aperture; 34 int gart_iommu_aperture_disabled __initdata; 35 int gart_iommu_aperture_allowed __initdata; 36 37 int fallback_aper_order __initdata = 1; /* 64MB */ 38 int fallback_aper_force __initdata; 39 40 int fix_aperture __initdata = 1; 41 42 static struct resource gart_resource = { 43 .name = "GART", 44 .flags = IORESOURCE_MEM, 45 }; 46 47 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 48 { 49 gart_resource.start = aper_base; 50 gart_resource.end = aper_base + aper_size - 1; 51 insert_resource(&iomem_resource, &gart_resource); 52 } 53 54 /* This code runs before the PCI subsystem is initialized, so just 55 access the northbridge directly. */ 56 57 static u32 __init allocate_aperture(void) 58 { 59 u32 aper_size; 60 unsigned long addr; 61 62 /* aper_size should <= 1G */ 63 if (fallback_aper_order > 5) 64 fallback_aper_order = 5; 65 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 66 67 /* 68 * Aperture has to be naturally aligned. This means a 2GB aperture 69 * won't have much chance of finding a place in the lower 4GB of 70 * memory. Unfortunately we cannot move it up because that would 71 * make the IOMMU useless. 72 */ 73 /* 74 * using 512M as goal, in case kexec will load kernel_big 75 * that will do the on position decompress, and could overlap with 76 * that position with gart that is used. 77 * sequende: 78 * kernel_small 79 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 80 * ==> kernel_small(gart area become e820_reserved) 81 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 82 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 83 * so don't use 512M below as gart iommu, leave the space for kernel 84 * code for safe 85 */ 86 addr = memblock_find_in_range(0, 1ULL<<32, aper_size, 512ULL<<20); 87 if (addr == MEMBLOCK_ERROR || addr + aper_size > 0xffffffff) { 88 printk(KERN_ERR 89 "Cannot allocate aperture memory hole (%lx,%uK)\n", 90 addr, aper_size>>10); 91 return 0; 92 } 93 memblock_x86_reserve_range(addr, addr + aper_size, "aperture64"); 94 /* 95 * Kmemleak should not scan this block as it may not be mapped via the 96 * kernel direct mapping. 97 */ 98 kmemleak_ignore(phys_to_virt(addr)); 99 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 100 aper_size >> 10, addr); 101 insert_aperture_resource((u32)addr, aper_size); 102 register_nosave_region(addr >> PAGE_SHIFT, 103 (addr+aper_size) >> PAGE_SHIFT); 104 105 return (u32)addr; 106 } 107 108 109 /* Find a PCI capability */ 110 static u32 __init find_cap(int bus, int slot, int func, int cap) 111 { 112 int bytes; 113 u8 pos; 114 115 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 116 PCI_STATUS_CAP_LIST)) 117 return 0; 118 119 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 120 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 121 u8 id; 122 123 pos &= ~3; 124 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 125 if (id == 0xff) 126 break; 127 if (id == cap) 128 return pos; 129 pos = read_pci_config_byte(bus, slot, func, 130 pos+PCI_CAP_LIST_NEXT); 131 } 132 return 0; 133 } 134 135 /* Read a standard AGPv3 bridge header */ 136 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 137 { 138 u32 apsize; 139 u32 apsizereg; 140 int nbits; 141 u32 aper_low, aper_hi; 142 u64 aper; 143 u32 old_order; 144 145 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 146 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 147 if (apsizereg == 0xffffffff) { 148 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 149 return 0; 150 } 151 152 /* old_order could be the value from NB gart setting */ 153 old_order = *order; 154 155 apsize = apsizereg & 0xfff; 156 /* Some BIOS use weird encodings not in the AGPv3 table. */ 157 if (apsize & 0xff) 158 apsize |= 0xf00; 159 nbits = hweight16(apsize); 160 *order = 7 - nbits; 161 if ((int)*order < 0) /* < 32MB */ 162 *order = 0; 163 164 aper_low = read_pci_config(bus, slot, func, 0x10); 165 aper_hi = read_pci_config(bus, slot, func, 0x14); 166 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 167 168 /* 169 * On some sick chips, APSIZE is 0. It means it wants 4G 170 * so let double check that order, and lets trust AMD NB settings: 171 */ 172 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 173 aper, 32 << old_order); 174 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 175 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 176 32 << *order, apsizereg); 177 *order = old_order; 178 } 179 180 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 181 aper, 32 << *order, apsizereg); 182 183 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 184 return 0; 185 return (u32)aper; 186 } 187 188 /* 189 * Look for an AGP bridge. Windows only expects the aperture in the 190 * AGP bridge and some BIOS forget to initialize the Northbridge too. 191 * Work around this here. 192 * 193 * Do an PCI bus scan by hand because we're running before the PCI 194 * subsystem. 195 * 196 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 197 * generically. It's probably overkill to always scan all slots because 198 * the AGP bridges should be always an own bus on the HT hierarchy, 199 * but do it here for future safety. 200 */ 201 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 202 { 203 int bus, slot, func; 204 205 /* Poor man's PCI discovery */ 206 for (bus = 0; bus < 256; bus++) { 207 for (slot = 0; slot < 32; slot++) { 208 for (func = 0; func < 8; func++) { 209 u32 class, cap; 210 u8 type; 211 class = read_pci_config(bus, slot, func, 212 PCI_CLASS_REVISION); 213 if (class == 0xffffffff) 214 break; 215 216 switch (class >> 16) { 217 case PCI_CLASS_BRIDGE_HOST: 218 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 219 /* AGP bridge? */ 220 cap = find_cap(bus, slot, func, 221 PCI_CAP_ID_AGP); 222 if (!cap) 223 break; 224 *valid_agp = 1; 225 return read_agp(bus, slot, func, cap, 226 order); 227 } 228 229 /* No multi-function device? */ 230 type = read_pci_config_byte(bus, slot, func, 231 PCI_HEADER_TYPE); 232 if (!(type & 0x80)) 233 break; 234 } 235 } 236 } 237 printk(KERN_INFO "No AGP bridge found\n"); 238 239 return 0; 240 } 241 242 static int gart_fix_e820 __initdata = 1; 243 244 static int __init parse_gart_mem(char *p) 245 { 246 if (!p) 247 return -EINVAL; 248 249 if (!strncmp(p, "off", 3)) 250 gart_fix_e820 = 0; 251 else if (!strncmp(p, "on", 2)) 252 gart_fix_e820 = 1; 253 254 return 0; 255 } 256 early_param("gart_fix_e820", parse_gart_mem); 257 258 void __init early_gart_iommu_check(void) 259 { 260 /* 261 * in case it is enabled before, esp for kexec/kdump, 262 * previous kernel already enable that. memset called 263 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 264 * or second kernel have different position for GART hole. and new 265 * kernel could use hole as RAM that is still used by GART set by 266 * first kernel 267 * or BIOS forget to put that in reserved. 268 * try to update e820 to make that region as reserved. 269 */ 270 u32 agp_aper_order = 0; 271 int i, fix, slot, valid_agp = 0; 272 u32 ctl; 273 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 274 u64 aper_base = 0, last_aper_base = 0; 275 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 276 277 if (!early_pci_allowed()) 278 return; 279 280 /* This is mostly duplicate of iommu_hole_init */ 281 search_agp_bridge(&agp_aper_order, &valid_agp); 282 283 fix = 0; 284 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 285 int bus; 286 int dev_base, dev_limit; 287 288 bus = amd_nb_bus_dev_ranges[i].bus; 289 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 290 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 291 292 for (slot = dev_base; slot < dev_limit; slot++) { 293 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 294 continue; 295 296 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 297 aper_enabled = ctl & GARTEN; 298 aper_order = (ctl >> 1) & 7; 299 aper_size = (32 * 1024 * 1024) << aper_order; 300 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 301 aper_base <<= 25; 302 303 if (last_valid) { 304 if ((aper_order != last_aper_order) || 305 (aper_base != last_aper_base) || 306 (aper_enabled != last_aper_enabled)) { 307 fix = 1; 308 break; 309 } 310 } 311 312 last_aper_order = aper_order; 313 last_aper_base = aper_base; 314 last_aper_enabled = aper_enabled; 315 last_valid = 1; 316 } 317 } 318 319 if (!fix && !aper_enabled) 320 return; 321 322 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 323 fix = 1; 324 325 if (gart_fix_e820 && !fix && aper_enabled) { 326 if (e820_any_mapped(aper_base, aper_base + aper_size, 327 E820_RAM)) { 328 /* reserve it, so we can reuse it in second kernel */ 329 printk(KERN_INFO "update e820 for GART\n"); 330 e820_add_region(aper_base, aper_size, E820_RESERVED); 331 update_e820(); 332 } 333 } 334 335 if (valid_agp) 336 return; 337 338 /* disable them all at first */ 339 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 340 int bus; 341 int dev_base, dev_limit; 342 343 bus = amd_nb_bus_dev_ranges[i].bus; 344 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 345 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 346 347 for (slot = dev_base; slot < dev_limit; slot++) { 348 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 349 continue; 350 351 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 352 ctl &= ~GARTEN; 353 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 354 } 355 } 356 357 } 358 359 static int __initdata printed_gart_size_msg; 360 361 int __init gart_iommu_hole_init(void) 362 { 363 u32 agp_aper_base = 0, agp_aper_order = 0; 364 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 365 u64 aper_base, last_aper_base = 0; 366 int fix, slot, valid_agp = 0; 367 int i, node; 368 369 if (gart_iommu_aperture_disabled || !fix_aperture || 370 !early_pci_allowed()) 371 return -ENODEV; 372 373 printk(KERN_INFO "Checking aperture...\n"); 374 375 if (!fallback_aper_force) 376 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 377 378 fix = 0; 379 node = 0; 380 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 381 int bus; 382 int dev_base, dev_limit; 383 u32 ctl; 384 385 bus = amd_nb_bus_dev_ranges[i].bus; 386 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 387 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 388 389 for (slot = dev_base; slot < dev_limit; slot++) { 390 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 391 continue; 392 393 iommu_detected = 1; 394 gart_iommu_aperture = 1; 395 x86_init.iommu.iommu_init = gart_iommu_init; 396 397 ctl = read_pci_config(bus, slot, 3, 398 AMD64_GARTAPERTURECTL); 399 400 /* 401 * Before we do anything else disable the GART. It may 402 * still be enabled if we boot into a crash-kernel here. 403 * Reconfiguring the GART while it is enabled could have 404 * unknown side-effects. 405 */ 406 ctl &= ~GARTEN; 407 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 408 409 aper_order = (ctl >> 1) & 7; 410 aper_size = (32 * 1024 * 1024) << aper_order; 411 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 412 aper_base <<= 25; 413 414 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 415 node, aper_base, aper_size >> 20); 416 node++; 417 418 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 419 if (valid_agp && agp_aper_base && 420 agp_aper_base == aper_base && 421 agp_aper_order == aper_order) { 422 /* the same between two setting from NB and agp */ 423 if (!no_iommu && 424 max_pfn > MAX_DMA32_PFN && 425 !printed_gart_size_msg) { 426 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 427 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 428 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 429 printed_gart_size_msg = 1; 430 } 431 } else { 432 fix = 1; 433 goto out; 434 } 435 } 436 437 if ((last_aper_order && aper_order != last_aper_order) || 438 (last_aper_base && aper_base != last_aper_base)) { 439 fix = 1; 440 goto out; 441 } 442 last_aper_order = aper_order; 443 last_aper_base = aper_base; 444 } 445 } 446 447 out: 448 if (!fix && !fallback_aper_force) { 449 if (last_aper_base) { 450 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 451 452 insert_aperture_resource((u32)last_aper_base, n); 453 return 1; 454 } 455 return 0; 456 } 457 458 if (!fallback_aper_force) { 459 aper_alloc = agp_aper_base; 460 aper_order = agp_aper_order; 461 } 462 463 if (aper_alloc) { 464 /* Got the aperture from the AGP bridge */ 465 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 466 force_iommu || 467 valid_agp || 468 fallback_aper_force) { 469 printk(KERN_INFO 470 "Your BIOS doesn't leave a aperture memory hole\n"); 471 printk(KERN_INFO 472 "Please enable the IOMMU option in the BIOS setup\n"); 473 printk(KERN_INFO 474 "This costs you %d MB of RAM\n", 475 32 << fallback_aper_order); 476 477 aper_order = fallback_aper_order; 478 aper_alloc = allocate_aperture(); 479 if (!aper_alloc) { 480 /* 481 * Could disable AGP and IOMMU here, but it's 482 * probably not worth it. But the later users 483 * cannot deal with bad apertures and turning 484 * on the aperture over memory causes very 485 * strange problems, so it's better to panic 486 * early. 487 */ 488 panic("Not enough memory for aperture"); 489 } 490 } else { 491 return 0; 492 } 493 494 /* Fix up the north bridges */ 495 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 496 int bus, dev_base, dev_limit; 497 498 /* 499 * Don't enable translation yet but enable GART IO and CPU 500 * accesses and set DISTLBWALKPRB since GART table memory is UC. 501 */ 502 u32 ctl = DISTLBWALKPRB | aper_order << 1; 503 504 bus = amd_nb_bus_dev_ranges[i].bus; 505 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 506 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 507 for (slot = dev_base; slot < dev_limit; slot++) { 508 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 509 continue; 510 511 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 512 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 513 } 514 } 515 516 set_up_gart_resume(aper_order, aper_alloc); 517 518 return 1; 519 } 520