1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/bootmem.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/ioport.h> 22 #include <linux/suspend.h> 23 #include <linux/kmemleak.h> 24 #include <asm/e820.h> 25 #include <asm/io.h> 26 #include <asm/iommu.h> 27 #include <asm/gart.h> 28 #include <asm/pci-direct.h> 29 #include <asm/dma.h> 30 #include <asm/k8.h> 31 #include <asm/x86_init.h> 32 33 int gart_iommu_aperture; 34 EXPORT_SYMBOL_GPL(gart_iommu_aperture); 35 int gart_iommu_aperture_disabled __initdata; 36 int gart_iommu_aperture_allowed __initdata; 37 38 int fallback_aper_order __initdata = 1; /* 64MB */ 39 int fallback_aper_force __initdata; 40 41 int fix_aperture __initdata = 1; 42 43 struct bus_dev_range { 44 int bus; 45 int dev_base; 46 int dev_limit; 47 }; 48 49 static struct bus_dev_range bus_dev_ranges[] __initdata = { 50 { 0x00, 0x18, 0x20}, 51 { 0xff, 0x00, 0x20}, 52 { 0xfe, 0x00, 0x20} 53 }; 54 55 static struct resource gart_resource = { 56 .name = "GART", 57 .flags = IORESOURCE_MEM, 58 }; 59 60 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 61 { 62 gart_resource.start = aper_base; 63 gart_resource.end = aper_base + aper_size - 1; 64 insert_resource(&iomem_resource, &gart_resource); 65 } 66 67 /* This code runs before the PCI subsystem is initialized, so just 68 access the northbridge directly. */ 69 70 static u32 __init allocate_aperture(void) 71 { 72 u32 aper_size; 73 void *p; 74 75 /* aper_size should <= 1G */ 76 if (fallback_aper_order > 5) 77 fallback_aper_order = 5; 78 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 79 80 /* 81 * Aperture has to be naturally aligned. This means a 2GB aperture 82 * won't have much chance of finding a place in the lower 4GB of 83 * memory. Unfortunately we cannot move it up because that would 84 * make the IOMMU useless. 85 */ 86 /* 87 * using 512M as goal, in case kexec will load kernel_big 88 * that will do the on position decompress, and could overlap with 89 * that positon with gart that is used. 90 * sequende: 91 * kernel_small 92 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 93 * ==> kernel_small(gart area become e820_reserved) 94 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 95 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 96 * so don't use 512M below as gart iommu, leave the space for kernel 97 * code for safe 98 */ 99 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20); 100 /* 101 * Kmemleak should not scan this block as it may not be mapped via the 102 * kernel direct mapping. 103 */ 104 kmemleak_ignore(p); 105 if (!p || __pa(p)+aper_size > 0xffffffff) { 106 printk(KERN_ERR 107 "Cannot allocate aperture memory hole (%p,%uK)\n", 108 p, aper_size>>10); 109 if (p) 110 free_bootmem(__pa(p), aper_size); 111 return 0; 112 } 113 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 114 aper_size >> 10, __pa(p)); 115 insert_aperture_resource((u32)__pa(p), aper_size); 116 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT, 117 (u32)__pa(p+aper_size) >> PAGE_SHIFT); 118 119 return (u32)__pa(p); 120 } 121 122 123 /* Find a PCI capability */ 124 static u32 __init find_cap(int bus, int slot, int func, int cap) 125 { 126 int bytes; 127 u8 pos; 128 129 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 130 PCI_STATUS_CAP_LIST)) 131 return 0; 132 133 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 134 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 135 u8 id; 136 137 pos &= ~3; 138 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 139 if (id == 0xff) 140 break; 141 if (id == cap) 142 return pos; 143 pos = read_pci_config_byte(bus, slot, func, 144 pos+PCI_CAP_LIST_NEXT); 145 } 146 return 0; 147 } 148 149 /* Read a standard AGPv3 bridge header */ 150 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 151 { 152 u32 apsize; 153 u32 apsizereg; 154 int nbits; 155 u32 aper_low, aper_hi; 156 u64 aper; 157 u32 old_order; 158 159 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 160 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 161 if (apsizereg == 0xffffffff) { 162 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 163 return 0; 164 } 165 166 /* old_order could be the value from NB gart setting */ 167 old_order = *order; 168 169 apsize = apsizereg & 0xfff; 170 /* Some BIOS use weird encodings not in the AGPv3 table. */ 171 if (apsize & 0xff) 172 apsize |= 0xf00; 173 nbits = hweight16(apsize); 174 *order = 7 - nbits; 175 if ((int)*order < 0) /* < 32MB */ 176 *order = 0; 177 178 aper_low = read_pci_config(bus, slot, func, 0x10); 179 aper_hi = read_pci_config(bus, slot, func, 0x14); 180 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 181 182 /* 183 * On some sick chips, APSIZE is 0. It means it wants 4G 184 * so let double check that order, and lets trust AMD NB settings: 185 */ 186 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 187 aper, 32 << old_order); 188 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 189 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 190 32 << *order, apsizereg); 191 *order = old_order; 192 } 193 194 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 195 aper, 32 << *order, apsizereg); 196 197 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 198 return 0; 199 return (u32)aper; 200 } 201 202 /* 203 * Look for an AGP bridge. Windows only expects the aperture in the 204 * AGP bridge and some BIOS forget to initialize the Northbridge too. 205 * Work around this here. 206 * 207 * Do an PCI bus scan by hand because we're running before the PCI 208 * subsystem. 209 * 210 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan 211 * generically. It's probably overkill to always scan all slots because 212 * the AGP bridges should be always an own bus on the HT hierarchy, 213 * but do it here for future safety. 214 */ 215 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 216 { 217 int bus, slot, func; 218 219 /* Poor man's PCI discovery */ 220 for (bus = 0; bus < 256; bus++) { 221 for (slot = 0; slot < 32; slot++) { 222 for (func = 0; func < 8; func++) { 223 u32 class, cap; 224 u8 type; 225 class = read_pci_config(bus, slot, func, 226 PCI_CLASS_REVISION); 227 if (class == 0xffffffff) 228 break; 229 230 switch (class >> 16) { 231 case PCI_CLASS_BRIDGE_HOST: 232 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 233 /* AGP bridge? */ 234 cap = find_cap(bus, slot, func, 235 PCI_CAP_ID_AGP); 236 if (!cap) 237 break; 238 *valid_agp = 1; 239 return read_agp(bus, slot, func, cap, 240 order); 241 } 242 243 /* No multi-function device? */ 244 type = read_pci_config_byte(bus, slot, func, 245 PCI_HEADER_TYPE); 246 if (!(type & 0x80)) 247 break; 248 } 249 } 250 } 251 printk(KERN_INFO "No AGP bridge found\n"); 252 253 return 0; 254 } 255 256 static int gart_fix_e820 __initdata = 1; 257 258 static int __init parse_gart_mem(char *p) 259 { 260 if (!p) 261 return -EINVAL; 262 263 if (!strncmp(p, "off", 3)) 264 gart_fix_e820 = 0; 265 else if (!strncmp(p, "on", 2)) 266 gart_fix_e820 = 1; 267 268 return 0; 269 } 270 early_param("gart_fix_e820", parse_gart_mem); 271 272 void __init early_gart_iommu_check(void) 273 { 274 /* 275 * in case it is enabled before, esp for kexec/kdump, 276 * previous kernel already enable that. memset called 277 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 278 * or second kernel have different position for GART hole. and new 279 * kernel could use hole as RAM that is still used by GART set by 280 * first kernel 281 * or BIOS forget to put that in reserved. 282 * try to update e820 to make that region as reserved. 283 */ 284 u32 agp_aper_base = 0, agp_aper_order = 0; 285 int i, fix, slot, valid_agp = 0; 286 u32 ctl; 287 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 288 u64 aper_base = 0, last_aper_base = 0; 289 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 290 291 if (!early_pci_allowed()) 292 return; 293 294 /* This is mostly duplicate of iommu_hole_init */ 295 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 296 297 fix = 0; 298 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 299 int bus; 300 int dev_base, dev_limit; 301 302 bus = bus_dev_ranges[i].bus; 303 dev_base = bus_dev_ranges[i].dev_base; 304 dev_limit = bus_dev_ranges[i].dev_limit; 305 306 for (slot = dev_base; slot < dev_limit; slot++) { 307 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 308 continue; 309 310 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 311 aper_enabled = ctl & AMD64_GARTEN; 312 aper_order = (ctl >> 1) & 7; 313 aper_size = (32 * 1024 * 1024) << aper_order; 314 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 315 aper_base <<= 25; 316 317 if (last_valid) { 318 if ((aper_order != last_aper_order) || 319 (aper_base != last_aper_base) || 320 (aper_enabled != last_aper_enabled)) { 321 fix = 1; 322 break; 323 } 324 } 325 326 last_aper_order = aper_order; 327 last_aper_base = aper_base; 328 last_aper_enabled = aper_enabled; 329 last_valid = 1; 330 } 331 } 332 333 if (!fix && !aper_enabled) 334 return; 335 336 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 337 fix = 1; 338 339 if (gart_fix_e820 && !fix && aper_enabled) { 340 if (e820_any_mapped(aper_base, aper_base + aper_size, 341 E820_RAM)) { 342 /* reserve it, so we can reuse it in second kernel */ 343 printk(KERN_INFO "update e820 for GART\n"); 344 e820_add_region(aper_base, aper_size, E820_RESERVED); 345 update_e820(); 346 } 347 } 348 349 if (valid_agp) 350 return; 351 352 /* disable them all at first */ 353 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 354 int bus; 355 int dev_base, dev_limit; 356 357 bus = bus_dev_ranges[i].bus; 358 dev_base = bus_dev_ranges[i].dev_base; 359 dev_limit = bus_dev_ranges[i].dev_limit; 360 361 for (slot = dev_base; slot < dev_limit; slot++) { 362 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 363 continue; 364 365 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 366 ctl &= ~AMD64_GARTEN; 367 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 368 } 369 } 370 371 } 372 373 static int __initdata printed_gart_size_msg; 374 375 void __init gart_iommu_hole_init(void) 376 { 377 u32 agp_aper_base = 0, agp_aper_order = 0; 378 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 379 u64 aper_base, last_aper_base = 0; 380 int fix, slot, valid_agp = 0; 381 int i, node; 382 383 if (gart_iommu_aperture_disabled || !fix_aperture || 384 !early_pci_allowed()) 385 return; 386 387 printk(KERN_INFO "Checking aperture...\n"); 388 389 if (!fallback_aper_force) 390 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 391 392 fix = 0; 393 node = 0; 394 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 395 int bus; 396 int dev_base, dev_limit; 397 398 bus = bus_dev_ranges[i].bus; 399 dev_base = bus_dev_ranges[i].dev_base; 400 dev_limit = bus_dev_ranges[i].dev_limit; 401 402 for (slot = dev_base; slot < dev_limit; slot++) { 403 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 404 continue; 405 406 iommu_detected = 1; 407 gart_iommu_aperture = 1; 408 x86_init.iommu.iommu_init = gart_iommu_init; 409 410 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7; 411 aper_size = (32 * 1024 * 1024) << aper_order; 412 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 413 aper_base <<= 25; 414 415 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 416 node, aper_base, aper_size >> 20); 417 node++; 418 419 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 420 if (valid_agp && agp_aper_base && 421 agp_aper_base == aper_base && 422 agp_aper_order == aper_order) { 423 /* the same between two setting from NB and agp */ 424 if (!no_iommu && 425 max_pfn > MAX_DMA32_PFN && 426 !printed_gart_size_msg) { 427 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 428 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 429 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 430 printed_gart_size_msg = 1; 431 } 432 } else { 433 fix = 1; 434 goto out; 435 } 436 } 437 438 if ((last_aper_order && aper_order != last_aper_order) || 439 (last_aper_base && aper_base != last_aper_base)) { 440 fix = 1; 441 goto out; 442 } 443 last_aper_order = aper_order; 444 last_aper_base = aper_base; 445 } 446 } 447 448 out: 449 if (!fix && !fallback_aper_force) { 450 if (last_aper_base) { 451 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 452 453 insert_aperture_resource((u32)last_aper_base, n); 454 } 455 return; 456 } 457 458 if (!fallback_aper_force) { 459 aper_alloc = agp_aper_base; 460 aper_order = agp_aper_order; 461 } 462 463 if (aper_alloc) { 464 /* Got the aperture from the AGP bridge */ 465 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 466 force_iommu || 467 valid_agp || 468 fallback_aper_force) { 469 printk(KERN_INFO 470 "Your BIOS doesn't leave a aperture memory hole\n"); 471 printk(KERN_INFO 472 "Please enable the IOMMU option in the BIOS setup\n"); 473 printk(KERN_INFO 474 "This costs you %d MB of RAM\n", 475 32 << fallback_aper_order); 476 477 aper_order = fallback_aper_order; 478 aper_alloc = allocate_aperture(); 479 if (!aper_alloc) { 480 /* 481 * Could disable AGP and IOMMU here, but it's 482 * probably not worth it. But the later users 483 * cannot deal with bad apertures and turning 484 * on the aperture over memory causes very 485 * strange problems, so it's better to panic 486 * early. 487 */ 488 panic("Not enough memory for aperture"); 489 } 490 } else { 491 return; 492 } 493 494 /* Fix up the north bridges */ 495 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 496 int bus; 497 int dev_base, dev_limit; 498 499 bus = bus_dev_ranges[i].bus; 500 dev_base = bus_dev_ranges[i].dev_base; 501 dev_limit = bus_dev_ranges[i].dev_limit; 502 for (slot = dev_base; slot < dev_limit; slot++) { 503 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 504 continue; 505 506 /* Don't enable translation yet. That is done later. 507 Assume this BIOS didn't initialise the GART so 508 just overwrite all previous bits */ 509 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1); 510 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 511 } 512 } 513 514 set_up_gart_resume(aper_order, aper_alloc); 515 } 516