1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/bootmem.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/ioport.h> 22 #include <linux/suspend.h> 23 #include <linux/kmemleak.h> 24 #include <asm/e820.h> 25 #include <asm/io.h> 26 #include <asm/iommu.h> 27 #include <asm/gart.h> 28 #include <asm/pci-direct.h> 29 #include <asm/dma.h> 30 #include <asm/k8.h> 31 #include <asm/x86_init.h> 32 33 int gart_iommu_aperture; 34 int gart_iommu_aperture_disabled __initdata; 35 int gart_iommu_aperture_allowed __initdata; 36 37 int fallback_aper_order __initdata = 1; /* 64MB */ 38 int fallback_aper_force __initdata; 39 40 int fix_aperture __initdata = 1; 41 42 struct bus_dev_range { 43 int bus; 44 int dev_base; 45 int dev_limit; 46 }; 47 48 static struct bus_dev_range bus_dev_ranges[] __initdata = { 49 { 0x00, 0x18, 0x20}, 50 { 0xff, 0x00, 0x20}, 51 { 0xfe, 0x00, 0x20} 52 }; 53 54 static struct resource gart_resource = { 55 .name = "GART", 56 .flags = IORESOURCE_MEM, 57 }; 58 59 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 60 { 61 gart_resource.start = aper_base; 62 gart_resource.end = aper_base + aper_size - 1; 63 insert_resource(&iomem_resource, &gart_resource); 64 } 65 66 /* This code runs before the PCI subsystem is initialized, so just 67 access the northbridge directly. */ 68 69 static u32 __init allocate_aperture(void) 70 { 71 u32 aper_size; 72 void *p; 73 74 /* aper_size should <= 1G */ 75 if (fallback_aper_order > 5) 76 fallback_aper_order = 5; 77 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 78 79 /* 80 * Aperture has to be naturally aligned. This means a 2GB aperture 81 * won't have much chance of finding a place in the lower 4GB of 82 * memory. Unfortunately we cannot move it up because that would 83 * make the IOMMU useless. 84 */ 85 /* 86 * using 512M as goal, in case kexec will load kernel_big 87 * that will do the on position decompress, and could overlap with 88 * that positon with gart that is used. 89 * sequende: 90 * kernel_small 91 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 92 * ==> kernel_small(gart area become e820_reserved) 93 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 94 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 95 * so don't use 512M below as gart iommu, leave the space for kernel 96 * code for safe 97 */ 98 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20); 99 /* 100 * Kmemleak should not scan this block as it may not be mapped via the 101 * kernel direct mapping. 102 */ 103 kmemleak_ignore(p); 104 if (!p || __pa(p)+aper_size > 0xffffffff) { 105 printk(KERN_ERR 106 "Cannot allocate aperture memory hole (%p,%uK)\n", 107 p, aper_size>>10); 108 if (p) 109 free_bootmem(__pa(p), aper_size); 110 return 0; 111 } 112 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 113 aper_size >> 10, __pa(p)); 114 insert_aperture_resource((u32)__pa(p), aper_size); 115 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT, 116 (u32)__pa(p+aper_size) >> PAGE_SHIFT); 117 118 return (u32)__pa(p); 119 } 120 121 122 /* Find a PCI capability */ 123 static u32 __init find_cap(int bus, int slot, int func, int cap) 124 { 125 int bytes; 126 u8 pos; 127 128 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 129 PCI_STATUS_CAP_LIST)) 130 return 0; 131 132 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 133 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 134 u8 id; 135 136 pos &= ~3; 137 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 138 if (id == 0xff) 139 break; 140 if (id == cap) 141 return pos; 142 pos = read_pci_config_byte(bus, slot, func, 143 pos+PCI_CAP_LIST_NEXT); 144 } 145 return 0; 146 } 147 148 /* Read a standard AGPv3 bridge header */ 149 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 150 { 151 u32 apsize; 152 u32 apsizereg; 153 int nbits; 154 u32 aper_low, aper_hi; 155 u64 aper; 156 u32 old_order; 157 158 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 159 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 160 if (apsizereg == 0xffffffff) { 161 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 162 return 0; 163 } 164 165 /* old_order could be the value from NB gart setting */ 166 old_order = *order; 167 168 apsize = apsizereg & 0xfff; 169 /* Some BIOS use weird encodings not in the AGPv3 table. */ 170 if (apsize & 0xff) 171 apsize |= 0xf00; 172 nbits = hweight16(apsize); 173 *order = 7 - nbits; 174 if ((int)*order < 0) /* < 32MB */ 175 *order = 0; 176 177 aper_low = read_pci_config(bus, slot, func, 0x10); 178 aper_hi = read_pci_config(bus, slot, func, 0x14); 179 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 180 181 /* 182 * On some sick chips, APSIZE is 0. It means it wants 4G 183 * so let double check that order, and lets trust AMD NB settings: 184 */ 185 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 186 aper, 32 << old_order); 187 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 188 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 189 32 << *order, apsizereg); 190 *order = old_order; 191 } 192 193 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 194 aper, 32 << *order, apsizereg); 195 196 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 197 return 0; 198 return (u32)aper; 199 } 200 201 /* 202 * Look for an AGP bridge. Windows only expects the aperture in the 203 * AGP bridge and some BIOS forget to initialize the Northbridge too. 204 * Work around this here. 205 * 206 * Do an PCI bus scan by hand because we're running before the PCI 207 * subsystem. 208 * 209 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan 210 * generically. It's probably overkill to always scan all slots because 211 * the AGP bridges should be always an own bus on the HT hierarchy, 212 * but do it here for future safety. 213 */ 214 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 215 { 216 int bus, slot, func; 217 218 /* Poor man's PCI discovery */ 219 for (bus = 0; bus < 256; bus++) { 220 for (slot = 0; slot < 32; slot++) { 221 for (func = 0; func < 8; func++) { 222 u32 class, cap; 223 u8 type; 224 class = read_pci_config(bus, slot, func, 225 PCI_CLASS_REVISION); 226 if (class == 0xffffffff) 227 break; 228 229 switch (class >> 16) { 230 case PCI_CLASS_BRIDGE_HOST: 231 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 232 /* AGP bridge? */ 233 cap = find_cap(bus, slot, func, 234 PCI_CAP_ID_AGP); 235 if (!cap) 236 break; 237 *valid_agp = 1; 238 return read_agp(bus, slot, func, cap, 239 order); 240 } 241 242 /* No multi-function device? */ 243 type = read_pci_config_byte(bus, slot, func, 244 PCI_HEADER_TYPE); 245 if (!(type & 0x80)) 246 break; 247 } 248 } 249 } 250 printk(KERN_INFO "No AGP bridge found\n"); 251 252 return 0; 253 } 254 255 static int gart_fix_e820 __initdata = 1; 256 257 static int __init parse_gart_mem(char *p) 258 { 259 if (!p) 260 return -EINVAL; 261 262 if (!strncmp(p, "off", 3)) 263 gart_fix_e820 = 0; 264 else if (!strncmp(p, "on", 2)) 265 gart_fix_e820 = 1; 266 267 return 0; 268 } 269 early_param("gart_fix_e820", parse_gart_mem); 270 271 void __init early_gart_iommu_check(void) 272 { 273 /* 274 * in case it is enabled before, esp for kexec/kdump, 275 * previous kernel already enable that. memset called 276 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 277 * or second kernel have different position for GART hole. and new 278 * kernel could use hole as RAM that is still used by GART set by 279 * first kernel 280 * or BIOS forget to put that in reserved. 281 * try to update e820 to make that region as reserved. 282 */ 283 int i, fix, slot; 284 u32 ctl; 285 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 286 u64 aper_base = 0, last_aper_base = 0; 287 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 288 289 if (!early_pci_allowed()) 290 return; 291 292 /* This is mostly duplicate of iommu_hole_init */ 293 fix = 0; 294 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 295 int bus; 296 int dev_base, dev_limit; 297 298 bus = bus_dev_ranges[i].bus; 299 dev_base = bus_dev_ranges[i].dev_base; 300 dev_limit = bus_dev_ranges[i].dev_limit; 301 302 for (slot = dev_base; slot < dev_limit; slot++) { 303 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 304 continue; 305 306 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 307 aper_enabled = ctl & AMD64_GARTEN; 308 aper_order = (ctl >> 1) & 7; 309 aper_size = (32 * 1024 * 1024) << aper_order; 310 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 311 aper_base <<= 25; 312 313 if (last_valid) { 314 if ((aper_order != last_aper_order) || 315 (aper_base != last_aper_base) || 316 (aper_enabled != last_aper_enabled)) { 317 fix = 1; 318 break; 319 } 320 } 321 322 last_aper_order = aper_order; 323 last_aper_base = aper_base; 324 last_aper_enabled = aper_enabled; 325 last_valid = 1; 326 } 327 } 328 329 if (!fix && !aper_enabled) 330 return; 331 332 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 333 fix = 1; 334 335 if (gart_fix_e820 && !fix && aper_enabled) { 336 if (e820_any_mapped(aper_base, aper_base + aper_size, 337 E820_RAM)) { 338 /* reserve it, so we can reuse it in second kernel */ 339 printk(KERN_INFO "update e820 for GART\n"); 340 e820_add_region(aper_base, aper_size, E820_RESERVED); 341 update_e820(); 342 } 343 } 344 345 if (!fix) 346 return; 347 348 /* different nodes have different setting, disable them all at first*/ 349 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 350 int bus; 351 int dev_base, dev_limit; 352 353 bus = bus_dev_ranges[i].bus; 354 dev_base = bus_dev_ranges[i].dev_base; 355 dev_limit = bus_dev_ranges[i].dev_limit; 356 357 for (slot = dev_base; slot < dev_limit; slot++) { 358 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 359 continue; 360 361 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 362 ctl &= ~AMD64_GARTEN; 363 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 364 } 365 } 366 367 } 368 369 static int __initdata printed_gart_size_msg; 370 371 void __init gart_iommu_hole_init(void) 372 { 373 u32 agp_aper_base = 0, agp_aper_order = 0; 374 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 375 u64 aper_base, last_aper_base = 0; 376 int fix, slot, valid_agp = 0; 377 int i, node; 378 379 if (gart_iommu_aperture_disabled || !fix_aperture || 380 !early_pci_allowed()) 381 return; 382 383 printk(KERN_INFO "Checking aperture...\n"); 384 385 if (!fallback_aper_force) 386 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 387 388 fix = 0; 389 node = 0; 390 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 391 int bus; 392 int dev_base, dev_limit; 393 394 bus = bus_dev_ranges[i].bus; 395 dev_base = bus_dev_ranges[i].dev_base; 396 dev_limit = bus_dev_ranges[i].dev_limit; 397 398 for (slot = dev_base; slot < dev_limit; slot++) { 399 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 400 continue; 401 402 iommu_detected = 1; 403 gart_iommu_aperture = 1; 404 x86_init.iommu.iommu_init = gart_iommu_init; 405 406 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7; 407 aper_size = (32 * 1024 * 1024) << aper_order; 408 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 409 aper_base <<= 25; 410 411 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 412 node, aper_base, aper_size >> 20); 413 node++; 414 415 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 416 if (valid_agp && agp_aper_base && 417 agp_aper_base == aper_base && 418 agp_aper_order == aper_order) { 419 /* the same between two setting from NB and agp */ 420 if (!no_iommu && 421 max_pfn > MAX_DMA32_PFN && 422 !printed_gart_size_msg) { 423 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 424 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 425 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 426 printed_gart_size_msg = 1; 427 } 428 } else { 429 fix = 1; 430 goto out; 431 } 432 } 433 434 if ((last_aper_order && aper_order != last_aper_order) || 435 (last_aper_base && aper_base != last_aper_base)) { 436 fix = 1; 437 goto out; 438 } 439 last_aper_order = aper_order; 440 last_aper_base = aper_base; 441 } 442 } 443 444 out: 445 if (!fix && !fallback_aper_force) { 446 if (last_aper_base) { 447 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 448 449 insert_aperture_resource((u32)last_aper_base, n); 450 } 451 return; 452 } 453 454 if (!fallback_aper_force) { 455 aper_alloc = agp_aper_base; 456 aper_order = agp_aper_order; 457 } 458 459 if (aper_alloc) { 460 /* Got the aperture from the AGP bridge */ 461 } else if (!valid_agp) { 462 /* Do nothing */ 463 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 464 force_iommu || 465 valid_agp || 466 fallback_aper_force) { 467 printk(KERN_INFO 468 "Your BIOS doesn't leave a aperture memory hole\n"); 469 printk(KERN_INFO 470 "Please enable the IOMMU option in the BIOS setup\n"); 471 printk(KERN_INFO 472 "This costs you %d MB of RAM\n", 473 32 << fallback_aper_order); 474 475 aper_order = fallback_aper_order; 476 aper_alloc = allocate_aperture(); 477 if (!aper_alloc) { 478 /* 479 * Could disable AGP and IOMMU here, but it's 480 * probably not worth it. But the later users 481 * cannot deal with bad apertures and turning 482 * on the aperture over memory causes very 483 * strange problems, so it's better to panic 484 * early. 485 */ 486 panic("Not enough memory for aperture"); 487 } 488 } else { 489 return; 490 } 491 492 /* Fix up the north bridges */ 493 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 494 int bus; 495 int dev_base, dev_limit; 496 497 bus = bus_dev_ranges[i].bus; 498 dev_base = bus_dev_ranges[i].dev_base; 499 dev_limit = bus_dev_ranges[i].dev_limit; 500 for (slot = dev_base; slot < dev_limit; slot++) { 501 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 502 continue; 503 504 /* Don't enable translation yet. That is done later. 505 Assume this BIOS didn't initialise the GART so 506 just overwrite all previous bits */ 507 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1); 508 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 509 } 510 } 511 512 set_up_gart_resume(aper_order, aper_alloc); 513 } 514