1f51b7662SDaniel Vetter /* 2f51b7662SDaniel Vetter * Intel GTT (Graphics Translation Table) routines 3f51b7662SDaniel Vetter * 4f51b7662SDaniel Vetter * Caveat: This driver implements the linux agp interface, but this is far from 5f51b7662SDaniel Vetter * a agp driver! GTT support ended up here for purely historical reasons: The 6f51b7662SDaniel Vetter * old userspace intel graphics drivers needed an interface to map memory into 7f51b7662SDaniel Vetter * the GTT. And the drm provides a default interface for graphic devices sitting 8f51b7662SDaniel Vetter * on an agp port. So it made sense to fake the GTT support as an agp port to 9f51b7662SDaniel Vetter * avoid having to create a new api. 10f51b7662SDaniel Vetter * 11f51b7662SDaniel Vetter * With gem this does not make much sense anymore, just needlessly complicates 12f51b7662SDaniel Vetter * the code. But as long as the old graphics stack is still support, it's stuck 13f51b7662SDaniel Vetter * here. 14f51b7662SDaniel Vetter * 15f51b7662SDaniel Vetter * /fairy-tale-mode off 16f51b7662SDaniel Vetter */ 17f51b7662SDaniel Vetter 18e2404e7cSDaniel Vetter #include <linux/module.h> 19e2404e7cSDaniel Vetter #include <linux/pci.h> 20e2404e7cSDaniel Vetter #include <linux/init.h> 21e2404e7cSDaniel Vetter #include <linux/kernel.h> 22e2404e7cSDaniel Vetter #include <linux/pagemap.h> 23e2404e7cSDaniel Vetter #include <linux/agp_backend.h> 24bdb8b975SChris Wilson #include <linux/delay.h> 25e2404e7cSDaniel Vetter #include <asm/smp.h> 26e2404e7cSDaniel Vetter #include "agp.h" 27e2404e7cSDaniel Vetter #include "intel-agp.h" 280ade6386SDaniel Vetter #include <drm/intel-gtt.h> 29e2404e7cSDaniel Vetter 30f51b7662SDaniel Vetter /* 31f51b7662SDaniel Vetter * If we have Intel graphics, we're not going to have anything other than 32f51b7662SDaniel Vetter * an Intel IOMMU. So make the correct use of the PCI DMA API contingent 33d3f13810SSuresh Siddha * on the Intel IOMMU support (CONFIG_INTEL_IOMMU). 34f51b7662SDaniel Vetter * Only newer chipsets need to bother with this, of course. 35f51b7662SDaniel Vetter */ 36d3f13810SSuresh Siddha #ifdef CONFIG_INTEL_IOMMU 37f51b7662SDaniel Vetter #define USE_PCI_DMA_API 1 380e87d2b0SDaniel Vetter #else 390e87d2b0SDaniel Vetter #define USE_PCI_DMA_API 0 40f51b7662SDaniel Vetter #endif 41f51b7662SDaniel Vetter 421a997ff2SDaniel Vetter struct intel_gtt_driver { 431a997ff2SDaniel Vetter unsigned int gen : 8; 441a997ff2SDaniel Vetter unsigned int is_g33 : 1; 451a997ff2SDaniel Vetter unsigned int is_pineview : 1; 461a997ff2SDaniel Vetter unsigned int is_ironlake : 1; 47100519e2SChris Wilson unsigned int has_pgtbl_enable : 1; 4822533b49SDaniel Vetter unsigned int dma_mask_size : 8; 4973800422SDaniel Vetter /* Chipset specific GTT setup */ 5073800422SDaniel Vetter int (*setup)(void); 51ae83dd5cSDaniel Vetter /* This should undo anything done in ->setup() save the unmapping 52ae83dd5cSDaniel Vetter * of the mmio register file, that's done in the generic code. */ 53ae83dd5cSDaniel Vetter void (*cleanup)(void); 54351bb278SDaniel Vetter void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags); 55351bb278SDaniel Vetter /* Flags is a more or less chipset specific opaque value. 56351bb278SDaniel Vetter * For chipsets that need to support old ums (non-gem) code, this 57351bb278SDaniel Vetter * needs to be identical to the various supported agp memory types! */ 585cbecafcSDaniel Vetter bool (*check_flags)(unsigned int flags); 591b263f24SDaniel Vetter void (*chipset_flush)(void); 601a997ff2SDaniel Vetter }; 611a997ff2SDaniel Vetter 62f51b7662SDaniel Vetter static struct _intel_private { 631a997ff2SDaniel Vetter const struct intel_gtt_driver *driver; 64f51b7662SDaniel Vetter struct pci_dev *pcidev; /* device one */ 65d7cca2f7SDaniel Vetter struct pci_dev *bridge_dev; 66f51b7662SDaniel Vetter u8 __iomem *registers; 67f67eab66SDaniel Vetter phys_addr_t gtt_bus_addr; 68b3eafc5aSDaniel Vetter u32 PGETBL_save; 69f51b7662SDaniel Vetter u32 __iomem *gtt; /* I915G */ 70bee4a186SChris Wilson bool clear_fake_agp; /* on first access via agp, fill with scratch */ 71f51b7662SDaniel Vetter int num_dcache_entries; 72f51b7662SDaniel Vetter void __iomem *i9xx_flush_page; 73820647b9SDaniel Vetter char *i81x_gtt_table; 74f51b7662SDaniel Vetter struct resource ifp_resource; 75f51b7662SDaniel Vetter int resource_valid; 760e87d2b0SDaniel Vetter struct page *scratch_page; 779c61a32dSBen Widawsky phys_addr_t scratch_page_dma; 7814be93ddSDaniel Vetter int refcount; 798d2e6308SBen Widawsky /* Whether i915 needs to use the dmar apis or not. */ 808d2e6308SBen Widawsky unsigned int needs_dmar : 1; 81e5c65377SBen Widawsky phys_addr_t gma_bus_addr; 82*a54c0c27SBen Widawsky /* Size of memory reserved for graphics by the BIOS */ 83*a54c0c27SBen Widawsky unsigned int stolen_size; 84*a54c0c27SBen Widawsky /* Total number of gtt entries. */ 85*a54c0c27SBen Widawsky unsigned int gtt_total_entries; 86*a54c0c27SBen Widawsky /* Part of the gtt that is mappable by the cpu, for those chips where 87*a54c0c27SBen Widawsky * this is not the full gtt. */ 88*a54c0c27SBen Widawsky unsigned int gtt_mappable_entries; 89f51b7662SDaniel Vetter } intel_private; 90f51b7662SDaniel Vetter 911a997ff2SDaniel Vetter #define INTEL_GTT_GEN intel_private.driver->gen 921a997ff2SDaniel Vetter #define IS_G33 intel_private.driver->is_g33 931a997ff2SDaniel Vetter #define IS_PINEVIEW intel_private.driver->is_pineview 941a997ff2SDaniel Vetter #define IS_IRONLAKE intel_private.driver->is_ironlake 95100519e2SChris Wilson #define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable 961a997ff2SDaniel Vetter 979da3da66SChris Wilson static int intel_gtt_map_memory(struct page **pages, 989da3da66SChris Wilson unsigned int num_entries, 999da3da66SChris Wilson struct sg_table *st) 100f51b7662SDaniel Vetter { 101f51b7662SDaniel Vetter struct scatterlist *sg; 102f51b7662SDaniel Vetter int i; 103f51b7662SDaniel Vetter 1044080775bSDaniel Vetter DBG("try mapping %lu pages\n", (unsigned long)num_entries); 105f51b7662SDaniel Vetter 1069da3da66SChris Wilson if (sg_alloc_table(st, num_entries, GFP_KERNEL)) 107831cd445SChris Wilson goto err; 108f51b7662SDaniel Vetter 1099da3da66SChris Wilson for_each_sg(st->sgl, sg, num_entries, i) 1104080775bSDaniel Vetter sg_set_page(sg, pages[i], PAGE_SIZE, 0); 111f51b7662SDaniel Vetter 1129da3da66SChris Wilson if (!pci_map_sg(intel_private.pcidev, 1139da3da66SChris Wilson st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL)) 114831cd445SChris Wilson goto err; 115831cd445SChris Wilson 116f51b7662SDaniel Vetter return 0; 117831cd445SChris Wilson 118831cd445SChris Wilson err: 1199da3da66SChris Wilson sg_free_table(st); 120831cd445SChris Wilson return -ENOMEM; 121f51b7662SDaniel Vetter } 122f51b7662SDaniel Vetter 1239da3da66SChris Wilson static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg) 124f51b7662SDaniel Vetter { 1254080775bSDaniel Vetter struct sg_table st; 126f51b7662SDaniel Vetter DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); 127f51b7662SDaniel Vetter 1284080775bSDaniel Vetter pci_unmap_sg(intel_private.pcidev, sg_list, 1294080775bSDaniel Vetter num_sg, PCI_DMA_BIDIRECTIONAL); 1304080775bSDaniel Vetter 1314080775bSDaniel Vetter st.sgl = sg_list; 1324080775bSDaniel Vetter st.orig_nents = st.nents = num_sg; 1334080775bSDaniel Vetter 1344080775bSDaniel Vetter sg_free_table(&st); 135f51b7662SDaniel Vetter } 136f51b7662SDaniel Vetter 137ffdd7510SDaniel Vetter static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) 138f51b7662SDaniel Vetter { 139f51b7662SDaniel Vetter return; 140f51b7662SDaniel Vetter } 141f51b7662SDaniel Vetter 142f51b7662SDaniel Vetter /* Exists to support ARGB cursors */ 143f51b7662SDaniel Vetter static struct page *i8xx_alloc_pages(void) 144f51b7662SDaniel Vetter { 145f51b7662SDaniel Vetter struct page *page; 146f51b7662SDaniel Vetter 147f51b7662SDaniel Vetter page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2); 148f51b7662SDaniel Vetter if (page == NULL) 149f51b7662SDaniel Vetter return NULL; 150f51b7662SDaniel Vetter 151f51b7662SDaniel Vetter if (set_pages_uc(page, 4) < 0) { 152f51b7662SDaniel Vetter set_pages_wb(page, 4); 153f51b7662SDaniel Vetter __free_pages(page, 2); 154f51b7662SDaniel Vetter return NULL; 155f51b7662SDaniel Vetter } 156f51b7662SDaniel Vetter get_page(page); 157f51b7662SDaniel Vetter atomic_inc(&agp_bridge->current_memory_agp); 158f51b7662SDaniel Vetter return page; 159f51b7662SDaniel Vetter } 160f51b7662SDaniel Vetter 161f51b7662SDaniel Vetter static void i8xx_destroy_pages(struct page *page) 162f51b7662SDaniel Vetter { 163f51b7662SDaniel Vetter if (page == NULL) 164f51b7662SDaniel Vetter return; 165f51b7662SDaniel Vetter 166f51b7662SDaniel Vetter set_pages_wb(page, 4); 167f51b7662SDaniel Vetter put_page(page); 168f51b7662SDaniel Vetter __free_pages(page, 2); 169f51b7662SDaniel Vetter atomic_dec(&agp_bridge->current_memory_agp); 170f51b7662SDaniel Vetter } 171f51b7662SDaniel Vetter 172820647b9SDaniel Vetter #define I810_GTT_ORDER 4 173820647b9SDaniel Vetter static int i810_setup(void) 174820647b9SDaniel Vetter { 175820647b9SDaniel Vetter u32 reg_addr; 176820647b9SDaniel Vetter char *gtt_table; 177820647b9SDaniel Vetter 178820647b9SDaniel Vetter /* i81x does not preallocate the gtt. It's always 64kb in size. */ 179820647b9SDaniel Vetter gtt_table = alloc_gatt_pages(I810_GTT_ORDER); 180820647b9SDaniel Vetter if (gtt_table == NULL) 181820647b9SDaniel Vetter return -ENOMEM; 182820647b9SDaniel Vetter intel_private.i81x_gtt_table = gtt_table; 183820647b9SDaniel Vetter 184820647b9SDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_MMADDR, ®_addr); 185820647b9SDaniel Vetter reg_addr &= 0xfff80000; 186820647b9SDaniel Vetter 187820647b9SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 188820647b9SDaniel Vetter if (!intel_private.registers) 189820647b9SDaniel Vetter return -ENOMEM; 190820647b9SDaniel Vetter 191820647b9SDaniel Vetter writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED, 192820647b9SDaniel Vetter intel_private.registers+I810_PGETBL_CTL); 193820647b9SDaniel Vetter 194820647b9SDaniel Vetter intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 195820647b9SDaniel Vetter 196820647b9SDaniel Vetter if ((readl(intel_private.registers+I810_DRAM_CTL) 197820647b9SDaniel Vetter & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) { 198820647b9SDaniel Vetter dev_info(&intel_private.pcidev->dev, 199820647b9SDaniel Vetter "detected 4MB dedicated video ram\n"); 200820647b9SDaniel Vetter intel_private.num_dcache_entries = 1024; 201820647b9SDaniel Vetter } 202820647b9SDaniel Vetter 203820647b9SDaniel Vetter return 0; 204820647b9SDaniel Vetter } 205820647b9SDaniel Vetter 206820647b9SDaniel Vetter static void i810_cleanup(void) 207820647b9SDaniel Vetter { 208820647b9SDaniel Vetter writel(0, intel_private.registers+I810_PGETBL_CTL); 209820647b9SDaniel Vetter free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER); 210820647b9SDaniel Vetter } 211820647b9SDaniel Vetter 212ff26860fSDaniel Vetter static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start, 213f51b7662SDaniel Vetter int type) 214f51b7662SDaniel Vetter { 215f51b7662SDaniel Vetter int i; 216f51b7662SDaniel Vetter 217625dd9d3SDaniel Vetter if ((pg_start + mem->page_count) 218625dd9d3SDaniel Vetter > intel_private.num_dcache_entries) 219625dd9d3SDaniel Vetter return -EINVAL; 220f51b7662SDaniel Vetter 221625dd9d3SDaniel Vetter if (!mem->is_flushed) 222625dd9d3SDaniel Vetter global_cache_flush(); 223625dd9d3SDaniel Vetter 224625dd9d3SDaniel Vetter for (i = pg_start; i < (pg_start + mem->page_count); i++) { 225625dd9d3SDaniel Vetter dma_addr_t addr = i << PAGE_SHIFT; 226625dd9d3SDaniel Vetter intel_private.driver->write_entry(addr, 227625dd9d3SDaniel Vetter i, type); 228f51b7662SDaniel Vetter } 229625dd9d3SDaniel Vetter readl(intel_private.gtt+i-1); 230f51b7662SDaniel Vetter 231f51b7662SDaniel Vetter return 0; 232f51b7662SDaniel Vetter } 233f51b7662SDaniel Vetter 234f51b7662SDaniel Vetter /* 235f51b7662SDaniel Vetter * The i810/i830 requires a physical address to program its mouse 236f51b7662SDaniel Vetter * pointer into hardware. 237f51b7662SDaniel Vetter * However the Xserver still writes to it through the agp aperture. 238f51b7662SDaniel Vetter */ 239f51b7662SDaniel Vetter static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type) 240f51b7662SDaniel Vetter { 241f51b7662SDaniel Vetter struct agp_memory *new; 242f51b7662SDaniel Vetter struct page *page; 243f51b7662SDaniel Vetter 244f51b7662SDaniel Vetter switch (pg_count) { 245f51b7662SDaniel Vetter case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge); 246f51b7662SDaniel Vetter break; 247f51b7662SDaniel Vetter case 4: 248f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 249f51b7662SDaniel Vetter page = i8xx_alloc_pages(); 250f51b7662SDaniel Vetter break; 251f51b7662SDaniel Vetter default: 252f51b7662SDaniel Vetter return NULL; 253f51b7662SDaniel Vetter } 254f51b7662SDaniel Vetter 255f51b7662SDaniel Vetter if (page == NULL) 256f51b7662SDaniel Vetter return NULL; 257f51b7662SDaniel Vetter 258f51b7662SDaniel Vetter new = agp_create_memory(pg_count); 259f51b7662SDaniel Vetter if (new == NULL) 260f51b7662SDaniel Vetter return NULL; 261f51b7662SDaniel Vetter 262f51b7662SDaniel Vetter new->pages[0] = page; 263f51b7662SDaniel Vetter if (pg_count == 4) { 264f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 265f51b7662SDaniel Vetter new->pages[1] = new->pages[0] + 1; 266f51b7662SDaniel Vetter new->pages[2] = new->pages[1] + 1; 267f51b7662SDaniel Vetter new->pages[3] = new->pages[2] + 1; 268f51b7662SDaniel Vetter } 269f51b7662SDaniel Vetter new->page_count = pg_count; 270f51b7662SDaniel Vetter new->num_scratch_pages = pg_count; 271f51b7662SDaniel Vetter new->type = AGP_PHYS_MEMORY; 272f51b7662SDaniel Vetter new->physical = page_to_phys(new->pages[0]); 273f51b7662SDaniel Vetter return new; 274f51b7662SDaniel Vetter } 275f51b7662SDaniel Vetter 276f51b7662SDaniel Vetter static void intel_i810_free_by_type(struct agp_memory *curr) 277f51b7662SDaniel Vetter { 278f51b7662SDaniel Vetter agp_free_key(curr->key); 279f51b7662SDaniel Vetter if (curr->type == AGP_PHYS_MEMORY) { 280f51b7662SDaniel Vetter if (curr->page_count == 4) 281f51b7662SDaniel Vetter i8xx_destroy_pages(curr->pages[0]); 282f51b7662SDaniel Vetter else { 283f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 284f51b7662SDaniel Vetter AGP_PAGE_DESTROY_UNMAP); 285f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 286f51b7662SDaniel Vetter AGP_PAGE_DESTROY_FREE); 287f51b7662SDaniel Vetter } 288f51b7662SDaniel Vetter agp_free_page_array(curr); 289f51b7662SDaniel Vetter } 290f51b7662SDaniel Vetter kfree(curr); 291f51b7662SDaniel Vetter } 292f51b7662SDaniel Vetter 2930e87d2b0SDaniel Vetter static int intel_gtt_setup_scratch_page(void) 2940e87d2b0SDaniel Vetter { 2950e87d2b0SDaniel Vetter struct page *page; 2960e87d2b0SDaniel Vetter dma_addr_t dma_addr; 2970e87d2b0SDaniel Vetter 2980e87d2b0SDaniel Vetter page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); 2990e87d2b0SDaniel Vetter if (page == NULL) 3000e87d2b0SDaniel Vetter return -ENOMEM; 3010e87d2b0SDaniel Vetter get_page(page); 3020e87d2b0SDaniel Vetter set_pages_uc(page, 1); 3030e87d2b0SDaniel Vetter 3048d2e6308SBen Widawsky if (intel_private.needs_dmar) { 3050e87d2b0SDaniel Vetter dma_addr = pci_map_page(intel_private.pcidev, page, 0, 3060e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 3070e87d2b0SDaniel Vetter if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) 3080e87d2b0SDaniel Vetter return -EINVAL; 3090e87d2b0SDaniel Vetter 3109c61a32dSBen Widawsky intel_private.scratch_page_dma = dma_addr; 3110e87d2b0SDaniel Vetter } else 3129c61a32dSBen Widawsky intel_private.scratch_page_dma = page_to_phys(page); 3130e87d2b0SDaniel Vetter 3140e87d2b0SDaniel Vetter intel_private.scratch_page = page; 3150e87d2b0SDaniel Vetter 3160e87d2b0SDaniel Vetter return 0; 3170e87d2b0SDaniel Vetter } 3180e87d2b0SDaniel Vetter 319625dd9d3SDaniel Vetter static void i810_write_entry(dma_addr_t addr, unsigned int entry, 320625dd9d3SDaniel Vetter unsigned int flags) 321625dd9d3SDaniel Vetter { 322625dd9d3SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 323625dd9d3SDaniel Vetter 324625dd9d3SDaniel Vetter switch (flags) { 325625dd9d3SDaniel Vetter case AGP_DCACHE_MEMORY: 326625dd9d3SDaniel Vetter pte_flags |= I810_PTE_LOCAL; 327625dd9d3SDaniel Vetter break; 328625dd9d3SDaniel Vetter case AGP_USER_CACHED_MEMORY: 329625dd9d3SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 330625dd9d3SDaniel Vetter break; 331625dd9d3SDaniel Vetter } 332625dd9d3SDaniel Vetter 333625dd9d3SDaniel Vetter writel(addr | pte_flags, intel_private.gtt + entry); 334625dd9d3SDaniel Vetter } 335625dd9d3SDaniel Vetter 3367bdc9ab0SChris Wilson static const struct aper_size_info_fixed intel_fake_agp_sizes[] = { 337820647b9SDaniel Vetter {32, 8192, 3}, 338820647b9SDaniel Vetter {64, 16384, 4}, 339f51b7662SDaniel Vetter {128, 32768, 5}, 340f51b7662SDaniel Vetter {256, 65536, 6}, 341f51b7662SDaniel Vetter {512, 131072, 7}, 342f51b7662SDaniel Vetter }; 343f51b7662SDaniel Vetter 344c64f7ba5SChris Wilson static unsigned int intel_gtt_stolen_size(void) 345f51b7662SDaniel Vetter { 346f51b7662SDaniel Vetter u16 gmch_ctrl; 347f51b7662SDaniel Vetter u8 rdct; 348f51b7662SDaniel Vetter int local = 0; 349f51b7662SDaniel Vetter static const int ddt[4] = { 0, 16, 32, 64 }; 350d8d9abcdSDaniel Vetter unsigned int stolen_size = 0; 351f51b7662SDaniel Vetter 352820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) 353820647b9SDaniel Vetter return 0; /* no stolen mem on i81x */ 354820647b9SDaniel Vetter 355d7cca2f7SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 356d7cca2f7SDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 357f51b7662SDaniel Vetter 358d7cca2f7SDaniel Vetter if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || 359d7cca2f7SDaniel Vetter intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { 360f51b7662SDaniel Vetter switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 361f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_512: 362d8d9abcdSDaniel Vetter stolen_size = KB(512); 363f51b7662SDaniel Vetter break; 364f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_1024: 365d8d9abcdSDaniel Vetter stolen_size = MB(1); 366f51b7662SDaniel Vetter break; 367f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_8192: 368d8d9abcdSDaniel Vetter stolen_size = MB(8); 369f51b7662SDaniel Vetter break; 370f51b7662SDaniel Vetter case I830_GMCH_GMS_LOCAL: 371f51b7662SDaniel Vetter rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE); 372d8d9abcdSDaniel Vetter stolen_size = (I830_RDRAM_ND(rdct) + 1) * 373f51b7662SDaniel Vetter MB(ddt[I830_RDRAM_DDT(rdct)]); 374f51b7662SDaniel Vetter local = 1; 375f51b7662SDaniel Vetter break; 376f51b7662SDaniel Vetter default: 377d8d9abcdSDaniel Vetter stolen_size = 0; 378f51b7662SDaniel Vetter break; 379f51b7662SDaniel Vetter } 380f51b7662SDaniel Vetter } else { 381f51b7662SDaniel Vetter switch (gmch_ctrl & I855_GMCH_GMS_MASK) { 382f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_1M: 383d8d9abcdSDaniel Vetter stolen_size = MB(1); 384f51b7662SDaniel Vetter break; 385f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_4M: 386d8d9abcdSDaniel Vetter stolen_size = MB(4); 387f51b7662SDaniel Vetter break; 388f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_8M: 389d8d9abcdSDaniel Vetter stolen_size = MB(8); 390f51b7662SDaniel Vetter break; 391f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_16M: 392d8d9abcdSDaniel Vetter stolen_size = MB(16); 393f51b7662SDaniel Vetter break; 394f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_32M: 395d8d9abcdSDaniel Vetter stolen_size = MB(32); 396f51b7662SDaniel Vetter break; 397f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_48M: 398d8d9abcdSDaniel Vetter stolen_size = MB(48); 399f51b7662SDaniel Vetter break; 400f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_64M: 401d8d9abcdSDaniel Vetter stolen_size = MB(64); 402f51b7662SDaniel Vetter break; 403f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_128M: 404d8d9abcdSDaniel Vetter stolen_size = MB(128); 405f51b7662SDaniel Vetter break; 406f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_256M: 407d8d9abcdSDaniel Vetter stolen_size = MB(256); 408f51b7662SDaniel Vetter break; 409f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_96M: 410d8d9abcdSDaniel Vetter stolen_size = MB(96); 411f51b7662SDaniel Vetter break; 412f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_160M: 413d8d9abcdSDaniel Vetter stolen_size = MB(160); 414f51b7662SDaniel Vetter break; 415f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_224M: 416d8d9abcdSDaniel Vetter stolen_size = MB(224); 417f51b7662SDaniel Vetter break; 418f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_352M: 419d8d9abcdSDaniel Vetter stolen_size = MB(352); 420f51b7662SDaniel Vetter break; 421f51b7662SDaniel Vetter default: 422d8d9abcdSDaniel Vetter stolen_size = 0; 423f51b7662SDaniel Vetter break; 424f51b7662SDaniel Vetter } 425f51b7662SDaniel Vetter } 4261784a5fbSDaniel Vetter 4271b6064d7SChris Wilson if (stolen_size > 0) { 428d7cca2f7SDaniel Vetter dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n", 429d8d9abcdSDaniel Vetter stolen_size / KB(1), local ? "local" : "stolen"); 430f51b7662SDaniel Vetter } else { 431d7cca2f7SDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 432f51b7662SDaniel Vetter "no pre-allocated video memory detected\n"); 433d8d9abcdSDaniel Vetter stolen_size = 0; 434f51b7662SDaniel Vetter } 435f51b7662SDaniel Vetter 436c64f7ba5SChris Wilson return stolen_size; 437f51b7662SDaniel Vetter } 438f51b7662SDaniel Vetter 43920172842SDaniel Vetter static void i965_adjust_pgetbl_size(unsigned int size_flag) 44020172842SDaniel Vetter { 44120172842SDaniel Vetter u32 pgetbl_ctl, pgetbl_ctl2; 44220172842SDaniel Vetter 44320172842SDaniel Vetter /* ensure that ppgtt is disabled */ 44420172842SDaniel Vetter pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2); 44520172842SDaniel Vetter pgetbl_ctl2 &= ~I810_PGETBL_ENABLED; 44620172842SDaniel Vetter writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2); 44720172842SDaniel Vetter 44820172842SDaniel Vetter /* write the new ggtt size */ 44920172842SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 45020172842SDaniel Vetter pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK; 45120172842SDaniel Vetter pgetbl_ctl |= size_flag; 45220172842SDaniel Vetter writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL); 45320172842SDaniel Vetter } 45420172842SDaniel Vetter 45520172842SDaniel Vetter static unsigned int i965_gtt_total_entries(void) 456fbe40783SDaniel Vetter { 457fbe40783SDaniel Vetter int size; 458fbe40783SDaniel Vetter u32 pgetbl_ctl; 45920172842SDaniel Vetter u16 gmch_ctl; 46020172842SDaniel Vetter 46120172842SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 46220172842SDaniel Vetter I830_GMCH_CTRL, &gmch_ctl); 46320172842SDaniel Vetter 46420172842SDaniel Vetter if (INTEL_GTT_GEN == 5) { 46520172842SDaniel Vetter switch (gmch_ctl & G4x_GMCH_SIZE_MASK) { 46620172842SDaniel Vetter case G4x_GMCH_SIZE_1M: 46720172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1M: 46820172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB); 46920172842SDaniel Vetter break; 47020172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1_5M: 47120172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB); 47220172842SDaniel Vetter break; 47320172842SDaniel Vetter case G4x_GMCH_SIZE_2M: 47420172842SDaniel Vetter case G4x_GMCH_SIZE_VT_2M: 47520172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB); 47620172842SDaniel Vetter break; 47720172842SDaniel Vetter } 47820172842SDaniel Vetter } 47920172842SDaniel Vetter 480fbe40783SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 481fbe40783SDaniel Vetter 482fbe40783SDaniel Vetter switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { 483fbe40783SDaniel Vetter case I965_PGETBL_SIZE_128KB: 484e5e408fcSDaniel Vetter size = KB(128); 485fbe40783SDaniel Vetter break; 486fbe40783SDaniel Vetter case I965_PGETBL_SIZE_256KB: 487e5e408fcSDaniel Vetter size = KB(256); 488fbe40783SDaniel Vetter break; 489fbe40783SDaniel Vetter case I965_PGETBL_SIZE_512KB: 490e5e408fcSDaniel Vetter size = KB(512); 491fbe40783SDaniel Vetter break; 49220172842SDaniel Vetter /* GTT pagetable sizes bigger than 512KB are not possible on G33! */ 493fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1MB: 494e5e408fcSDaniel Vetter size = KB(1024); 495fbe40783SDaniel Vetter break; 496fbe40783SDaniel Vetter case I965_PGETBL_SIZE_2MB: 497e5e408fcSDaniel Vetter size = KB(2048); 498fbe40783SDaniel Vetter break; 499fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1_5MB: 500e5e408fcSDaniel Vetter size = KB(1024 + 512); 501fbe40783SDaniel Vetter break; 502fbe40783SDaniel Vetter default: 503fbe40783SDaniel Vetter dev_info(&intel_private.pcidev->dev, 504fbe40783SDaniel Vetter "unknown page table size, assuming 512KB\n"); 505e5e408fcSDaniel Vetter size = KB(512); 506fbe40783SDaniel Vetter } 507e5e408fcSDaniel Vetter 508e5e408fcSDaniel Vetter return size/4; 50920172842SDaniel Vetter } 51020172842SDaniel Vetter 51120172842SDaniel Vetter static unsigned int intel_gtt_total_entries(void) 51220172842SDaniel Vetter { 51320172842SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) 51420172842SDaniel Vetter return i965_gtt_total_entries(); 515009946f8SBen Widawsky else { 516fbe40783SDaniel Vetter /* On previous hardware, the GTT size was just what was 517fbe40783SDaniel Vetter * required to map the aperture. 518fbe40783SDaniel Vetter */ 519*a54c0c27SBen Widawsky return intel_private.gtt_mappable_entries; 520fbe40783SDaniel Vetter } 521fbe40783SDaniel Vetter } 522fbe40783SDaniel Vetter 5231784a5fbSDaniel Vetter static unsigned int intel_gtt_mappable_entries(void) 5241784a5fbSDaniel Vetter { 5251784a5fbSDaniel Vetter unsigned int aperture_size; 5261784a5fbSDaniel Vetter 527820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) { 528820647b9SDaniel Vetter u32 smram_miscc; 529820647b9SDaniel Vetter 530820647b9SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, 531820647b9SDaniel Vetter I810_SMRAM_MISCC, &smram_miscc); 532820647b9SDaniel Vetter 533820647b9SDaniel Vetter if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) 534820647b9SDaniel Vetter == I810_GFX_MEM_WIN_32M) 535820647b9SDaniel Vetter aperture_size = MB(32); 536820647b9SDaniel Vetter else 537820647b9SDaniel Vetter aperture_size = MB(64); 538820647b9SDaniel Vetter } else if (INTEL_GTT_GEN == 2) { 539b1c5b0f8SChris Wilson u16 gmch_ctrl; 5401784a5fbSDaniel Vetter 5411784a5fbSDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 5421784a5fbSDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 5431784a5fbSDaniel Vetter 5441784a5fbSDaniel Vetter if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M) 545b1c5b0f8SChris Wilson aperture_size = MB(64); 5461784a5fbSDaniel Vetter else 547b1c5b0f8SChris Wilson aperture_size = MB(128); 548239918f7SDaniel Vetter } else { 5491784a5fbSDaniel Vetter /* 9xx supports large sizes, just look at the length */ 5501784a5fbSDaniel Vetter aperture_size = pci_resource_len(intel_private.pcidev, 2); 5511784a5fbSDaniel Vetter } 5521784a5fbSDaniel Vetter 5531784a5fbSDaniel Vetter return aperture_size >> PAGE_SHIFT; 5541784a5fbSDaniel Vetter } 5551784a5fbSDaniel Vetter 5560e87d2b0SDaniel Vetter static void intel_gtt_teardown_scratch_page(void) 5570e87d2b0SDaniel Vetter { 5580e87d2b0SDaniel Vetter set_pages_wb(intel_private.scratch_page, 1); 5599c61a32dSBen Widawsky pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma, 5600e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 5610e87d2b0SDaniel Vetter put_page(intel_private.scratch_page); 5620e87d2b0SDaniel Vetter __free_page(intel_private.scratch_page); 5630e87d2b0SDaniel Vetter } 5640e87d2b0SDaniel Vetter 5650e87d2b0SDaniel Vetter static void intel_gtt_cleanup(void) 5660e87d2b0SDaniel Vetter { 567ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 568ae83dd5cSDaniel Vetter 5690e87d2b0SDaniel Vetter iounmap(intel_private.gtt); 5700e87d2b0SDaniel Vetter iounmap(intel_private.registers); 5710e87d2b0SDaniel Vetter 5720e87d2b0SDaniel Vetter intel_gtt_teardown_scratch_page(); 5730e87d2b0SDaniel Vetter } 5740e87d2b0SDaniel Vetter 5751784a5fbSDaniel Vetter static int intel_gtt_init(void) 5761784a5fbSDaniel Vetter { 57732e3cd6eSDaniel Vetter u32 gma_addr; 578f67eab66SDaniel Vetter u32 gtt_map_size; 5793b15a9d7SDaniel Vetter int ret; 5803b15a9d7SDaniel Vetter 5813b15a9d7SDaniel Vetter ret = intel_private.driver->setup(); 5823b15a9d7SDaniel Vetter if (ret != 0) 5833b15a9d7SDaniel Vetter return ret; 584f67eab66SDaniel Vetter 585*a54c0c27SBen Widawsky intel_private.gtt_mappable_entries = intel_gtt_mappable_entries(); 586*a54c0c27SBen Widawsky intel_private.gtt_total_entries = intel_gtt_total_entries(); 587f67eab66SDaniel Vetter 588b3eafc5aSDaniel Vetter /* save the PGETBL reg for resume */ 589b3eafc5aSDaniel Vetter intel_private.PGETBL_save = 590b3eafc5aSDaniel Vetter readl(intel_private.registers+I810_PGETBL_CTL) 591b3eafc5aSDaniel Vetter & ~I810_PGETBL_ENABLED; 592100519e2SChris Wilson /* we only ever restore the register when enabling the PGTBL... */ 593100519e2SChris Wilson if (HAS_PGTBL_EN) 594100519e2SChris Wilson intel_private.PGETBL_save |= I810_PGETBL_ENABLED; 595b3eafc5aSDaniel Vetter 5960af9e92eSDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 5970af9e92eSDaniel Vetter "detected gtt size: %dK total, %dK mappable\n", 598*a54c0c27SBen Widawsky intel_private.gtt_total_entries * 4, 599*a54c0c27SBen Widawsky intel_private.gtt_mappable_entries * 4); 6000af9e92eSDaniel Vetter 601*a54c0c27SBen Widawsky gtt_map_size = intel_private.gtt_total_entries * 4; 602f67eab66SDaniel Vetter 603edef7e68SChris Wilson intel_private.gtt = NULL; 6049169d3a8SDaniel Vetter if (INTEL_GTT_GEN < 6 && INTEL_GTT_GEN > 2) 605edef7e68SChris Wilson intel_private.gtt = ioremap_wc(intel_private.gtt_bus_addr, 606edef7e68SChris Wilson gtt_map_size); 607edef7e68SChris Wilson if (intel_private.gtt == NULL) 608f67eab66SDaniel Vetter intel_private.gtt = ioremap(intel_private.gtt_bus_addr, 609f67eab66SDaniel Vetter gtt_map_size); 610edef7e68SChris Wilson if (intel_private.gtt == NULL) { 611ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 612f67eab66SDaniel Vetter iounmap(intel_private.registers); 613f67eab66SDaniel Vetter return -ENOMEM; 614f67eab66SDaniel Vetter } 615f67eab66SDaniel Vetter 616f67eab66SDaniel Vetter global_cache_flush(); /* FIXME: ? */ 617f67eab66SDaniel Vetter 618*a54c0c27SBen Widawsky intel_private.stolen_size = intel_gtt_stolen_size(); 6191784a5fbSDaniel Vetter 6208d2e6308SBen Widawsky intel_private.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2; 621a46f3108SDave Airlie 6220e87d2b0SDaniel Vetter ret = intel_gtt_setup_scratch_page(); 6230e87d2b0SDaniel Vetter if (ret != 0) { 6240e87d2b0SDaniel Vetter intel_gtt_cleanup(); 6250e87d2b0SDaniel Vetter return ret; 6260e87d2b0SDaniel Vetter } 6270e87d2b0SDaniel Vetter 62832e3cd6eSDaniel Vetter if (INTEL_GTT_GEN <= 2) 62932e3cd6eSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_GMADDR, 63032e3cd6eSDaniel Vetter &gma_addr); 63132e3cd6eSDaniel Vetter else 63232e3cd6eSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I915_GMADDR, 63332e3cd6eSDaniel Vetter &gma_addr); 63432e3cd6eSDaniel Vetter 635e5c65377SBen Widawsky intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK); 63632e3cd6eSDaniel Vetter 6371784a5fbSDaniel Vetter return 0; 6381784a5fbSDaniel Vetter } 6391784a5fbSDaniel Vetter 6403e921f98SDaniel Vetter static int intel_fake_agp_fetch_size(void) 6413e921f98SDaniel Vetter { 6429e76e7b8SChris Wilson int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes); 6433e921f98SDaniel Vetter unsigned int aper_size; 6443e921f98SDaniel Vetter int i; 6453e921f98SDaniel Vetter 646*a54c0c27SBen Widawsky aper_size = (intel_private.gtt_mappable_entries << PAGE_SHIFT) / MB(1); 6473e921f98SDaniel Vetter 6483e921f98SDaniel Vetter for (i = 0; i < num_sizes; i++) { 649ffdd7510SDaniel Vetter if (aper_size == intel_fake_agp_sizes[i].size) { 6509e76e7b8SChris Wilson agp_bridge->current_size = 6519e76e7b8SChris Wilson (void *) (intel_fake_agp_sizes + i); 6523e921f98SDaniel Vetter return aper_size; 6533e921f98SDaniel Vetter } 6543e921f98SDaniel Vetter } 6553e921f98SDaniel Vetter 6563e921f98SDaniel Vetter return 0; 6573e921f98SDaniel Vetter } 6583e921f98SDaniel Vetter 659ae83dd5cSDaniel Vetter static void i830_cleanup(void) 660f51b7662SDaniel Vetter { 661f51b7662SDaniel Vetter } 662f51b7662SDaniel Vetter 663f51b7662SDaniel Vetter /* The chipset_flush interface needs to get data that has already been 664f51b7662SDaniel Vetter * flushed out of the CPU all the way out to main memory, because the GPU 665f51b7662SDaniel Vetter * doesn't snoop those buffers. 666f51b7662SDaniel Vetter * 667f51b7662SDaniel Vetter * The 8xx series doesn't have the same lovely interface for flushing the 668f51b7662SDaniel Vetter * chipset write buffers that the later chips do. According to the 865 669f51b7662SDaniel Vetter * specs, it's 64 octwords, or 1KB. So, to get those previous things in 670f51b7662SDaniel Vetter * that buffer out, we just fill 1KB and clflush it out, on the assumption 671f51b7662SDaniel Vetter * that it'll push whatever was in there out. It appears to work. 672f51b7662SDaniel Vetter */ 6731b263f24SDaniel Vetter static void i830_chipset_flush(void) 674f51b7662SDaniel Vetter { 675bdb8b975SChris Wilson unsigned long timeout = jiffies + msecs_to_jiffies(1000); 676f51b7662SDaniel Vetter 677bdb8b975SChris Wilson /* Forcibly evict everything from the CPU write buffers. 678bdb8b975SChris Wilson * clflush appears to be insufficient. 679bdb8b975SChris Wilson */ 680bdb8b975SChris Wilson wbinvd_on_all_cpus(); 681f51b7662SDaniel Vetter 682bdb8b975SChris Wilson /* Now we've only seen documents for this magic bit on 855GM, 683bdb8b975SChris Wilson * we hope it exists for the other gen2 chipsets... 684bdb8b975SChris Wilson * 685bdb8b975SChris Wilson * Also works as advertised on my 845G. 686bdb8b975SChris Wilson */ 687bdb8b975SChris Wilson writel(readl(intel_private.registers+I830_HIC) | (1<<31), 688bdb8b975SChris Wilson intel_private.registers+I830_HIC); 689bdb8b975SChris Wilson 690bdb8b975SChris Wilson while (readl(intel_private.registers+I830_HIC) & (1<<31)) { 691bdb8b975SChris Wilson if (time_after(jiffies, timeout)) 692bdb8b975SChris Wilson break; 693bdb8b975SChris Wilson 694bdb8b975SChris Wilson udelay(50); 695bdb8b975SChris Wilson } 696f51b7662SDaniel Vetter } 697f51b7662SDaniel Vetter 698351bb278SDaniel Vetter static void i830_write_entry(dma_addr_t addr, unsigned int entry, 699351bb278SDaniel Vetter unsigned int flags) 700351bb278SDaniel Vetter { 701351bb278SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 702351bb278SDaniel Vetter 703b47cf66fSDaniel Vetter if (flags == AGP_USER_CACHED_MEMORY) 704351bb278SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 705351bb278SDaniel Vetter 706351bb278SDaniel Vetter writel(addr | pte_flags, intel_private.gtt + entry); 707351bb278SDaniel Vetter } 708351bb278SDaniel Vetter 7098ecd1a66SDaniel Vetter bool intel_enable_gtt(void) 71073800422SDaniel Vetter { 711e380f60bSChris Wilson u8 __iomem *reg; 71273800422SDaniel Vetter 713100519e2SChris Wilson if (INTEL_GTT_GEN == 2) { 714100519e2SChris Wilson u16 gmch_ctrl; 715100519e2SChris Wilson 716e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 717e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 718e380f60bSChris Wilson gmch_ctrl |= I830_GMCH_ENABLED; 719e380f60bSChris Wilson pci_write_config_word(intel_private.bridge_dev, 720e380f60bSChris Wilson I830_GMCH_CTRL, gmch_ctrl); 721e380f60bSChris Wilson 722e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 723e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 724e380f60bSChris Wilson if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) { 725e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 726e380f60bSChris Wilson "failed to enable the GTT: GMCH_CTRL=%x\n", 727e380f60bSChris Wilson gmch_ctrl); 728e380f60bSChris Wilson return false; 729e380f60bSChris Wilson } 730100519e2SChris Wilson } 731e380f60bSChris Wilson 732c97689d8SChris Wilson /* On the resume path we may be adjusting the PGTBL value, so 733c97689d8SChris Wilson * be paranoid and flush all chipset write buffers... 734c97689d8SChris Wilson */ 735c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 736c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 737c97689d8SChris Wilson 738e380f60bSChris Wilson reg = intel_private.registers+I810_PGETBL_CTL; 739100519e2SChris Wilson writel(intel_private.PGETBL_save, reg); 740100519e2SChris Wilson if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) { 741e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 742100519e2SChris Wilson "failed to enable the GTT: PGETBL=%x [expected %x]\n", 743e380f60bSChris Wilson readl(reg), intel_private.PGETBL_save); 744e380f60bSChris Wilson return false; 745e380f60bSChris Wilson } 746e380f60bSChris Wilson 747c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 748c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 749c97689d8SChris Wilson 750e380f60bSChris Wilson return true; 75173800422SDaniel Vetter } 7528ecd1a66SDaniel Vetter EXPORT_SYMBOL(intel_enable_gtt); 75373800422SDaniel Vetter 75473800422SDaniel Vetter static int i830_setup(void) 75573800422SDaniel Vetter { 75673800422SDaniel Vetter u32 reg_addr; 75773800422SDaniel Vetter 75873800422SDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_MMADDR, ®_addr); 75973800422SDaniel Vetter reg_addr &= 0xfff80000; 76073800422SDaniel Vetter 76173800422SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 76273800422SDaniel Vetter if (!intel_private.registers) 76373800422SDaniel Vetter return -ENOMEM; 76473800422SDaniel Vetter 76573800422SDaniel Vetter intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 76673800422SDaniel Vetter 76773800422SDaniel Vetter return 0; 76873800422SDaniel Vetter } 76973800422SDaniel Vetter 7703b15a9d7SDaniel Vetter static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge) 771f51b7662SDaniel Vetter { 77273800422SDaniel Vetter agp_bridge->gatt_table_real = NULL; 773f51b7662SDaniel Vetter agp_bridge->gatt_table = NULL; 77473800422SDaniel Vetter agp_bridge->gatt_bus_addr = 0; 775f51b7662SDaniel Vetter 776f51b7662SDaniel Vetter return 0; 777f51b7662SDaniel Vetter } 778f51b7662SDaniel Vetter 779ffdd7510SDaniel Vetter static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge) 780f51b7662SDaniel Vetter { 781f51b7662SDaniel Vetter return 0; 782f51b7662SDaniel Vetter } 783f51b7662SDaniel Vetter 784351bb278SDaniel Vetter static int intel_fake_agp_configure(void) 785f51b7662SDaniel Vetter { 786e380f60bSChris Wilson if (!intel_enable_gtt()) 787e380f60bSChris Wilson return -EIO; 788f51b7662SDaniel Vetter 789bee4a186SChris Wilson intel_private.clear_fake_agp = true; 790e5c65377SBen Widawsky agp_bridge->gart_bus_addr = intel_private.gma_bus_addr; 791f51b7662SDaniel Vetter 792f51b7662SDaniel Vetter return 0; 793f51b7662SDaniel Vetter } 794f51b7662SDaniel Vetter 7955cbecafcSDaniel Vetter static bool i830_check_flags(unsigned int flags) 796f51b7662SDaniel Vetter { 7975cbecafcSDaniel Vetter switch (flags) { 7985cbecafcSDaniel Vetter case 0: 7995cbecafcSDaniel Vetter case AGP_PHYS_MEMORY: 8005cbecafcSDaniel Vetter case AGP_USER_CACHED_MEMORY: 8015cbecafcSDaniel Vetter case AGP_USER_MEMORY: 8025cbecafcSDaniel Vetter return true; 8035cbecafcSDaniel Vetter } 8045cbecafcSDaniel Vetter 8055cbecafcSDaniel Vetter return false; 8065cbecafcSDaniel Vetter } 8075cbecafcSDaniel Vetter 8089da3da66SChris Wilson void intel_gtt_insert_sg_entries(struct sg_table *st, 809fefaa70fSDaniel Vetter unsigned int pg_start, 810fefaa70fSDaniel Vetter unsigned int flags) 811fefaa70fSDaniel Vetter { 812fefaa70fSDaniel Vetter struct scatterlist *sg; 813fefaa70fSDaniel Vetter unsigned int len, m; 814fefaa70fSDaniel Vetter int i, j; 815fefaa70fSDaniel Vetter 816fefaa70fSDaniel Vetter j = pg_start; 817fefaa70fSDaniel Vetter 818fefaa70fSDaniel Vetter /* sg may merge pages, but we have to separate 819fefaa70fSDaniel Vetter * per-page addr for GTT */ 8209da3da66SChris Wilson for_each_sg(st->sgl, sg, st->nents, i) { 821fefaa70fSDaniel Vetter len = sg_dma_len(sg) >> PAGE_SHIFT; 822fefaa70fSDaniel Vetter for (m = 0; m < len; m++) { 823fefaa70fSDaniel Vetter dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT); 8249da3da66SChris Wilson intel_private.driver->write_entry(addr, j, flags); 825fefaa70fSDaniel Vetter j++; 826fefaa70fSDaniel Vetter } 827fefaa70fSDaniel Vetter } 828fefaa70fSDaniel Vetter readl(intel_private.gtt+j-1); 829fefaa70fSDaniel Vetter } 8304080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_insert_sg_entries); 8314080775bSDaniel Vetter 8329da3da66SChris Wilson static void intel_gtt_insert_pages(unsigned int first_entry, 8339da3da66SChris Wilson unsigned int num_entries, 8349da3da66SChris Wilson struct page **pages, 8359da3da66SChris Wilson unsigned int flags) 8364080775bSDaniel Vetter { 8374080775bSDaniel Vetter int i, j; 8384080775bSDaniel Vetter 8394080775bSDaniel Vetter for (i = 0, j = first_entry; i < num_entries; i++, j++) { 8404080775bSDaniel Vetter dma_addr_t addr = page_to_phys(pages[i]); 8414080775bSDaniel Vetter intel_private.driver->write_entry(addr, 8424080775bSDaniel Vetter j, flags); 8434080775bSDaniel Vetter } 8444080775bSDaniel Vetter readl(intel_private.gtt+j-1); 8454080775bSDaniel Vetter } 846fefaa70fSDaniel Vetter 8475cbecafcSDaniel Vetter static int intel_fake_agp_insert_entries(struct agp_memory *mem, 8485cbecafcSDaniel Vetter off_t pg_start, int type) 8495cbecafcSDaniel Vetter { 850f51b7662SDaniel Vetter int ret = -EINVAL; 851f51b7662SDaniel Vetter 852bee4a186SChris Wilson if (intel_private.clear_fake_agp) { 853*a54c0c27SBen Widawsky int start = intel_private.stolen_size / PAGE_SIZE; 854*a54c0c27SBen Widawsky int end = intel_private.gtt_mappable_entries; 855bee4a186SChris Wilson intel_gtt_clear_range(start, end - start); 856bee4a186SChris Wilson intel_private.clear_fake_agp = false; 857bee4a186SChris Wilson } 858bee4a186SChris Wilson 859ff26860fSDaniel Vetter if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY) 860ff26860fSDaniel Vetter return i810_insert_dcache_entries(mem, pg_start, type); 861ff26860fSDaniel Vetter 862f51b7662SDaniel Vetter if (mem->page_count == 0) 863f51b7662SDaniel Vetter goto out; 864f51b7662SDaniel Vetter 865*a54c0c27SBen Widawsky if (pg_start + mem->page_count > intel_private.gtt_total_entries) 866f51b7662SDaniel Vetter goto out_err; 867f51b7662SDaniel Vetter 868f51b7662SDaniel Vetter if (type != mem->type) 869f51b7662SDaniel Vetter goto out_err; 870f51b7662SDaniel Vetter 8715cbecafcSDaniel Vetter if (!intel_private.driver->check_flags(type)) 872f51b7662SDaniel Vetter goto out_err; 873f51b7662SDaniel Vetter 874f51b7662SDaniel Vetter if (!mem->is_flushed) 875f51b7662SDaniel Vetter global_cache_flush(); 876f51b7662SDaniel Vetter 8778d2e6308SBen Widawsky if (intel_private.needs_dmar) { 8789da3da66SChris Wilson struct sg_table st; 8799da3da66SChris Wilson 8809da3da66SChris Wilson ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st); 881fefaa70fSDaniel Vetter if (ret != 0) 882fefaa70fSDaniel Vetter return ret; 883fefaa70fSDaniel Vetter 8849da3da66SChris Wilson intel_gtt_insert_sg_entries(&st, pg_start, type); 8859da3da66SChris Wilson mem->sg_list = st.sgl; 8869da3da66SChris Wilson mem->num_sg = st.nents; 8874080775bSDaniel Vetter } else 8884080775bSDaniel Vetter intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages, 8894080775bSDaniel Vetter type); 890f51b7662SDaniel Vetter 891f51b7662SDaniel Vetter out: 892f51b7662SDaniel Vetter ret = 0; 893f51b7662SDaniel Vetter out_err: 894f51b7662SDaniel Vetter mem->is_flushed = true; 895f51b7662SDaniel Vetter return ret; 896f51b7662SDaniel Vetter } 897f51b7662SDaniel Vetter 8984080775bSDaniel Vetter void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries) 899f51b7662SDaniel Vetter { 9004080775bSDaniel Vetter unsigned int i; 901f51b7662SDaniel Vetter 9024080775bSDaniel Vetter for (i = first_entry; i < (first_entry + num_entries); i++) { 9039c61a32dSBen Widawsky intel_private.driver->write_entry(intel_private.scratch_page_dma, 9045cbecafcSDaniel Vetter i, 0); 905f51b7662SDaniel Vetter } 906fdfb58a9SDaniel Vetter readl(intel_private.gtt+i-1); 9074080775bSDaniel Vetter } 9084080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_clear_range); 9094080775bSDaniel Vetter 9104080775bSDaniel Vetter static int intel_fake_agp_remove_entries(struct agp_memory *mem, 9114080775bSDaniel Vetter off_t pg_start, int type) 9124080775bSDaniel Vetter { 9134080775bSDaniel Vetter if (mem->page_count == 0) 9144080775bSDaniel Vetter return 0; 9154080775bSDaniel Vetter 916d15eda5cSDave Airlie intel_gtt_clear_range(pg_start, mem->page_count); 917d15eda5cSDave Airlie 9188d2e6308SBen Widawsky if (intel_private.needs_dmar) { 9194080775bSDaniel Vetter intel_gtt_unmap_memory(mem->sg_list, mem->num_sg); 9204080775bSDaniel Vetter mem->sg_list = NULL; 9214080775bSDaniel Vetter mem->num_sg = 0; 9224080775bSDaniel Vetter } 9234080775bSDaniel Vetter 924f51b7662SDaniel Vetter return 0; 925f51b7662SDaniel Vetter } 926f51b7662SDaniel Vetter 927ffdd7510SDaniel Vetter static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, 928ffdd7510SDaniel Vetter int type) 929f51b7662SDaniel Vetter { 930625dd9d3SDaniel Vetter struct agp_memory *new; 931625dd9d3SDaniel Vetter 932625dd9d3SDaniel Vetter if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) { 933625dd9d3SDaniel Vetter if (pg_count != intel_private.num_dcache_entries) 934625dd9d3SDaniel Vetter return NULL; 935625dd9d3SDaniel Vetter 936625dd9d3SDaniel Vetter new = agp_create_memory(1); 937625dd9d3SDaniel Vetter if (new == NULL) 938625dd9d3SDaniel Vetter return NULL; 939625dd9d3SDaniel Vetter 940625dd9d3SDaniel Vetter new->type = AGP_DCACHE_MEMORY; 941625dd9d3SDaniel Vetter new->page_count = pg_count; 942625dd9d3SDaniel Vetter new->num_scratch_pages = 0; 943625dd9d3SDaniel Vetter agp_free_page_array(new); 944625dd9d3SDaniel Vetter return new; 945625dd9d3SDaniel Vetter } 946f51b7662SDaniel Vetter if (type == AGP_PHYS_MEMORY) 947f51b7662SDaniel Vetter return alloc_agpphysmem_i8xx(pg_count, type); 948f51b7662SDaniel Vetter /* always return NULL for other allocation types for now */ 949f51b7662SDaniel Vetter return NULL; 950f51b7662SDaniel Vetter } 951f51b7662SDaniel Vetter 952f51b7662SDaniel Vetter static int intel_alloc_chipset_flush_resource(void) 953f51b7662SDaniel Vetter { 954f51b7662SDaniel Vetter int ret; 955d7cca2f7SDaniel Vetter ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE, 956f51b7662SDaniel Vetter PAGE_SIZE, PCIBIOS_MIN_MEM, 0, 957d7cca2f7SDaniel Vetter pcibios_align_resource, intel_private.bridge_dev); 958f51b7662SDaniel Vetter 959f51b7662SDaniel Vetter return ret; 960f51b7662SDaniel Vetter } 961f51b7662SDaniel Vetter 962f51b7662SDaniel Vetter static void intel_i915_setup_chipset_flush(void) 963f51b7662SDaniel Vetter { 964f51b7662SDaniel Vetter int ret; 965f51b7662SDaniel Vetter u32 temp; 966f51b7662SDaniel Vetter 967d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp); 968f51b7662SDaniel Vetter if (!(temp & 0x1)) { 969f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 970f51b7662SDaniel Vetter intel_private.resource_valid = 1; 971d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 972f51b7662SDaniel Vetter } else { 973f51b7662SDaniel Vetter temp &= ~1; 974f51b7662SDaniel Vetter 975f51b7662SDaniel Vetter intel_private.resource_valid = 1; 976f51b7662SDaniel Vetter intel_private.ifp_resource.start = temp; 977f51b7662SDaniel Vetter intel_private.ifp_resource.end = temp + PAGE_SIZE; 978f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 979f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 980f51b7662SDaniel Vetter if (ret) 981f51b7662SDaniel Vetter intel_private.resource_valid = 0; 982f51b7662SDaniel Vetter } 983f51b7662SDaniel Vetter } 984f51b7662SDaniel Vetter 985f51b7662SDaniel Vetter static void intel_i965_g33_setup_chipset_flush(void) 986f51b7662SDaniel Vetter { 987f51b7662SDaniel Vetter u32 temp_hi, temp_lo; 988f51b7662SDaniel Vetter int ret; 989f51b7662SDaniel Vetter 990d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi); 991d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo); 992f51b7662SDaniel Vetter 993f51b7662SDaniel Vetter if (!(temp_lo & 0x1)) { 994f51b7662SDaniel Vetter 995f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 996f51b7662SDaniel Vetter 997f51b7662SDaniel Vetter intel_private.resource_valid = 1; 998d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, 999f51b7662SDaniel Vetter upper_32_bits(intel_private.ifp_resource.start)); 1000d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 1001f51b7662SDaniel Vetter } else { 1002f51b7662SDaniel Vetter u64 l64; 1003f51b7662SDaniel Vetter 1004f51b7662SDaniel Vetter temp_lo &= ~0x1; 1005f51b7662SDaniel Vetter l64 = ((u64)temp_hi << 32) | temp_lo; 1006f51b7662SDaniel Vetter 1007f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1008f51b7662SDaniel Vetter intel_private.ifp_resource.start = l64; 1009f51b7662SDaniel Vetter intel_private.ifp_resource.end = l64 + PAGE_SIZE; 1010f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 1011f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 1012f51b7662SDaniel Vetter if (ret) 1013f51b7662SDaniel Vetter intel_private.resource_valid = 0; 1014f51b7662SDaniel Vetter } 1015f51b7662SDaniel Vetter } 1016f51b7662SDaniel Vetter 1017f51b7662SDaniel Vetter static void intel_i9xx_setup_flush(void) 1018f51b7662SDaniel Vetter { 1019f51b7662SDaniel Vetter /* return if already configured */ 1020f51b7662SDaniel Vetter if (intel_private.ifp_resource.start) 1021f51b7662SDaniel Vetter return; 1022f51b7662SDaniel Vetter 10231a997ff2SDaniel Vetter if (INTEL_GTT_GEN == 6) 1024f51b7662SDaniel Vetter return; 1025f51b7662SDaniel Vetter 1026f51b7662SDaniel Vetter /* setup a resource for this object */ 1027f51b7662SDaniel Vetter intel_private.ifp_resource.name = "Intel Flush Page"; 1028f51b7662SDaniel Vetter intel_private.ifp_resource.flags = IORESOURCE_MEM; 1029f51b7662SDaniel Vetter 1030f51b7662SDaniel Vetter /* Setup chipset flush for 915 */ 10311a997ff2SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN >= 4) { 1032f51b7662SDaniel Vetter intel_i965_g33_setup_chipset_flush(); 1033f51b7662SDaniel Vetter } else { 1034f51b7662SDaniel Vetter intel_i915_setup_chipset_flush(); 1035f51b7662SDaniel Vetter } 1036f51b7662SDaniel Vetter 1037df51e7aaSChris Wilson if (intel_private.ifp_resource.start) 1038f51b7662SDaniel Vetter intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE); 1039f51b7662SDaniel Vetter if (!intel_private.i9xx_flush_page) 1040df51e7aaSChris Wilson dev_err(&intel_private.pcidev->dev, 1041df51e7aaSChris Wilson "can't ioremap flush page - no chipset flushing\n"); 1042f51b7662SDaniel Vetter } 1043f51b7662SDaniel Vetter 1044ae83dd5cSDaniel Vetter static void i9xx_cleanup(void) 1045ae83dd5cSDaniel Vetter { 1046ae83dd5cSDaniel Vetter if (intel_private.i9xx_flush_page) 1047ae83dd5cSDaniel Vetter iounmap(intel_private.i9xx_flush_page); 1048ae83dd5cSDaniel Vetter if (intel_private.resource_valid) 1049ae83dd5cSDaniel Vetter release_resource(&intel_private.ifp_resource); 1050ae83dd5cSDaniel Vetter intel_private.ifp_resource.start = 0; 1051ae83dd5cSDaniel Vetter intel_private.resource_valid = 0; 1052ae83dd5cSDaniel Vetter } 1053ae83dd5cSDaniel Vetter 10541b263f24SDaniel Vetter static void i9xx_chipset_flush(void) 1055f51b7662SDaniel Vetter { 1056f51b7662SDaniel Vetter if (intel_private.i9xx_flush_page) 1057f51b7662SDaniel Vetter writel(1, intel_private.i9xx_flush_page); 1058f51b7662SDaniel Vetter } 1059f51b7662SDaniel Vetter 106071f45660SChris Wilson static void i965_write_entry(dma_addr_t addr, 106171f45660SChris Wilson unsigned int entry, 1062a6963596SDaniel Vetter unsigned int flags) 1063a6963596SDaniel Vetter { 106471f45660SChris Wilson u32 pte_flags; 106571f45660SChris Wilson 106671f45660SChris Wilson pte_flags = I810_PTE_VALID; 106771f45660SChris Wilson if (flags == AGP_USER_CACHED_MEMORY) 106871f45660SChris Wilson pte_flags |= I830_PTE_SYSTEM_CACHED; 106971f45660SChris Wilson 1070a6963596SDaniel Vetter /* Shift high bits down */ 1071a6963596SDaniel Vetter addr |= (addr >> 28) & 0xf0; 107271f45660SChris Wilson writel(addr | pte_flags, intel_private.gtt + entry); 1073a6963596SDaniel Vetter } 1074a6963596SDaniel Vetter 10755c042287SBen Widawsky 10762d2430cfSDaniel Vetter static int i9xx_setup(void) 10772d2430cfSDaniel Vetter { 1078009946f8SBen Widawsky u32 reg_addr, gtt_addr; 10794b60d29eSJesse Barnes int size = KB(512); 10802d2430cfSDaniel Vetter 10812d2430cfSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I915_MMADDR, ®_addr); 10822d2430cfSDaniel Vetter 10832d2430cfSDaniel Vetter reg_addr &= 0xfff80000; 10842d2430cfSDaniel Vetter 10854b60d29eSJesse Barnes intel_private.registers = ioremap(reg_addr, size); 10862d2430cfSDaniel Vetter if (!intel_private.registers) 10872d2430cfSDaniel Vetter return -ENOMEM; 10882d2430cfSDaniel Vetter 1089009946f8SBen Widawsky switch (INTEL_GTT_GEN) { 1090009946f8SBen Widawsky case 3: 10912d2430cfSDaniel Vetter pci_read_config_dword(intel_private.pcidev, 10922d2430cfSDaniel Vetter I915_PTEADDR, >t_addr); 10932d2430cfSDaniel Vetter intel_private.gtt_bus_addr = gtt_addr; 1094009946f8SBen Widawsky break; 10952d2430cfSDaniel Vetter case 5: 1096009946f8SBen Widawsky intel_private.gtt_bus_addr = reg_addr + MB(2); 10972d2430cfSDaniel Vetter break; 10982d2430cfSDaniel Vetter default: 1099009946f8SBen Widawsky intel_private.gtt_bus_addr = reg_addr + KB(512); 11002d2430cfSDaniel Vetter break; 11012d2430cfSDaniel Vetter } 11022d2430cfSDaniel Vetter 11032d2430cfSDaniel Vetter intel_i9xx_setup_flush(); 11042d2430cfSDaniel Vetter 11052d2430cfSDaniel Vetter return 0; 11062d2430cfSDaniel Vetter } 11072d2430cfSDaniel Vetter 1108e9b1cc81SDaniel Vetter static const struct agp_bridge_driver intel_fake_agp_driver = { 1109f51b7662SDaniel Vetter .owner = THIS_MODULE, 1110f51b7662SDaniel Vetter .size_type = FIXED_APER_SIZE, 11119e76e7b8SChris Wilson .aperture_sizes = intel_fake_agp_sizes, 11129e76e7b8SChris Wilson .num_aperture_sizes = ARRAY_SIZE(intel_fake_agp_sizes), 1113a6963596SDaniel Vetter .configure = intel_fake_agp_configure, 11143e921f98SDaniel Vetter .fetch_size = intel_fake_agp_fetch_size, 1115fdfb58a9SDaniel Vetter .cleanup = intel_gtt_cleanup, 1116ffdd7510SDaniel Vetter .agp_enable = intel_fake_agp_enable, 1117f51b7662SDaniel Vetter .cache_flush = global_cache_flush, 11183b15a9d7SDaniel Vetter .create_gatt_table = intel_fake_agp_create_gatt_table, 1119ffdd7510SDaniel Vetter .free_gatt_table = intel_fake_agp_free_gatt_table, 1120450f2b3dSDaniel Vetter .insert_memory = intel_fake_agp_insert_entries, 1121450f2b3dSDaniel Vetter .remove_memory = intel_fake_agp_remove_entries, 1122ffdd7510SDaniel Vetter .alloc_by_type = intel_fake_agp_alloc_by_type, 1123f51b7662SDaniel Vetter .free_by_type = intel_i810_free_by_type, 1124f51b7662SDaniel Vetter .agp_alloc_page = agp_generic_alloc_page, 1125f51b7662SDaniel Vetter .agp_alloc_pages = agp_generic_alloc_pages, 1126f51b7662SDaniel Vetter .agp_destroy_page = agp_generic_destroy_page, 1127f51b7662SDaniel Vetter .agp_destroy_pages = agp_generic_destroy_pages, 1128f51b7662SDaniel Vetter }; 112902c026ceSDaniel Vetter 1130bdd30729SDaniel Vetter static const struct intel_gtt_driver i81x_gtt_driver = { 1131bdd30729SDaniel Vetter .gen = 1, 1132820647b9SDaniel Vetter .has_pgtbl_enable = 1, 113322533b49SDaniel Vetter .dma_mask_size = 32, 1134820647b9SDaniel Vetter .setup = i810_setup, 1135820647b9SDaniel Vetter .cleanup = i810_cleanup, 1136625dd9d3SDaniel Vetter .check_flags = i830_check_flags, 1137625dd9d3SDaniel Vetter .write_entry = i810_write_entry, 1138bdd30729SDaniel Vetter }; 11391a997ff2SDaniel Vetter static const struct intel_gtt_driver i8xx_gtt_driver = { 11401a997ff2SDaniel Vetter .gen = 2, 1141100519e2SChris Wilson .has_pgtbl_enable = 1, 114273800422SDaniel Vetter .setup = i830_setup, 1143ae83dd5cSDaniel Vetter .cleanup = i830_cleanup, 1144351bb278SDaniel Vetter .write_entry = i830_write_entry, 114522533b49SDaniel Vetter .dma_mask_size = 32, 11465cbecafcSDaniel Vetter .check_flags = i830_check_flags, 11471b263f24SDaniel Vetter .chipset_flush = i830_chipset_flush, 11481a997ff2SDaniel Vetter }; 11491a997ff2SDaniel Vetter static const struct intel_gtt_driver i915_gtt_driver = { 11501a997ff2SDaniel Vetter .gen = 3, 1151100519e2SChris Wilson .has_pgtbl_enable = 1, 11522d2430cfSDaniel Vetter .setup = i9xx_setup, 1153ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1154351bb278SDaniel Vetter /* i945 is the last gpu to need phys mem (for overlay and cursors). */ 1155351bb278SDaniel Vetter .write_entry = i830_write_entry, 115622533b49SDaniel Vetter .dma_mask_size = 32, 1157fefaa70fSDaniel Vetter .check_flags = i830_check_flags, 11581b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11591a997ff2SDaniel Vetter }; 11601a997ff2SDaniel Vetter static const struct intel_gtt_driver g33_gtt_driver = { 11611a997ff2SDaniel Vetter .gen = 3, 11621a997ff2SDaniel Vetter .is_g33 = 1, 11632d2430cfSDaniel Vetter .setup = i9xx_setup, 1164ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1165a6963596SDaniel Vetter .write_entry = i965_write_entry, 116622533b49SDaniel Vetter .dma_mask_size = 36, 1167450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11681b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11691a997ff2SDaniel Vetter }; 11701a997ff2SDaniel Vetter static const struct intel_gtt_driver pineview_gtt_driver = { 11711a997ff2SDaniel Vetter .gen = 3, 11721a997ff2SDaniel Vetter .is_pineview = 1, .is_g33 = 1, 11732d2430cfSDaniel Vetter .setup = i9xx_setup, 1174ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1175a6963596SDaniel Vetter .write_entry = i965_write_entry, 117622533b49SDaniel Vetter .dma_mask_size = 36, 1177450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11781b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11791a997ff2SDaniel Vetter }; 11801a997ff2SDaniel Vetter static const struct intel_gtt_driver i965_gtt_driver = { 11811a997ff2SDaniel Vetter .gen = 4, 1182100519e2SChris Wilson .has_pgtbl_enable = 1, 11832d2430cfSDaniel Vetter .setup = i9xx_setup, 1184ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1185a6963596SDaniel Vetter .write_entry = i965_write_entry, 118622533b49SDaniel Vetter .dma_mask_size = 36, 1187450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11881b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11891a997ff2SDaniel Vetter }; 11901a997ff2SDaniel Vetter static const struct intel_gtt_driver g4x_gtt_driver = { 11911a997ff2SDaniel Vetter .gen = 5, 11922d2430cfSDaniel Vetter .setup = i9xx_setup, 1193ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1194a6963596SDaniel Vetter .write_entry = i965_write_entry, 119522533b49SDaniel Vetter .dma_mask_size = 36, 1196450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11971b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11981a997ff2SDaniel Vetter }; 11991a997ff2SDaniel Vetter static const struct intel_gtt_driver ironlake_gtt_driver = { 12001a997ff2SDaniel Vetter .gen = 5, 12011a997ff2SDaniel Vetter .is_ironlake = 1, 12022d2430cfSDaniel Vetter .setup = i9xx_setup, 1203ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1204a6963596SDaniel Vetter .write_entry = i965_write_entry, 120522533b49SDaniel Vetter .dma_mask_size = 36, 1206450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12071b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12081a997ff2SDaniel Vetter }; 12091a997ff2SDaniel Vetter 121002c026ceSDaniel Vetter /* Table to describe Intel GMCH and AGP/PCIE GART drivers. At least one of 121102c026ceSDaniel Vetter * driver and gmch_driver must be non-null, and find_gmch will determine 121202c026ceSDaniel Vetter * which one should be used if a gmch_chip_id is present. 121302c026ceSDaniel Vetter */ 121402c026ceSDaniel Vetter static const struct intel_gtt_driver_description { 121502c026ceSDaniel Vetter unsigned int gmch_chip_id; 121602c026ceSDaniel Vetter char *name; 12171a997ff2SDaniel Vetter const struct intel_gtt_driver *gtt_driver; 121802c026ceSDaniel Vetter } intel_gtt_chipsets[] = { 1219ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", 1220bdd30729SDaniel Vetter &i81x_gtt_driver}, 1221ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", 1222bdd30729SDaniel Vetter &i81x_gtt_driver}, 1223ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", 1224bdd30729SDaniel Vetter &i81x_gtt_driver}, 1225ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", 1226bdd30729SDaniel Vetter &i81x_gtt_driver}, 12271a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", 1228ff26860fSDaniel Vetter &i8xx_gtt_driver}, 122953371edaSOswald Buddenhagen { PCI_DEVICE_ID_INTEL_82845G_IG, "845G", 1230ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12311a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82854_IG, "854", 1232ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12331a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", 1234ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12351a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82865_IG, "865", 1236ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12371a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", 1238ff26860fSDaniel Vetter &i915_gtt_driver }, 12391a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", 1240ff26860fSDaniel Vetter &i915_gtt_driver }, 12411a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", 1242ff26860fSDaniel Vetter &i915_gtt_driver }, 12431a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", 1244ff26860fSDaniel Vetter &i915_gtt_driver }, 12451a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", 1246ff26860fSDaniel Vetter &i915_gtt_driver }, 12471a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", 1248ff26860fSDaniel Vetter &i915_gtt_driver }, 12491a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", 1250ff26860fSDaniel Vetter &i965_gtt_driver }, 12511a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", 1252ff26860fSDaniel Vetter &i965_gtt_driver }, 12531a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", 1254ff26860fSDaniel Vetter &i965_gtt_driver }, 12551a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", 1256ff26860fSDaniel Vetter &i965_gtt_driver }, 12571a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", 1258ff26860fSDaniel Vetter &i965_gtt_driver }, 12591a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", 1260ff26860fSDaniel Vetter &i965_gtt_driver }, 12611a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G33_IG, "G33", 1262ff26860fSDaniel Vetter &g33_gtt_driver }, 12631a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", 1264ff26860fSDaniel Vetter &g33_gtt_driver }, 12651a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", 1266ff26860fSDaniel Vetter &g33_gtt_driver }, 12671a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", 1268ff26860fSDaniel Vetter &pineview_gtt_driver }, 12691a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", 1270ff26860fSDaniel Vetter &pineview_gtt_driver }, 12711a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", 1272ff26860fSDaniel Vetter &g4x_gtt_driver }, 12731a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", 1274ff26860fSDaniel Vetter &g4x_gtt_driver }, 12751a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", 1276ff26860fSDaniel Vetter &g4x_gtt_driver }, 12771a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", 1278ff26860fSDaniel Vetter &g4x_gtt_driver }, 12791a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_B43_IG, "B43", 1280ff26860fSDaniel Vetter &g4x_gtt_driver }, 1281e9e5f8e8SChris Wilson { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", 1282ff26860fSDaniel Vetter &g4x_gtt_driver }, 12831a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G41_IG, "G41", 1284ff26860fSDaniel Vetter &g4x_gtt_driver }, 128502c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, 1286ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 128702c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, 1288ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 128902c026ceSDaniel Vetter { 0, NULL, NULL } 129002c026ceSDaniel Vetter }; 129102c026ceSDaniel Vetter 129202c026ceSDaniel Vetter static int find_gmch(u16 device) 129302c026ceSDaniel Vetter { 129402c026ceSDaniel Vetter struct pci_dev *gmch_device; 129502c026ceSDaniel Vetter 129602c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL); 129702c026ceSDaniel Vetter if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) { 129802c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, 129902c026ceSDaniel Vetter device, gmch_device); 130002c026ceSDaniel Vetter } 130102c026ceSDaniel Vetter 130202c026ceSDaniel Vetter if (!gmch_device) 130302c026ceSDaniel Vetter return 0; 130402c026ceSDaniel Vetter 130502c026ceSDaniel Vetter intel_private.pcidev = gmch_device; 130602c026ceSDaniel Vetter return 1; 130702c026ceSDaniel Vetter } 130802c026ceSDaniel Vetter 130914be93ddSDaniel Vetter int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev, 131002c026ceSDaniel Vetter struct agp_bridge_data *bridge) 131102c026ceSDaniel Vetter { 131202c026ceSDaniel Vetter int i, mask; 131314be93ddSDaniel Vetter 131414be93ddSDaniel Vetter /* 131514be93ddSDaniel Vetter * Can be called from the fake agp driver but also directly from 131614be93ddSDaniel Vetter * drm/i915.ko. Hence we need to check whether everything is set up 131714be93ddSDaniel Vetter * already. 131814be93ddSDaniel Vetter */ 131914be93ddSDaniel Vetter if (intel_private.driver) { 132014be93ddSDaniel Vetter intel_private.refcount++; 132114be93ddSDaniel Vetter return 1; 132214be93ddSDaniel Vetter } 132302c026ceSDaniel Vetter 132402c026ceSDaniel Vetter for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { 132514be93ddSDaniel Vetter if (gpu_pdev) { 132614be93ddSDaniel Vetter if (gpu_pdev->device == 132714be93ddSDaniel Vetter intel_gtt_chipsets[i].gmch_chip_id) { 132814be93ddSDaniel Vetter intel_private.pcidev = pci_dev_get(gpu_pdev); 132914be93ddSDaniel Vetter intel_private.driver = 133014be93ddSDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 133114be93ddSDaniel Vetter 133214be93ddSDaniel Vetter break; 133314be93ddSDaniel Vetter } 133414be93ddSDaniel Vetter } else if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { 13351a997ff2SDaniel Vetter intel_private.driver = 13361a997ff2SDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 133702c026ceSDaniel Vetter break; 133802c026ceSDaniel Vetter } 133902c026ceSDaniel Vetter } 134002c026ceSDaniel Vetter 1341ff26860fSDaniel Vetter if (!intel_private.driver) 134202c026ceSDaniel Vetter return 0; 134302c026ceSDaniel Vetter 134414be93ddSDaniel Vetter intel_private.refcount++; 134514be93ddSDaniel Vetter 13467e8f6306SDaniel Vetter if (bridge) { 1347ff26860fSDaniel Vetter bridge->driver = &intel_fake_agp_driver; 134802c026ceSDaniel Vetter bridge->dev_private_data = &intel_private; 134914be93ddSDaniel Vetter bridge->dev = bridge_pdev; 13507e8f6306SDaniel Vetter } 135102c026ceSDaniel Vetter 135214be93ddSDaniel Vetter intel_private.bridge_dev = pci_dev_get(bridge_pdev); 1353d7cca2f7SDaniel Vetter 135414be93ddSDaniel Vetter dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name); 135502c026ceSDaniel Vetter 135622533b49SDaniel Vetter mask = intel_private.driver->dma_mask_size; 135702c026ceSDaniel Vetter if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask))) 135802c026ceSDaniel Vetter dev_err(&intel_private.pcidev->dev, 135902c026ceSDaniel Vetter "set gfx device dma mask %d-bit failed!\n", mask); 136002c026ceSDaniel Vetter else 136102c026ceSDaniel Vetter pci_set_consistent_dma_mask(intel_private.pcidev, 136202c026ceSDaniel Vetter DMA_BIT_MASK(mask)); 136302c026ceSDaniel Vetter 136414be93ddSDaniel Vetter if (intel_gtt_init() != 0) { 136514be93ddSDaniel Vetter intel_gmch_remove(); 136614be93ddSDaniel Vetter 13673b15a9d7SDaniel Vetter return 0; 136814be93ddSDaniel Vetter } 13691784a5fbSDaniel Vetter 137002c026ceSDaniel Vetter return 1; 137102c026ceSDaniel Vetter } 1372e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_probe); 137302c026ceSDaniel Vetter 1374*a54c0c27SBen Widawsky void intel_gtt_get(size_t *gtt_total, size_t *stolen_size) 137519966754SDaniel Vetter { 1376*a54c0c27SBen Widawsky *gtt_total = intel_private.gtt_total_entries << PAGE_SHIFT; 1377*a54c0c27SBen Widawsky *stolen_size = intel_private.stolen_size; 137819966754SDaniel Vetter } 137919966754SDaniel Vetter EXPORT_SYMBOL(intel_gtt_get); 138019966754SDaniel Vetter 138140ce6575SDaniel Vetter void intel_gtt_chipset_flush(void) 138240ce6575SDaniel Vetter { 138340ce6575SDaniel Vetter if (intel_private.driver->chipset_flush) 138440ce6575SDaniel Vetter intel_private.driver->chipset_flush(); 138540ce6575SDaniel Vetter } 138640ce6575SDaniel Vetter EXPORT_SYMBOL(intel_gtt_chipset_flush); 138740ce6575SDaniel Vetter 138814be93ddSDaniel Vetter void intel_gmch_remove(void) 138902c026ceSDaniel Vetter { 139014be93ddSDaniel Vetter if (--intel_private.refcount) 139114be93ddSDaniel Vetter return; 139214be93ddSDaniel Vetter 139302c026ceSDaniel Vetter if (intel_private.pcidev) 139402c026ceSDaniel Vetter pci_dev_put(intel_private.pcidev); 1395d7cca2f7SDaniel Vetter if (intel_private.bridge_dev) 1396d7cca2f7SDaniel Vetter pci_dev_put(intel_private.bridge_dev); 139714be93ddSDaniel Vetter intel_private.driver = NULL; 139802c026ceSDaniel Vetter } 1399e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_remove); 1400e2404e7cSDaniel Vetter 1401e2404e7cSDaniel Vetter MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); 1402e2404e7cSDaniel Vetter MODULE_LICENSE("GPL and additional rights"); 1403