1f51b7662SDaniel Vetter /* 2f51b7662SDaniel Vetter * Intel GTT (Graphics Translation Table) routines 3f51b7662SDaniel Vetter * 4f51b7662SDaniel Vetter * Caveat: This driver implements the linux agp interface, but this is far from 5f51b7662SDaniel Vetter * a agp driver! GTT support ended up here for purely historical reasons: The 6f51b7662SDaniel Vetter * old userspace intel graphics drivers needed an interface to map memory into 7f51b7662SDaniel Vetter * the GTT. And the drm provides a default interface for graphic devices sitting 8f51b7662SDaniel Vetter * on an agp port. So it made sense to fake the GTT support as an agp port to 9f51b7662SDaniel Vetter * avoid having to create a new api. 10f51b7662SDaniel Vetter * 11f51b7662SDaniel Vetter * With gem this does not make much sense anymore, just needlessly complicates 12f51b7662SDaniel Vetter * the code. But as long as the old graphics stack is still support, it's stuck 13f51b7662SDaniel Vetter * here. 14f51b7662SDaniel Vetter * 15f51b7662SDaniel Vetter * /fairy-tale-mode off 16f51b7662SDaniel Vetter */ 17f51b7662SDaniel Vetter 18e2404e7cSDaniel Vetter #include <linux/module.h> 19e2404e7cSDaniel Vetter #include <linux/pci.h> 20e2404e7cSDaniel Vetter #include <linux/kernel.h> 21e2404e7cSDaniel Vetter #include <linux/pagemap.h> 22e2404e7cSDaniel Vetter #include <linux/agp_backend.h> 23bdb8b975SChris Wilson #include <linux/delay.h> 24e2404e7cSDaniel Vetter #include <asm/smp.h> 25e2404e7cSDaniel Vetter #include "agp.h" 26e2404e7cSDaniel Vetter #include "intel-agp.h" 270ade6386SDaniel Vetter #include <drm/intel-gtt.h> 28e47036b4SLaura Abbott #include <asm/set_memory.h> 29e2404e7cSDaniel Vetter 30f51b7662SDaniel Vetter /* 31f51b7662SDaniel Vetter * If we have Intel graphics, we're not going to have anything other than 32f51b7662SDaniel Vetter * an Intel IOMMU. So make the correct use of the PCI DMA API contingent 33d3f13810SSuresh Siddha * on the Intel IOMMU support (CONFIG_INTEL_IOMMU). 34f51b7662SDaniel Vetter * Only newer chipsets need to bother with this, of course. 35f51b7662SDaniel Vetter */ 36d3f13810SSuresh Siddha #ifdef CONFIG_INTEL_IOMMU 37f51b7662SDaniel Vetter #define USE_PCI_DMA_API 1 380e87d2b0SDaniel Vetter #else 390e87d2b0SDaniel Vetter #define USE_PCI_DMA_API 0 40f51b7662SDaniel Vetter #endif 41f51b7662SDaniel Vetter 421a997ff2SDaniel Vetter struct intel_gtt_driver { 431a997ff2SDaniel Vetter unsigned int gen : 8; 441a997ff2SDaniel Vetter unsigned int is_g33 : 1; 451a997ff2SDaniel Vetter unsigned int is_pineview : 1; 461a997ff2SDaniel Vetter unsigned int is_ironlake : 1; 47100519e2SChris Wilson unsigned int has_pgtbl_enable : 1; 4822533b49SDaniel Vetter unsigned int dma_mask_size : 8; 4973800422SDaniel Vetter /* Chipset specific GTT setup */ 5073800422SDaniel Vetter int (*setup)(void); 51ae83dd5cSDaniel Vetter /* This should undo anything done in ->setup() save the unmapping 52ae83dd5cSDaniel Vetter * of the mmio register file, that's done in the generic code. */ 53ae83dd5cSDaniel Vetter void (*cleanup)(void); 54351bb278SDaniel Vetter void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags); 55351bb278SDaniel Vetter /* Flags is a more or less chipset specific opaque value. 56351bb278SDaniel Vetter * For chipsets that need to support old ums (non-gem) code, this 57351bb278SDaniel Vetter * needs to be identical to the various supported agp memory types! */ 585cbecafcSDaniel Vetter bool (*check_flags)(unsigned int flags); 591b263f24SDaniel Vetter void (*chipset_flush)(void); 601a997ff2SDaniel Vetter }; 611a997ff2SDaniel Vetter 62f51b7662SDaniel Vetter static struct _intel_private { 631a997ff2SDaniel Vetter const struct intel_gtt_driver *driver; 64f51b7662SDaniel Vetter struct pci_dev *pcidev; /* device one */ 65d7cca2f7SDaniel Vetter struct pci_dev *bridge_dev; 66f51b7662SDaniel Vetter u8 __iomem *registers; 675acc4ce4SBjorn Helgaas phys_addr_t gtt_phys_addr; 68b3eafc5aSDaniel Vetter u32 PGETBL_save; 69f51b7662SDaniel Vetter u32 __iomem *gtt; /* I915G */ 70bee4a186SChris Wilson bool clear_fake_agp; /* on first access via agp, fill with scratch */ 71f51b7662SDaniel Vetter int num_dcache_entries; 72f51b7662SDaniel Vetter void __iomem *i9xx_flush_page; 73820647b9SDaniel Vetter char *i81x_gtt_table; 74f51b7662SDaniel Vetter struct resource ifp_resource; 75f51b7662SDaniel Vetter int resource_valid; 760e87d2b0SDaniel Vetter struct page *scratch_page; 779c61a32dSBen Widawsky phys_addr_t scratch_page_dma; 7814be93ddSDaniel Vetter int refcount; 798d2e6308SBen Widawsky /* Whether i915 needs to use the dmar apis or not. */ 808d2e6308SBen Widawsky unsigned int needs_dmar : 1; 81e5c65377SBen Widawsky phys_addr_t gma_bus_addr; 82a54c0c27SBen Widawsky /* Size of memory reserved for graphics by the BIOS */ 83*b7128ef1SMatthew Auld resource_size_t stolen_size; 84a54c0c27SBen Widawsky /* Total number of gtt entries. */ 85a54c0c27SBen Widawsky unsigned int gtt_total_entries; 86a54c0c27SBen Widawsky /* Part of the gtt that is mappable by the cpu, for those chips where 87a54c0c27SBen Widawsky * this is not the full gtt. */ 88a54c0c27SBen Widawsky unsigned int gtt_mappable_entries; 89f51b7662SDaniel Vetter } intel_private; 90f51b7662SDaniel Vetter 911a997ff2SDaniel Vetter #define INTEL_GTT_GEN intel_private.driver->gen 921a997ff2SDaniel Vetter #define IS_G33 intel_private.driver->is_g33 931a997ff2SDaniel Vetter #define IS_PINEVIEW intel_private.driver->is_pineview 941a997ff2SDaniel Vetter #define IS_IRONLAKE intel_private.driver->is_ironlake 95100519e2SChris Wilson #define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable 961a997ff2SDaniel Vetter 9700fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 989da3da66SChris Wilson static int intel_gtt_map_memory(struct page **pages, 999da3da66SChris Wilson unsigned int num_entries, 1009da3da66SChris Wilson struct sg_table *st) 101f51b7662SDaniel Vetter { 102f51b7662SDaniel Vetter struct scatterlist *sg; 103f51b7662SDaniel Vetter int i; 104f51b7662SDaniel Vetter 1054080775bSDaniel Vetter DBG("try mapping %lu pages\n", (unsigned long)num_entries); 106f51b7662SDaniel Vetter 1079da3da66SChris Wilson if (sg_alloc_table(st, num_entries, GFP_KERNEL)) 108831cd445SChris Wilson goto err; 109f51b7662SDaniel Vetter 1109da3da66SChris Wilson for_each_sg(st->sgl, sg, num_entries, i) 1114080775bSDaniel Vetter sg_set_page(sg, pages[i], PAGE_SIZE, 0); 112f51b7662SDaniel Vetter 1139da3da66SChris Wilson if (!pci_map_sg(intel_private.pcidev, 1149da3da66SChris Wilson st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL)) 115831cd445SChris Wilson goto err; 116831cd445SChris Wilson 117f51b7662SDaniel Vetter return 0; 118831cd445SChris Wilson 119831cd445SChris Wilson err: 1209da3da66SChris Wilson sg_free_table(st); 121831cd445SChris Wilson return -ENOMEM; 122f51b7662SDaniel Vetter } 123f51b7662SDaniel Vetter 1249da3da66SChris Wilson static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg) 125f51b7662SDaniel Vetter { 1264080775bSDaniel Vetter struct sg_table st; 127f51b7662SDaniel Vetter DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); 128f51b7662SDaniel Vetter 1294080775bSDaniel Vetter pci_unmap_sg(intel_private.pcidev, sg_list, 1304080775bSDaniel Vetter num_sg, PCI_DMA_BIDIRECTIONAL); 1314080775bSDaniel Vetter 1324080775bSDaniel Vetter st.sgl = sg_list; 1334080775bSDaniel Vetter st.orig_nents = st.nents = num_sg; 1344080775bSDaniel Vetter 1354080775bSDaniel Vetter sg_free_table(&st); 136f51b7662SDaniel Vetter } 137f51b7662SDaniel Vetter 138ffdd7510SDaniel Vetter static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) 139f51b7662SDaniel Vetter { 140f51b7662SDaniel Vetter return; 141f51b7662SDaniel Vetter } 142f51b7662SDaniel Vetter 143f51b7662SDaniel Vetter /* Exists to support ARGB cursors */ 144f51b7662SDaniel Vetter static struct page *i8xx_alloc_pages(void) 145f51b7662SDaniel Vetter { 146f51b7662SDaniel Vetter struct page *page; 147f51b7662SDaniel Vetter 148f51b7662SDaniel Vetter page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2); 149f51b7662SDaniel Vetter if (page == NULL) 150f51b7662SDaniel Vetter return NULL; 151f51b7662SDaniel Vetter 152f51b7662SDaniel Vetter if (set_pages_uc(page, 4) < 0) { 153f51b7662SDaniel Vetter set_pages_wb(page, 4); 154f51b7662SDaniel Vetter __free_pages(page, 2); 155f51b7662SDaniel Vetter return NULL; 156f51b7662SDaniel Vetter } 157f51b7662SDaniel Vetter atomic_inc(&agp_bridge->current_memory_agp); 158f51b7662SDaniel Vetter return page; 159f51b7662SDaniel Vetter } 160f51b7662SDaniel Vetter 161f51b7662SDaniel Vetter static void i8xx_destroy_pages(struct page *page) 162f51b7662SDaniel Vetter { 163f51b7662SDaniel Vetter if (page == NULL) 164f51b7662SDaniel Vetter return; 165f51b7662SDaniel Vetter 166f51b7662SDaniel Vetter set_pages_wb(page, 4); 167f51b7662SDaniel Vetter __free_pages(page, 2); 168f51b7662SDaniel Vetter atomic_dec(&agp_bridge->current_memory_agp); 169f51b7662SDaniel Vetter } 17000fe639aSVille Syrjälä #endif 171f51b7662SDaniel Vetter 172820647b9SDaniel Vetter #define I810_GTT_ORDER 4 173820647b9SDaniel Vetter static int i810_setup(void) 174820647b9SDaniel Vetter { 175d3572532SBjorn Helgaas phys_addr_t reg_addr; 176820647b9SDaniel Vetter char *gtt_table; 177820647b9SDaniel Vetter 178820647b9SDaniel Vetter /* i81x does not preallocate the gtt. It's always 64kb in size. */ 179820647b9SDaniel Vetter gtt_table = alloc_gatt_pages(I810_GTT_ORDER); 180820647b9SDaniel Vetter if (gtt_table == NULL) 181820647b9SDaniel Vetter return -ENOMEM; 182820647b9SDaniel Vetter intel_private.i81x_gtt_table = gtt_table; 183820647b9SDaniel Vetter 184d3572532SBjorn Helgaas reg_addr = pci_resource_start(intel_private.pcidev, I810_MMADR_BAR); 185820647b9SDaniel Vetter 186820647b9SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 187820647b9SDaniel Vetter if (!intel_private.registers) 188820647b9SDaniel Vetter return -ENOMEM; 189820647b9SDaniel Vetter 190820647b9SDaniel Vetter writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED, 191820647b9SDaniel Vetter intel_private.registers+I810_PGETBL_CTL); 192820647b9SDaniel Vetter 1935acc4ce4SBjorn Helgaas intel_private.gtt_phys_addr = reg_addr + I810_PTE_BASE; 194820647b9SDaniel Vetter 195820647b9SDaniel Vetter if ((readl(intel_private.registers+I810_DRAM_CTL) 196820647b9SDaniel Vetter & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) { 197820647b9SDaniel Vetter dev_info(&intel_private.pcidev->dev, 198820647b9SDaniel Vetter "detected 4MB dedicated video ram\n"); 199820647b9SDaniel Vetter intel_private.num_dcache_entries = 1024; 200820647b9SDaniel Vetter } 201820647b9SDaniel Vetter 202820647b9SDaniel Vetter return 0; 203820647b9SDaniel Vetter } 204820647b9SDaniel Vetter 205820647b9SDaniel Vetter static void i810_cleanup(void) 206820647b9SDaniel Vetter { 207820647b9SDaniel Vetter writel(0, intel_private.registers+I810_PGETBL_CTL); 208820647b9SDaniel Vetter free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER); 209820647b9SDaniel Vetter } 210820647b9SDaniel Vetter 21100fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 212ff26860fSDaniel Vetter static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start, 213f51b7662SDaniel Vetter int type) 214f51b7662SDaniel Vetter { 215f51b7662SDaniel Vetter int i; 216f51b7662SDaniel Vetter 217625dd9d3SDaniel Vetter if ((pg_start + mem->page_count) 218625dd9d3SDaniel Vetter > intel_private.num_dcache_entries) 219625dd9d3SDaniel Vetter return -EINVAL; 220f51b7662SDaniel Vetter 221625dd9d3SDaniel Vetter if (!mem->is_flushed) 222625dd9d3SDaniel Vetter global_cache_flush(); 223625dd9d3SDaniel Vetter 224625dd9d3SDaniel Vetter for (i = pg_start; i < (pg_start + mem->page_count); i++) { 225625dd9d3SDaniel Vetter dma_addr_t addr = i << PAGE_SHIFT; 226625dd9d3SDaniel Vetter intel_private.driver->write_entry(addr, 227625dd9d3SDaniel Vetter i, type); 228f51b7662SDaniel Vetter } 229983d308cSChris Wilson wmb(); 230f51b7662SDaniel Vetter 231f51b7662SDaniel Vetter return 0; 232f51b7662SDaniel Vetter } 233f51b7662SDaniel Vetter 234f51b7662SDaniel Vetter /* 235f51b7662SDaniel Vetter * The i810/i830 requires a physical address to program its mouse 236f51b7662SDaniel Vetter * pointer into hardware. 237f51b7662SDaniel Vetter * However the Xserver still writes to it through the agp aperture. 238f51b7662SDaniel Vetter */ 239f51b7662SDaniel Vetter static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type) 240f51b7662SDaniel Vetter { 241f51b7662SDaniel Vetter struct agp_memory *new; 242f51b7662SDaniel Vetter struct page *page; 243f51b7662SDaniel Vetter 244f51b7662SDaniel Vetter switch (pg_count) { 245f51b7662SDaniel Vetter case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge); 246f51b7662SDaniel Vetter break; 247f51b7662SDaniel Vetter case 4: 248f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 249f51b7662SDaniel Vetter page = i8xx_alloc_pages(); 250f51b7662SDaniel Vetter break; 251f51b7662SDaniel Vetter default: 252f51b7662SDaniel Vetter return NULL; 253f51b7662SDaniel Vetter } 254f51b7662SDaniel Vetter 255f51b7662SDaniel Vetter if (page == NULL) 256f51b7662SDaniel Vetter return NULL; 257f51b7662SDaniel Vetter 258f51b7662SDaniel Vetter new = agp_create_memory(pg_count); 259f51b7662SDaniel Vetter if (new == NULL) 260f51b7662SDaniel Vetter return NULL; 261f51b7662SDaniel Vetter 262f51b7662SDaniel Vetter new->pages[0] = page; 263f51b7662SDaniel Vetter if (pg_count == 4) { 264f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 265f51b7662SDaniel Vetter new->pages[1] = new->pages[0] + 1; 266f51b7662SDaniel Vetter new->pages[2] = new->pages[1] + 1; 267f51b7662SDaniel Vetter new->pages[3] = new->pages[2] + 1; 268f51b7662SDaniel Vetter } 269f51b7662SDaniel Vetter new->page_count = pg_count; 270f51b7662SDaniel Vetter new->num_scratch_pages = pg_count; 271f51b7662SDaniel Vetter new->type = AGP_PHYS_MEMORY; 272f51b7662SDaniel Vetter new->physical = page_to_phys(new->pages[0]); 273f51b7662SDaniel Vetter return new; 274f51b7662SDaniel Vetter } 275f51b7662SDaniel Vetter 276f51b7662SDaniel Vetter static void intel_i810_free_by_type(struct agp_memory *curr) 277f51b7662SDaniel Vetter { 278f51b7662SDaniel Vetter agp_free_key(curr->key); 279f51b7662SDaniel Vetter if (curr->type == AGP_PHYS_MEMORY) { 280f51b7662SDaniel Vetter if (curr->page_count == 4) 281f51b7662SDaniel Vetter i8xx_destroy_pages(curr->pages[0]); 282f51b7662SDaniel Vetter else { 283f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 284f51b7662SDaniel Vetter AGP_PAGE_DESTROY_UNMAP); 285f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 286f51b7662SDaniel Vetter AGP_PAGE_DESTROY_FREE); 287f51b7662SDaniel Vetter } 288f51b7662SDaniel Vetter agp_free_page_array(curr); 289f51b7662SDaniel Vetter } 290f51b7662SDaniel Vetter kfree(curr); 291f51b7662SDaniel Vetter } 29200fe639aSVille Syrjälä #endif 293f51b7662SDaniel Vetter 2940e87d2b0SDaniel Vetter static int intel_gtt_setup_scratch_page(void) 2950e87d2b0SDaniel Vetter { 2960e87d2b0SDaniel Vetter struct page *page; 2970e87d2b0SDaniel Vetter dma_addr_t dma_addr; 2980e87d2b0SDaniel Vetter 2990e87d2b0SDaniel Vetter page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); 3000e87d2b0SDaniel Vetter if (page == NULL) 3010e87d2b0SDaniel Vetter return -ENOMEM; 3020e87d2b0SDaniel Vetter set_pages_uc(page, 1); 3030e87d2b0SDaniel Vetter 3048d2e6308SBen Widawsky if (intel_private.needs_dmar) { 3050e87d2b0SDaniel Vetter dma_addr = pci_map_page(intel_private.pcidev, page, 0, 3060e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 3070e87d2b0SDaniel Vetter if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) 3080e87d2b0SDaniel Vetter return -EINVAL; 3090e87d2b0SDaniel Vetter 3109c61a32dSBen Widawsky intel_private.scratch_page_dma = dma_addr; 3110e87d2b0SDaniel Vetter } else 3129c61a32dSBen Widawsky intel_private.scratch_page_dma = page_to_phys(page); 3130e87d2b0SDaniel Vetter 3140e87d2b0SDaniel Vetter intel_private.scratch_page = page; 3150e87d2b0SDaniel Vetter 3160e87d2b0SDaniel Vetter return 0; 3170e87d2b0SDaniel Vetter } 3180e87d2b0SDaniel Vetter 319625dd9d3SDaniel Vetter static void i810_write_entry(dma_addr_t addr, unsigned int entry, 320625dd9d3SDaniel Vetter unsigned int flags) 321625dd9d3SDaniel Vetter { 322625dd9d3SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 323625dd9d3SDaniel Vetter 324625dd9d3SDaniel Vetter switch (flags) { 325625dd9d3SDaniel Vetter case AGP_DCACHE_MEMORY: 326625dd9d3SDaniel Vetter pte_flags |= I810_PTE_LOCAL; 327625dd9d3SDaniel Vetter break; 328625dd9d3SDaniel Vetter case AGP_USER_CACHED_MEMORY: 329625dd9d3SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 330625dd9d3SDaniel Vetter break; 331625dd9d3SDaniel Vetter } 332625dd9d3SDaniel Vetter 333983d308cSChris Wilson writel_relaxed(addr | pte_flags, intel_private.gtt + entry); 334625dd9d3SDaniel Vetter } 335625dd9d3SDaniel Vetter 336*b7128ef1SMatthew Auld static resource_size_t intel_gtt_stolen_size(void) 337f51b7662SDaniel Vetter { 338f51b7662SDaniel Vetter u16 gmch_ctrl; 339f51b7662SDaniel Vetter u8 rdct; 340f51b7662SDaniel Vetter int local = 0; 341f51b7662SDaniel Vetter static const int ddt[4] = { 0, 16, 32, 64 }; 342*b7128ef1SMatthew Auld resource_size_t stolen_size = 0; 343f51b7662SDaniel Vetter 344820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) 345820647b9SDaniel Vetter return 0; /* no stolen mem on i81x */ 346820647b9SDaniel Vetter 347d7cca2f7SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 348d7cca2f7SDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 349f51b7662SDaniel Vetter 350d7cca2f7SDaniel Vetter if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || 351d7cca2f7SDaniel Vetter intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { 352f51b7662SDaniel Vetter switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 353f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_512: 354d8d9abcdSDaniel Vetter stolen_size = KB(512); 355f51b7662SDaniel Vetter break; 356f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_1024: 357d8d9abcdSDaniel Vetter stolen_size = MB(1); 358f51b7662SDaniel Vetter break; 359f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_8192: 360d8d9abcdSDaniel Vetter stolen_size = MB(8); 361f51b7662SDaniel Vetter break; 362f51b7662SDaniel Vetter case I830_GMCH_GMS_LOCAL: 363f51b7662SDaniel Vetter rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE); 364d8d9abcdSDaniel Vetter stolen_size = (I830_RDRAM_ND(rdct) + 1) * 365f51b7662SDaniel Vetter MB(ddt[I830_RDRAM_DDT(rdct)]); 366f51b7662SDaniel Vetter local = 1; 367f51b7662SDaniel Vetter break; 368f51b7662SDaniel Vetter default: 369d8d9abcdSDaniel Vetter stolen_size = 0; 370f51b7662SDaniel Vetter break; 371f51b7662SDaniel Vetter } 372f51b7662SDaniel Vetter } else { 373f51b7662SDaniel Vetter switch (gmch_ctrl & I855_GMCH_GMS_MASK) { 374f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_1M: 375d8d9abcdSDaniel Vetter stolen_size = MB(1); 376f51b7662SDaniel Vetter break; 377f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_4M: 378d8d9abcdSDaniel Vetter stolen_size = MB(4); 379f51b7662SDaniel Vetter break; 380f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_8M: 381d8d9abcdSDaniel Vetter stolen_size = MB(8); 382f51b7662SDaniel Vetter break; 383f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_16M: 384d8d9abcdSDaniel Vetter stolen_size = MB(16); 385f51b7662SDaniel Vetter break; 386f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_32M: 387d8d9abcdSDaniel Vetter stolen_size = MB(32); 388f51b7662SDaniel Vetter break; 389f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_48M: 390d8d9abcdSDaniel Vetter stolen_size = MB(48); 391f51b7662SDaniel Vetter break; 392f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_64M: 393d8d9abcdSDaniel Vetter stolen_size = MB(64); 394f51b7662SDaniel Vetter break; 395f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_128M: 396d8d9abcdSDaniel Vetter stolen_size = MB(128); 397f51b7662SDaniel Vetter break; 398f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_256M: 399d8d9abcdSDaniel Vetter stolen_size = MB(256); 400f51b7662SDaniel Vetter break; 401f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_96M: 402d8d9abcdSDaniel Vetter stolen_size = MB(96); 403f51b7662SDaniel Vetter break; 404f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_160M: 405d8d9abcdSDaniel Vetter stolen_size = MB(160); 406f51b7662SDaniel Vetter break; 407f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_224M: 408d8d9abcdSDaniel Vetter stolen_size = MB(224); 409f51b7662SDaniel Vetter break; 410f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_352M: 411d8d9abcdSDaniel Vetter stolen_size = MB(352); 412f51b7662SDaniel Vetter break; 413f51b7662SDaniel Vetter default: 414d8d9abcdSDaniel Vetter stolen_size = 0; 415f51b7662SDaniel Vetter break; 416f51b7662SDaniel Vetter } 417f51b7662SDaniel Vetter } 4181784a5fbSDaniel Vetter 4191b6064d7SChris Wilson if (stolen_size > 0) { 420*b7128ef1SMatthew Auld dev_info(&intel_private.bridge_dev->dev, "detected %lluK %s memory\n", 421*b7128ef1SMatthew Auld (u64)stolen_size / KB(1), local ? "local" : "stolen"); 422f51b7662SDaniel Vetter } else { 423d7cca2f7SDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 424f51b7662SDaniel Vetter "no pre-allocated video memory detected\n"); 425d8d9abcdSDaniel Vetter stolen_size = 0; 426f51b7662SDaniel Vetter } 427f51b7662SDaniel Vetter 428c64f7ba5SChris Wilson return stolen_size; 429f51b7662SDaniel Vetter } 430f51b7662SDaniel Vetter 43120172842SDaniel Vetter static void i965_adjust_pgetbl_size(unsigned int size_flag) 43220172842SDaniel Vetter { 43320172842SDaniel Vetter u32 pgetbl_ctl, pgetbl_ctl2; 43420172842SDaniel Vetter 43520172842SDaniel Vetter /* ensure that ppgtt is disabled */ 43620172842SDaniel Vetter pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2); 43720172842SDaniel Vetter pgetbl_ctl2 &= ~I810_PGETBL_ENABLED; 43820172842SDaniel Vetter writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2); 43920172842SDaniel Vetter 44020172842SDaniel Vetter /* write the new ggtt size */ 44120172842SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 44220172842SDaniel Vetter pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK; 44320172842SDaniel Vetter pgetbl_ctl |= size_flag; 44420172842SDaniel Vetter writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL); 44520172842SDaniel Vetter } 44620172842SDaniel Vetter 44720172842SDaniel Vetter static unsigned int i965_gtt_total_entries(void) 448fbe40783SDaniel Vetter { 449fbe40783SDaniel Vetter int size; 450fbe40783SDaniel Vetter u32 pgetbl_ctl; 45120172842SDaniel Vetter u16 gmch_ctl; 45220172842SDaniel Vetter 45320172842SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 45420172842SDaniel Vetter I830_GMCH_CTRL, &gmch_ctl); 45520172842SDaniel Vetter 45620172842SDaniel Vetter if (INTEL_GTT_GEN == 5) { 45720172842SDaniel Vetter switch (gmch_ctl & G4x_GMCH_SIZE_MASK) { 45820172842SDaniel Vetter case G4x_GMCH_SIZE_1M: 45920172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1M: 46020172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB); 46120172842SDaniel Vetter break; 46220172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1_5M: 46320172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB); 46420172842SDaniel Vetter break; 46520172842SDaniel Vetter case G4x_GMCH_SIZE_2M: 46620172842SDaniel Vetter case G4x_GMCH_SIZE_VT_2M: 46720172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB); 46820172842SDaniel Vetter break; 46920172842SDaniel Vetter } 47020172842SDaniel Vetter } 47120172842SDaniel Vetter 472fbe40783SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 473fbe40783SDaniel Vetter 474fbe40783SDaniel Vetter switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { 475fbe40783SDaniel Vetter case I965_PGETBL_SIZE_128KB: 476e5e408fcSDaniel Vetter size = KB(128); 477fbe40783SDaniel Vetter break; 478fbe40783SDaniel Vetter case I965_PGETBL_SIZE_256KB: 479e5e408fcSDaniel Vetter size = KB(256); 480fbe40783SDaniel Vetter break; 481fbe40783SDaniel Vetter case I965_PGETBL_SIZE_512KB: 482e5e408fcSDaniel Vetter size = KB(512); 483fbe40783SDaniel Vetter break; 48420172842SDaniel Vetter /* GTT pagetable sizes bigger than 512KB are not possible on G33! */ 485fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1MB: 486e5e408fcSDaniel Vetter size = KB(1024); 487fbe40783SDaniel Vetter break; 488fbe40783SDaniel Vetter case I965_PGETBL_SIZE_2MB: 489e5e408fcSDaniel Vetter size = KB(2048); 490fbe40783SDaniel Vetter break; 491fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1_5MB: 492e5e408fcSDaniel Vetter size = KB(1024 + 512); 493fbe40783SDaniel Vetter break; 494fbe40783SDaniel Vetter default: 495fbe40783SDaniel Vetter dev_info(&intel_private.pcidev->dev, 496fbe40783SDaniel Vetter "unknown page table size, assuming 512KB\n"); 497e5e408fcSDaniel Vetter size = KB(512); 498fbe40783SDaniel Vetter } 499e5e408fcSDaniel Vetter 500e5e408fcSDaniel Vetter return size/4; 50120172842SDaniel Vetter } 50220172842SDaniel Vetter 50320172842SDaniel Vetter static unsigned int intel_gtt_total_entries(void) 50420172842SDaniel Vetter { 50520172842SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) 50620172842SDaniel Vetter return i965_gtt_total_entries(); 507009946f8SBen Widawsky else { 508fbe40783SDaniel Vetter /* On previous hardware, the GTT size was just what was 509fbe40783SDaniel Vetter * required to map the aperture. 510fbe40783SDaniel Vetter */ 511a54c0c27SBen Widawsky return intel_private.gtt_mappable_entries; 512fbe40783SDaniel Vetter } 513fbe40783SDaniel Vetter } 514fbe40783SDaniel Vetter 5151784a5fbSDaniel Vetter static unsigned int intel_gtt_mappable_entries(void) 5161784a5fbSDaniel Vetter { 5171784a5fbSDaniel Vetter unsigned int aperture_size; 5181784a5fbSDaniel Vetter 519820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) { 520820647b9SDaniel Vetter u32 smram_miscc; 521820647b9SDaniel Vetter 522820647b9SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, 523820647b9SDaniel Vetter I810_SMRAM_MISCC, &smram_miscc); 524820647b9SDaniel Vetter 525820647b9SDaniel Vetter if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) 526820647b9SDaniel Vetter == I810_GFX_MEM_WIN_32M) 527820647b9SDaniel Vetter aperture_size = MB(32); 528820647b9SDaniel Vetter else 529820647b9SDaniel Vetter aperture_size = MB(64); 530820647b9SDaniel Vetter } else if (INTEL_GTT_GEN == 2) { 531b1c5b0f8SChris Wilson u16 gmch_ctrl; 5321784a5fbSDaniel Vetter 5331784a5fbSDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 5341784a5fbSDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 5351784a5fbSDaniel Vetter 5361784a5fbSDaniel Vetter if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M) 537b1c5b0f8SChris Wilson aperture_size = MB(64); 5381784a5fbSDaniel Vetter else 539b1c5b0f8SChris Wilson aperture_size = MB(128); 540239918f7SDaniel Vetter } else { 5411784a5fbSDaniel Vetter /* 9xx supports large sizes, just look at the length */ 5421784a5fbSDaniel Vetter aperture_size = pci_resource_len(intel_private.pcidev, 2); 5431784a5fbSDaniel Vetter } 5441784a5fbSDaniel Vetter 5451784a5fbSDaniel Vetter return aperture_size >> PAGE_SHIFT; 5461784a5fbSDaniel Vetter } 5471784a5fbSDaniel Vetter 5480e87d2b0SDaniel Vetter static void intel_gtt_teardown_scratch_page(void) 5490e87d2b0SDaniel Vetter { 5500e87d2b0SDaniel Vetter set_pages_wb(intel_private.scratch_page, 1); 5519f5ac8edSDaniel Vetter if (intel_private.needs_dmar) 5529f5ac8edSDaniel Vetter pci_unmap_page(intel_private.pcidev, 5539f5ac8edSDaniel Vetter intel_private.scratch_page_dma, 5540e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 5550e87d2b0SDaniel Vetter __free_page(intel_private.scratch_page); 5560e87d2b0SDaniel Vetter } 5570e87d2b0SDaniel Vetter 5580e87d2b0SDaniel Vetter static void intel_gtt_cleanup(void) 5590e87d2b0SDaniel Vetter { 560ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 561ae83dd5cSDaniel Vetter 5620e87d2b0SDaniel Vetter iounmap(intel_private.gtt); 5630e87d2b0SDaniel Vetter iounmap(intel_private.registers); 5640e87d2b0SDaniel Vetter 5650e87d2b0SDaniel Vetter intel_gtt_teardown_scratch_page(); 5660e87d2b0SDaniel Vetter } 5670e87d2b0SDaniel Vetter 568da88a5f7SChris Wilson /* Certain Gen5 chipsets require require idling the GPU before 569da88a5f7SChris Wilson * unmapping anything from the GTT when VT-d is enabled. 570da88a5f7SChris Wilson */ 571da88a5f7SChris Wilson static inline int needs_ilk_vtd_wa(void) 572da88a5f7SChris Wilson { 573da88a5f7SChris Wilson #ifdef CONFIG_INTEL_IOMMU 574da88a5f7SChris Wilson const unsigned short gpu_devid = intel_private.pcidev->device; 575da88a5f7SChris Wilson 576da88a5f7SChris Wilson /* Query intel_iommu to see if we need the workaround. Presumably that 577da88a5f7SChris Wilson * was loaded first. 578da88a5f7SChris Wilson */ 5798b572a42SChris Wilson if ((gpu_devid == PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG || 580da88a5f7SChris Wilson gpu_devid == PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG) && 581da88a5f7SChris Wilson intel_iommu_gfx_mapped) 582da88a5f7SChris Wilson return 1; 583da88a5f7SChris Wilson #endif 584da88a5f7SChris Wilson return 0; 585da88a5f7SChris Wilson } 586da88a5f7SChris Wilson 587da88a5f7SChris Wilson static bool intel_gtt_can_wc(void) 588da88a5f7SChris Wilson { 589da88a5f7SChris Wilson if (INTEL_GTT_GEN <= 2) 590da88a5f7SChris Wilson return false; 591da88a5f7SChris Wilson 592da88a5f7SChris Wilson if (INTEL_GTT_GEN >= 6) 593da88a5f7SChris Wilson return false; 594da88a5f7SChris Wilson 595da88a5f7SChris Wilson /* Reports of major corruption with ILK vt'd enabled */ 596da88a5f7SChris Wilson if (needs_ilk_vtd_wa()) 597da88a5f7SChris Wilson return false; 598da88a5f7SChris Wilson 599da88a5f7SChris Wilson return true; 600da88a5f7SChris Wilson } 601da88a5f7SChris Wilson 6021784a5fbSDaniel Vetter static int intel_gtt_init(void) 6031784a5fbSDaniel Vetter { 604f67eab66SDaniel Vetter u32 gtt_map_size; 605545b0a74SYinghai Lu int ret, bar; 6063b15a9d7SDaniel Vetter 6073b15a9d7SDaniel Vetter ret = intel_private.driver->setup(); 6083b15a9d7SDaniel Vetter if (ret != 0) 6093b15a9d7SDaniel Vetter return ret; 610f67eab66SDaniel Vetter 611a54c0c27SBen Widawsky intel_private.gtt_mappable_entries = intel_gtt_mappable_entries(); 612a54c0c27SBen Widawsky intel_private.gtt_total_entries = intel_gtt_total_entries(); 613f67eab66SDaniel Vetter 614b3eafc5aSDaniel Vetter /* save the PGETBL reg for resume */ 615b3eafc5aSDaniel Vetter intel_private.PGETBL_save = 616b3eafc5aSDaniel Vetter readl(intel_private.registers+I810_PGETBL_CTL) 617b3eafc5aSDaniel Vetter & ~I810_PGETBL_ENABLED; 618100519e2SChris Wilson /* we only ever restore the register when enabling the PGTBL... */ 619100519e2SChris Wilson if (HAS_PGTBL_EN) 620100519e2SChris Wilson intel_private.PGETBL_save |= I810_PGETBL_ENABLED; 621b3eafc5aSDaniel Vetter 6220af9e92eSDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 6230af9e92eSDaniel Vetter "detected gtt size: %dK total, %dK mappable\n", 624a54c0c27SBen Widawsky intel_private.gtt_total_entries * 4, 625a54c0c27SBen Widawsky intel_private.gtt_mappable_entries * 4); 6260af9e92eSDaniel Vetter 627a54c0c27SBen Widawsky gtt_map_size = intel_private.gtt_total_entries * 4; 628f67eab66SDaniel Vetter 629edef7e68SChris Wilson intel_private.gtt = NULL; 630da88a5f7SChris Wilson if (intel_gtt_can_wc()) 6315acc4ce4SBjorn Helgaas intel_private.gtt = ioremap_wc(intel_private.gtt_phys_addr, 632edef7e68SChris Wilson gtt_map_size); 633edef7e68SChris Wilson if (intel_private.gtt == NULL) 6345acc4ce4SBjorn Helgaas intel_private.gtt = ioremap(intel_private.gtt_phys_addr, 635f67eab66SDaniel Vetter gtt_map_size); 636edef7e68SChris Wilson if (intel_private.gtt == NULL) { 637ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 638f67eab66SDaniel Vetter iounmap(intel_private.registers); 639f67eab66SDaniel Vetter return -ENOMEM; 640f67eab66SDaniel Vetter } 641f67eab66SDaniel Vetter 64200fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 643f67eab66SDaniel Vetter global_cache_flush(); /* FIXME: ? */ 64400fe639aSVille Syrjälä #endif 645f67eab66SDaniel Vetter 646a54c0c27SBen Widawsky intel_private.stolen_size = intel_gtt_stolen_size(); 6471784a5fbSDaniel Vetter 6488d2e6308SBen Widawsky intel_private.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2; 649a46f3108SDave Airlie 6500e87d2b0SDaniel Vetter ret = intel_gtt_setup_scratch_page(); 6510e87d2b0SDaniel Vetter if (ret != 0) { 6520e87d2b0SDaniel Vetter intel_gtt_cleanup(); 6530e87d2b0SDaniel Vetter return ret; 6540e87d2b0SDaniel Vetter } 6550e87d2b0SDaniel Vetter 65632e3cd6eSDaniel Vetter if (INTEL_GTT_GEN <= 2) 657545b0a74SYinghai Lu bar = I810_GMADR_BAR; 65832e3cd6eSDaniel Vetter else 659545b0a74SYinghai Lu bar = I915_GMADR_BAR; 66032e3cd6eSDaniel Vetter 661545b0a74SYinghai Lu intel_private.gma_bus_addr = pci_bus_address(intel_private.pcidev, bar); 6621784a5fbSDaniel Vetter return 0; 6631784a5fbSDaniel Vetter } 6641784a5fbSDaniel Vetter 66500fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 66662fa0ce2SChris Wilson static const struct aper_size_info_fixed intel_fake_agp_sizes[] = { 66762fa0ce2SChris Wilson {32, 8192, 3}, 66862fa0ce2SChris Wilson {64, 16384, 4}, 66962fa0ce2SChris Wilson {128, 32768, 5}, 67062fa0ce2SChris Wilson {256, 65536, 6}, 67162fa0ce2SChris Wilson {512, 131072, 7}, 67262fa0ce2SChris Wilson }; 67362fa0ce2SChris Wilson 6743e921f98SDaniel Vetter static int intel_fake_agp_fetch_size(void) 6753e921f98SDaniel Vetter { 6769e76e7b8SChris Wilson int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes); 6773e921f98SDaniel Vetter unsigned int aper_size; 6783e921f98SDaniel Vetter int i; 6793e921f98SDaniel Vetter 680a54c0c27SBen Widawsky aper_size = (intel_private.gtt_mappable_entries << PAGE_SHIFT) / MB(1); 6813e921f98SDaniel Vetter 6823e921f98SDaniel Vetter for (i = 0; i < num_sizes; i++) { 683ffdd7510SDaniel Vetter if (aper_size == intel_fake_agp_sizes[i].size) { 6849e76e7b8SChris Wilson agp_bridge->current_size = 6859e76e7b8SChris Wilson (void *) (intel_fake_agp_sizes + i); 6863e921f98SDaniel Vetter return aper_size; 6873e921f98SDaniel Vetter } 6883e921f98SDaniel Vetter } 6893e921f98SDaniel Vetter 6903e921f98SDaniel Vetter return 0; 6913e921f98SDaniel Vetter } 69200fe639aSVille Syrjälä #endif 6933e921f98SDaniel Vetter 694ae83dd5cSDaniel Vetter static void i830_cleanup(void) 695f51b7662SDaniel Vetter { 696f51b7662SDaniel Vetter } 697f51b7662SDaniel Vetter 698f51b7662SDaniel Vetter /* The chipset_flush interface needs to get data that has already been 699f51b7662SDaniel Vetter * flushed out of the CPU all the way out to main memory, because the GPU 700f51b7662SDaniel Vetter * doesn't snoop those buffers. 701f51b7662SDaniel Vetter * 702f51b7662SDaniel Vetter * The 8xx series doesn't have the same lovely interface for flushing the 703f51b7662SDaniel Vetter * chipset write buffers that the later chips do. According to the 865 704f51b7662SDaniel Vetter * specs, it's 64 octwords, or 1KB. So, to get those previous things in 705f51b7662SDaniel Vetter * that buffer out, we just fill 1KB and clflush it out, on the assumption 706f51b7662SDaniel Vetter * that it'll push whatever was in there out. It appears to work. 707f51b7662SDaniel Vetter */ 7081b263f24SDaniel Vetter static void i830_chipset_flush(void) 709f51b7662SDaniel Vetter { 710bdb8b975SChris Wilson unsigned long timeout = jiffies + msecs_to_jiffies(1000); 711f51b7662SDaniel Vetter 712bdb8b975SChris Wilson /* Forcibly evict everything from the CPU write buffers. 713bdb8b975SChris Wilson * clflush appears to be insufficient. 714bdb8b975SChris Wilson */ 715bdb8b975SChris Wilson wbinvd_on_all_cpus(); 716f51b7662SDaniel Vetter 717bdb8b975SChris Wilson /* Now we've only seen documents for this magic bit on 855GM, 718bdb8b975SChris Wilson * we hope it exists for the other gen2 chipsets... 719bdb8b975SChris Wilson * 720bdb8b975SChris Wilson * Also works as advertised on my 845G. 721bdb8b975SChris Wilson */ 722bdb8b975SChris Wilson writel(readl(intel_private.registers+I830_HIC) | (1<<31), 723bdb8b975SChris Wilson intel_private.registers+I830_HIC); 724bdb8b975SChris Wilson 725bdb8b975SChris Wilson while (readl(intel_private.registers+I830_HIC) & (1<<31)) { 726bdb8b975SChris Wilson if (time_after(jiffies, timeout)) 727bdb8b975SChris Wilson break; 728bdb8b975SChris Wilson 729bdb8b975SChris Wilson udelay(50); 730bdb8b975SChris Wilson } 731f51b7662SDaniel Vetter } 732f51b7662SDaniel Vetter 733351bb278SDaniel Vetter static void i830_write_entry(dma_addr_t addr, unsigned int entry, 734351bb278SDaniel Vetter unsigned int flags) 735351bb278SDaniel Vetter { 736351bb278SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 737351bb278SDaniel Vetter 738b47cf66fSDaniel Vetter if (flags == AGP_USER_CACHED_MEMORY) 739351bb278SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 740351bb278SDaniel Vetter 741983d308cSChris Wilson writel_relaxed(addr | pte_flags, intel_private.gtt + entry); 742351bb278SDaniel Vetter } 743351bb278SDaniel Vetter 7448ecd1a66SDaniel Vetter bool intel_enable_gtt(void) 74573800422SDaniel Vetter { 746e380f60bSChris Wilson u8 __iomem *reg; 74773800422SDaniel Vetter 748100519e2SChris Wilson if (INTEL_GTT_GEN == 2) { 749100519e2SChris Wilson u16 gmch_ctrl; 750100519e2SChris Wilson 751e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 752e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 753e380f60bSChris Wilson gmch_ctrl |= I830_GMCH_ENABLED; 754e380f60bSChris Wilson pci_write_config_word(intel_private.bridge_dev, 755e380f60bSChris Wilson I830_GMCH_CTRL, gmch_ctrl); 756e380f60bSChris Wilson 757e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 758e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 759e380f60bSChris Wilson if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) { 760e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 761e380f60bSChris Wilson "failed to enable the GTT: GMCH_CTRL=%x\n", 762e380f60bSChris Wilson gmch_ctrl); 763e380f60bSChris Wilson return false; 764e380f60bSChris Wilson } 765100519e2SChris Wilson } 766e380f60bSChris Wilson 767c97689d8SChris Wilson /* On the resume path we may be adjusting the PGTBL value, so 768c97689d8SChris Wilson * be paranoid and flush all chipset write buffers... 769c97689d8SChris Wilson */ 770c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 771c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 772c97689d8SChris Wilson 773e380f60bSChris Wilson reg = intel_private.registers+I810_PGETBL_CTL; 774100519e2SChris Wilson writel(intel_private.PGETBL_save, reg); 775100519e2SChris Wilson if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) { 776e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 777100519e2SChris Wilson "failed to enable the GTT: PGETBL=%x [expected %x]\n", 778e380f60bSChris Wilson readl(reg), intel_private.PGETBL_save); 779e380f60bSChris Wilson return false; 780e380f60bSChris Wilson } 781e380f60bSChris Wilson 782c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 783c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 784c97689d8SChris Wilson 785e380f60bSChris Wilson return true; 78673800422SDaniel Vetter } 7878ecd1a66SDaniel Vetter EXPORT_SYMBOL(intel_enable_gtt); 78873800422SDaniel Vetter 78973800422SDaniel Vetter static int i830_setup(void) 79073800422SDaniel Vetter { 791d3572532SBjorn Helgaas phys_addr_t reg_addr; 79273800422SDaniel Vetter 793d3572532SBjorn Helgaas reg_addr = pci_resource_start(intel_private.pcidev, I810_MMADR_BAR); 79473800422SDaniel Vetter 79573800422SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 79673800422SDaniel Vetter if (!intel_private.registers) 79773800422SDaniel Vetter return -ENOMEM; 79873800422SDaniel Vetter 7995acc4ce4SBjorn Helgaas intel_private.gtt_phys_addr = reg_addr + I810_PTE_BASE; 80073800422SDaniel Vetter 80173800422SDaniel Vetter return 0; 80273800422SDaniel Vetter } 80373800422SDaniel Vetter 80400fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 8053b15a9d7SDaniel Vetter static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge) 806f51b7662SDaniel Vetter { 80773800422SDaniel Vetter agp_bridge->gatt_table_real = NULL; 808f51b7662SDaniel Vetter agp_bridge->gatt_table = NULL; 80973800422SDaniel Vetter agp_bridge->gatt_bus_addr = 0; 810f51b7662SDaniel Vetter 811f51b7662SDaniel Vetter return 0; 812f51b7662SDaniel Vetter } 813f51b7662SDaniel Vetter 814ffdd7510SDaniel Vetter static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge) 815f51b7662SDaniel Vetter { 816f51b7662SDaniel Vetter return 0; 817f51b7662SDaniel Vetter } 818f51b7662SDaniel Vetter 819351bb278SDaniel Vetter static int intel_fake_agp_configure(void) 820f51b7662SDaniel Vetter { 821e380f60bSChris Wilson if (!intel_enable_gtt()) 822e380f60bSChris Wilson return -EIO; 823f51b7662SDaniel Vetter 824bee4a186SChris Wilson intel_private.clear_fake_agp = true; 825e5c65377SBen Widawsky agp_bridge->gart_bus_addr = intel_private.gma_bus_addr; 826f51b7662SDaniel Vetter 827f51b7662SDaniel Vetter return 0; 828f51b7662SDaniel Vetter } 82900fe639aSVille Syrjälä #endif 830f51b7662SDaniel Vetter 8315cbecafcSDaniel Vetter static bool i830_check_flags(unsigned int flags) 832f51b7662SDaniel Vetter { 8335cbecafcSDaniel Vetter switch (flags) { 8345cbecafcSDaniel Vetter case 0: 8355cbecafcSDaniel Vetter case AGP_PHYS_MEMORY: 8365cbecafcSDaniel Vetter case AGP_USER_CACHED_MEMORY: 8375cbecafcSDaniel Vetter case AGP_USER_MEMORY: 8385cbecafcSDaniel Vetter return true; 8395cbecafcSDaniel Vetter } 8405cbecafcSDaniel Vetter 8415cbecafcSDaniel Vetter return false; 8425cbecafcSDaniel Vetter } 8435cbecafcSDaniel Vetter 844d6473f56SChris Wilson void intel_gtt_insert_page(dma_addr_t addr, 845d6473f56SChris Wilson unsigned int pg, 846d6473f56SChris Wilson unsigned int flags) 847d6473f56SChris Wilson { 848d6473f56SChris Wilson intel_private.driver->write_entry(addr, pg, flags); 8493497971aSChris Wilson if (intel_private.driver->chipset_flush) 8503497971aSChris Wilson intel_private.driver->chipset_flush(); 851d6473f56SChris Wilson } 852d6473f56SChris Wilson EXPORT_SYMBOL(intel_gtt_insert_page); 853d6473f56SChris Wilson 8549da3da66SChris Wilson void intel_gtt_insert_sg_entries(struct sg_table *st, 855fefaa70fSDaniel Vetter unsigned int pg_start, 856fefaa70fSDaniel Vetter unsigned int flags) 857fefaa70fSDaniel Vetter { 858fefaa70fSDaniel Vetter struct scatterlist *sg; 859fefaa70fSDaniel Vetter unsigned int len, m; 860fefaa70fSDaniel Vetter int i, j; 861fefaa70fSDaniel Vetter 862fefaa70fSDaniel Vetter j = pg_start; 863fefaa70fSDaniel Vetter 864fefaa70fSDaniel Vetter /* sg may merge pages, but we have to separate 865fefaa70fSDaniel Vetter * per-page addr for GTT */ 8669da3da66SChris Wilson for_each_sg(st->sgl, sg, st->nents, i) { 867fefaa70fSDaniel Vetter len = sg_dma_len(sg) >> PAGE_SHIFT; 868fefaa70fSDaniel Vetter for (m = 0; m < len; m++) { 869fefaa70fSDaniel Vetter dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT); 8709da3da66SChris Wilson intel_private.driver->write_entry(addr, j, flags); 871fefaa70fSDaniel Vetter j++; 872fefaa70fSDaniel Vetter } 873fefaa70fSDaniel Vetter } 874983d308cSChris Wilson wmb(); 8758516673aSChris Wilson if (intel_private.driver->chipset_flush) 8768516673aSChris Wilson intel_private.driver->chipset_flush(); 877fefaa70fSDaniel Vetter } 8784080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_insert_sg_entries); 8794080775bSDaniel Vetter 88000fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 8819da3da66SChris Wilson static void intel_gtt_insert_pages(unsigned int first_entry, 8829da3da66SChris Wilson unsigned int num_entries, 8839da3da66SChris Wilson struct page **pages, 8849da3da66SChris Wilson unsigned int flags) 8854080775bSDaniel Vetter { 8864080775bSDaniel Vetter int i, j; 8874080775bSDaniel Vetter 8884080775bSDaniel Vetter for (i = 0, j = first_entry; i < num_entries; i++, j++) { 8894080775bSDaniel Vetter dma_addr_t addr = page_to_phys(pages[i]); 8904080775bSDaniel Vetter intel_private.driver->write_entry(addr, 8914080775bSDaniel Vetter j, flags); 8924080775bSDaniel Vetter } 893983d308cSChris Wilson wmb(); 8944080775bSDaniel Vetter } 895fefaa70fSDaniel Vetter 8965cbecafcSDaniel Vetter static int intel_fake_agp_insert_entries(struct agp_memory *mem, 8975cbecafcSDaniel Vetter off_t pg_start, int type) 8985cbecafcSDaniel Vetter { 899f51b7662SDaniel Vetter int ret = -EINVAL; 900f51b7662SDaniel Vetter 901bee4a186SChris Wilson if (intel_private.clear_fake_agp) { 902a54c0c27SBen Widawsky int start = intel_private.stolen_size / PAGE_SIZE; 903a54c0c27SBen Widawsky int end = intel_private.gtt_mappable_entries; 904bee4a186SChris Wilson intel_gtt_clear_range(start, end - start); 905bee4a186SChris Wilson intel_private.clear_fake_agp = false; 906bee4a186SChris Wilson } 907bee4a186SChris Wilson 908ff26860fSDaniel Vetter if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY) 909ff26860fSDaniel Vetter return i810_insert_dcache_entries(mem, pg_start, type); 910ff26860fSDaniel Vetter 911f51b7662SDaniel Vetter if (mem->page_count == 0) 912f51b7662SDaniel Vetter goto out; 913f51b7662SDaniel Vetter 914a54c0c27SBen Widawsky if (pg_start + mem->page_count > intel_private.gtt_total_entries) 915f51b7662SDaniel Vetter goto out_err; 916f51b7662SDaniel Vetter 917f51b7662SDaniel Vetter if (type != mem->type) 918f51b7662SDaniel Vetter goto out_err; 919f51b7662SDaniel Vetter 9205cbecafcSDaniel Vetter if (!intel_private.driver->check_flags(type)) 921f51b7662SDaniel Vetter goto out_err; 922f51b7662SDaniel Vetter 923f51b7662SDaniel Vetter if (!mem->is_flushed) 924f51b7662SDaniel Vetter global_cache_flush(); 925f51b7662SDaniel Vetter 9268d2e6308SBen Widawsky if (intel_private.needs_dmar) { 9279da3da66SChris Wilson struct sg_table st; 9289da3da66SChris Wilson 9299da3da66SChris Wilson ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st); 930fefaa70fSDaniel Vetter if (ret != 0) 931fefaa70fSDaniel Vetter return ret; 932fefaa70fSDaniel Vetter 9339da3da66SChris Wilson intel_gtt_insert_sg_entries(&st, pg_start, type); 9349da3da66SChris Wilson mem->sg_list = st.sgl; 9359da3da66SChris Wilson mem->num_sg = st.nents; 9364080775bSDaniel Vetter } else 9374080775bSDaniel Vetter intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages, 9384080775bSDaniel Vetter type); 939f51b7662SDaniel Vetter 940f51b7662SDaniel Vetter out: 941f51b7662SDaniel Vetter ret = 0; 942f51b7662SDaniel Vetter out_err: 943f51b7662SDaniel Vetter mem->is_flushed = true; 944f51b7662SDaniel Vetter return ret; 945f51b7662SDaniel Vetter } 94600fe639aSVille Syrjälä #endif 947f51b7662SDaniel Vetter 9484080775bSDaniel Vetter void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries) 949f51b7662SDaniel Vetter { 9504080775bSDaniel Vetter unsigned int i; 951f51b7662SDaniel Vetter 9524080775bSDaniel Vetter for (i = first_entry; i < (first_entry + num_entries); i++) { 9539c61a32dSBen Widawsky intel_private.driver->write_entry(intel_private.scratch_page_dma, 9545cbecafcSDaniel Vetter i, 0); 955f51b7662SDaniel Vetter } 956983d308cSChris Wilson wmb(); 9574080775bSDaniel Vetter } 9584080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_clear_range); 9594080775bSDaniel Vetter 96000fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 9614080775bSDaniel Vetter static int intel_fake_agp_remove_entries(struct agp_memory *mem, 9624080775bSDaniel Vetter off_t pg_start, int type) 9634080775bSDaniel Vetter { 9644080775bSDaniel Vetter if (mem->page_count == 0) 9654080775bSDaniel Vetter return 0; 9664080775bSDaniel Vetter 967d15eda5cSDave Airlie intel_gtt_clear_range(pg_start, mem->page_count); 968d15eda5cSDave Airlie 9698d2e6308SBen Widawsky if (intel_private.needs_dmar) { 9704080775bSDaniel Vetter intel_gtt_unmap_memory(mem->sg_list, mem->num_sg); 9714080775bSDaniel Vetter mem->sg_list = NULL; 9724080775bSDaniel Vetter mem->num_sg = 0; 9734080775bSDaniel Vetter } 9744080775bSDaniel Vetter 975f51b7662SDaniel Vetter return 0; 976f51b7662SDaniel Vetter } 977f51b7662SDaniel Vetter 978ffdd7510SDaniel Vetter static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, 979ffdd7510SDaniel Vetter int type) 980f51b7662SDaniel Vetter { 981625dd9d3SDaniel Vetter struct agp_memory *new; 982625dd9d3SDaniel Vetter 983625dd9d3SDaniel Vetter if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) { 984625dd9d3SDaniel Vetter if (pg_count != intel_private.num_dcache_entries) 985625dd9d3SDaniel Vetter return NULL; 986625dd9d3SDaniel Vetter 987625dd9d3SDaniel Vetter new = agp_create_memory(1); 988625dd9d3SDaniel Vetter if (new == NULL) 989625dd9d3SDaniel Vetter return NULL; 990625dd9d3SDaniel Vetter 991625dd9d3SDaniel Vetter new->type = AGP_DCACHE_MEMORY; 992625dd9d3SDaniel Vetter new->page_count = pg_count; 993625dd9d3SDaniel Vetter new->num_scratch_pages = 0; 994625dd9d3SDaniel Vetter agp_free_page_array(new); 995625dd9d3SDaniel Vetter return new; 996625dd9d3SDaniel Vetter } 997f51b7662SDaniel Vetter if (type == AGP_PHYS_MEMORY) 998f51b7662SDaniel Vetter return alloc_agpphysmem_i8xx(pg_count, type); 999f51b7662SDaniel Vetter /* always return NULL for other allocation types for now */ 1000f51b7662SDaniel Vetter return NULL; 1001f51b7662SDaniel Vetter } 100200fe639aSVille Syrjälä #endif 1003f51b7662SDaniel Vetter 1004f51b7662SDaniel Vetter static int intel_alloc_chipset_flush_resource(void) 1005f51b7662SDaniel Vetter { 1006f51b7662SDaniel Vetter int ret; 1007d7cca2f7SDaniel Vetter ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE, 1008f51b7662SDaniel Vetter PAGE_SIZE, PCIBIOS_MIN_MEM, 0, 1009d7cca2f7SDaniel Vetter pcibios_align_resource, intel_private.bridge_dev); 1010f51b7662SDaniel Vetter 1011f51b7662SDaniel Vetter return ret; 1012f51b7662SDaniel Vetter } 1013f51b7662SDaniel Vetter 1014f51b7662SDaniel Vetter static void intel_i915_setup_chipset_flush(void) 1015f51b7662SDaniel Vetter { 1016f51b7662SDaniel Vetter int ret; 1017f51b7662SDaniel Vetter u32 temp; 1018f51b7662SDaniel Vetter 1019d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp); 1020f51b7662SDaniel Vetter if (!(temp & 0x1)) { 1021f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 1022f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1023d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 1024f51b7662SDaniel Vetter } else { 1025f51b7662SDaniel Vetter temp &= ~1; 1026f51b7662SDaniel Vetter 1027f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1028f51b7662SDaniel Vetter intel_private.ifp_resource.start = temp; 1029f51b7662SDaniel Vetter intel_private.ifp_resource.end = temp + PAGE_SIZE; 1030f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 1031f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 1032f51b7662SDaniel Vetter if (ret) 1033f51b7662SDaniel Vetter intel_private.resource_valid = 0; 1034f51b7662SDaniel Vetter } 1035f51b7662SDaniel Vetter } 1036f51b7662SDaniel Vetter 1037f51b7662SDaniel Vetter static void intel_i965_g33_setup_chipset_flush(void) 1038f51b7662SDaniel Vetter { 1039f51b7662SDaniel Vetter u32 temp_hi, temp_lo; 1040f51b7662SDaniel Vetter int ret; 1041f51b7662SDaniel Vetter 1042d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi); 1043d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo); 1044f51b7662SDaniel Vetter 1045f51b7662SDaniel Vetter if (!(temp_lo & 0x1)) { 1046f51b7662SDaniel Vetter 1047f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 1048f51b7662SDaniel Vetter 1049f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1050d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, 1051f51b7662SDaniel Vetter upper_32_bits(intel_private.ifp_resource.start)); 1052d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 1053f51b7662SDaniel Vetter } else { 1054f51b7662SDaniel Vetter u64 l64; 1055f51b7662SDaniel Vetter 1056f51b7662SDaniel Vetter temp_lo &= ~0x1; 1057f51b7662SDaniel Vetter l64 = ((u64)temp_hi << 32) | temp_lo; 1058f51b7662SDaniel Vetter 1059f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1060f51b7662SDaniel Vetter intel_private.ifp_resource.start = l64; 1061f51b7662SDaniel Vetter intel_private.ifp_resource.end = l64 + PAGE_SIZE; 1062f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 1063f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 1064f51b7662SDaniel Vetter if (ret) 1065f51b7662SDaniel Vetter intel_private.resource_valid = 0; 1066f51b7662SDaniel Vetter } 1067f51b7662SDaniel Vetter } 1068f51b7662SDaniel Vetter 1069f51b7662SDaniel Vetter static void intel_i9xx_setup_flush(void) 1070f51b7662SDaniel Vetter { 1071f51b7662SDaniel Vetter /* return if already configured */ 1072f51b7662SDaniel Vetter if (intel_private.ifp_resource.start) 1073f51b7662SDaniel Vetter return; 1074f51b7662SDaniel Vetter 10751a997ff2SDaniel Vetter if (INTEL_GTT_GEN == 6) 1076f51b7662SDaniel Vetter return; 1077f51b7662SDaniel Vetter 1078f51b7662SDaniel Vetter /* setup a resource for this object */ 1079f51b7662SDaniel Vetter intel_private.ifp_resource.name = "Intel Flush Page"; 1080f51b7662SDaniel Vetter intel_private.ifp_resource.flags = IORESOURCE_MEM; 1081f51b7662SDaniel Vetter 1082f51b7662SDaniel Vetter /* Setup chipset flush for 915 */ 10831a997ff2SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN >= 4) { 1084f51b7662SDaniel Vetter intel_i965_g33_setup_chipset_flush(); 1085f51b7662SDaniel Vetter } else { 1086f51b7662SDaniel Vetter intel_i915_setup_chipset_flush(); 1087f51b7662SDaniel Vetter } 1088f51b7662SDaniel Vetter 1089df51e7aaSChris Wilson if (intel_private.ifp_resource.start) 1090f51b7662SDaniel Vetter intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE); 1091f51b7662SDaniel Vetter if (!intel_private.i9xx_flush_page) 1092df51e7aaSChris Wilson dev_err(&intel_private.pcidev->dev, 1093df51e7aaSChris Wilson "can't ioremap flush page - no chipset flushing\n"); 1094f51b7662SDaniel Vetter } 1095f51b7662SDaniel Vetter 1096ae83dd5cSDaniel Vetter static void i9xx_cleanup(void) 1097ae83dd5cSDaniel Vetter { 1098ae83dd5cSDaniel Vetter if (intel_private.i9xx_flush_page) 1099ae83dd5cSDaniel Vetter iounmap(intel_private.i9xx_flush_page); 1100ae83dd5cSDaniel Vetter if (intel_private.resource_valid) 1101ae83dd5cSDaniel Vetter release_resource(&intel_private.ifp_resource); 1102ae83dd5cSDaniel Vetter intel_private.ifp_resource.start = 0; 1103ae83dd5cSDaniel Vetter intel_private.resource_valid = 0; 1104ae83dd5cSDaniel Vetter } 1105ae83dd5cSDaniel Vetter 11061b263f24SDaniel Vetter static void i9xx_chipset_flush(void) 1107f51b7662SDaniel Vetter { 1108f51b7662SDaniel Vetter if (intel_private.i9xx_flush_page) 1109f51b7662SDaniel Vetter writel(1, intel_private.i9xx_flush_page); 1110f51b7662SDaniel Vetter } 1111f51b7662SDaniel Vetter 111271f45660SChris Wilson static void i965_write_entry(dma_addr_t addr, 111371f45660SChris Wilson unsigned int entry, 1114a6963596SDaniel Vetter unsigned int flags) 1115a6963596SDaniel Vetter { 111671f45660SChris Wilson u32 pte_flags; 111771f45660SChris Wilson 111871f45660SChris Wilson pte_flags = I810_PTE_VALID; 111971f45660SChris Wilson if (flags == AGP_USER_CACHED_MEMORY) 112071f45660SChris Wilson pte_flags |= I830_PTE_SYSTEM_CACHED; 112171f45660SChris Wilson 1122a6963596SDaniel Vetter /* Shift high bits down */ 1123a6963596SDaniel Vetter addr |= (addr >> 28) & 0xf0; 1124983d308cSChris Wilson writel_relaxed(addr | pte_flags, intel_private.gtt + entry); 1125a6963596SDaniel Vetter } 1126a6963596SDaniel Vetter 11272d2430cfSDaniel Vetter static int i9xx_setup(void) 11282d2430cfSDaniel Vetter { 1129d3572532SBjorn Helgaas phys_addr_t reg_addr; 11304b60d29eSJesse Barnes int size = KB(512); 11312d2430cfSDaniel Vetter 1132d3572532SBjorn Helgaas reg_addr = pci_resource_start(intel_private.pcidev, I915_MMADR_BAR); 11332d2430cfSDaniel Vetter 11344b60d29eSJesse Barnes intel_private.registers = ioremap(reg_addr, size); 11352d2430cfSDaniel Vetter if (!intel_private.registers) 11362d2430cfSDaniel Vetter return -ENOMEM; 11372d2430cfSDaniel Vetter 1138009946f8SBen Widawsky switch (INTEL_GTT_GEN) { 1139009946f8SBen Widawsky case 3: 1140b5e350f9SBjorn Helgaas intel_private.gtt_phys_addr = 1141d3572532SBjorn Helgaas pci_resource_start(intel_private.pcidev, I915_PTE_BAR); 1142009946f8SBen Widawsky break; 11432d2430cfSDaniel Vetter case 5: 11445acc4ce4SBjorn Helgaas intel_private.gtt_phys_addr = reg_addr + MB(2); 11452d2430cfSDaniel Vetter break; 11462d2430cfSDaniel Vetter default: 11475acc4ce4SBjorn Helgaas intel_private.gtt_phys_addr = reg_addr + KB(512); 11482d2430cfSDaniel Vetter break; 11492d2430cfSDaniel Vetter } 11502d2430cfSDaniel Vetter 11512d2430cfSDaniel Vetter intel_i9xx_setup_flush(); 11522d2430cfSDaniel Vetter 11532d2430cfSDaniel Vetter return 0; 11542d2430cfSDaniel Vetter } 11552d2430cfSDaniel Vetter 115600fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 1157e9b1cc81SDaniel Vetter static const struct agp_bridge_driver intel_fake_agp_driver = { 1158f51b7662SDaniel Vetter .owner = THIS_MODULE, 1159f51b7662SDaniel Vetter .size_type = FIXED_APER_SIZE, 11609e76e7b8SChris Wilson .aperture_sizes = intel_fake_agp_sizes, 11619e76e7b8SChris Wilson .num_aperture_sizes = ARRAY_SIZE(intel_fake_agp_sizes), 1162a6963596SDaniel Vetter .configure = intel_fake_agp_configure, 11633e921f98SDaniel Vetter .fetch_size = intel_fake_agp_fetch_size, 1164fdfb58a9SDaniel Vetter .cleanup = intel_gtt_cleanup, 1165ffdd7510SDaniel Vetter .agp_enable = intel_fake_agp_enable, 1166f51b7662SDaniel Vetter .cache_flush = global_cache_flush, 11673b15a9d7SDaniel Vetter .create_gatt_table = intel_fake_agp_create_gatt_table, 1168ffdd7510SDaniel Vetter .free_gatt_table = intel_fake_agp_free_gatt_table, 1169450f2b3dSDaniel Vetter .insert_memory = intel_fake_agp_insert_entries, 1170450f2b3dSDaniel Vetter .remove_memory = intel_fake_agp_remove_entries, 1171ffdd7510SDaniel Vetter .alloc_by_type = intel_fake_agp_alloc_by_type, 1172f51b7662SDaniel Vetter .free_by_type = intel_i810_free_by_type, 1173f51b7662SDaniel Vetter .agp_alloc_page = agp_generic_alloc_page, 1174f51b7662SDaniel Vetter .agp_alloc_pages = agp_generic_alloc_pages, 1175f51b7662SDaniel Vetter .agp_destroy_page = agp_generic_destroy_page, 1176f51b7662SDaniel Vetter .agp_destroy_pages = agp_generic_destroy_pages, 1177f51b7662SDaniel Vetter }; 117800fe639aSVille Syrjälä #endif 117902c026ceSDaniel Vetter 1180bdd30729SDaniel Vetter static const struct intel_gtt_driver i81x_gtt_driver = { 1181bdd30729SDaniel Vetter .gen = 1, 1182820647b9SDaniel Vetter .has_pgtbl_enable = 1, 118322533b49SDaniel Vetter .dma_mask_size = 32, 1184820647b9SDaniel Vetter .setup = i810_setup, 1185820647b9SDaniel Vetter .cleanup = i810_cleanup, 1186625dd9d3SDaniel Vetter .check_flags = i830_check_flags, 1187625dd9d3SDaniel Vetter .write_entry = i810_write_entry, 1188bdd30729SDaniel Vetter }; 11891a997ff2SDaniel Vetter static const struct intel_gtt_driver i8xx_gtt_driver = { 11901a997ff2SDaniel Vetter .gen = 2, 1191100519e2SChris Wilson .has_pgtbl_enable = 1, 119273800422SDaniel Vetter .setup = i830_setup, 1193ae83dd5cSDaniel Vetter .cleanup = i830_cleanup, 1194351bb278SDaniel Vetter .write_entry = i830_write_entry, 119522533b49SDaniel Vetter .dma_mask_size = 32, 11965cbecafcSDaniel Vetter .check_flags = i830_check_flags, 11971b263f24SDaniel Vetter .chipset_flush = i830_chipset_flush, 11981a997ff2SDaniel Vetter }; 11991a997ff2SDaniel Vetter static const struct intel_gtt_driver i915_gtt_driver = { 12001a997ff2SDaniel Vetter .gen = 3, 1201100519e2SChris Wilson .has_pgtbl_enable = 1, 12022d2430cfSDaniel Vetter .setup = i9xx_setup, 1203ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1204351bb278SDaniel Vetter /* i945 is the last gpu to need phys mem (for overlay and cursors). */ 1205351bb278SDaniel Vetter .write_entry = i830_write_entry, 120622533b49SDaniel Vetter .dma_mask_size = 32, 1207fefaa70fSDaniel Vetter .check_flags = i830_check_flags, 12081b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12091a997ff2SDaniel Vetter }; 12101a997ff2SDaniel Vetter static const struct intel_gtt_driver g33_gtt_driver = { 12111a997ff2SDaniel Vetter .gen = 3, 12121a997ff2SDaniel Vetter .is_g33 = 1, 12132d2430cfSDaniel Vetter .setup = i9xx_setup, 1214ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1215a6963596SDaniel Vetter .write_entry = i965_write_entry, 121622533b49SDaniel Vetter .dma_mask_size = 36, 1217450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12181b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12191a997ff2SDaniel Vetter }; 12201a997ff2SDaniel Vetter static const struct intel_gtt_driver pineview_gtt_driver = { 12211a997ff2SDaniel Vetter .gen = 3, 12221a997ff2SDaniel Vetter .is_pineview = 1, .is_g33 = 1, 12232d2430cfSDaniel Vetter .setup = i9xx_setup, 1224ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1225a6963596SDaniel Vetter .write_entry = i965_write_entry, 122622533b49SDaniel Vetter .dma_mask_size = 36, 1227450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12281b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12291a997ff2SDaniel Vetter }; 12301a997ff2SDaniel Vetter static const struct intel_gtt_driver i965_gtt_driver = { 12311a997ff2SDaniel Vetter .gen = 4, 1232100519e2SChris Wilson .has_pgtbl_enable = 1, 12332d2430cfSDaniel Vetter .setup = i9xx_setup, 1234ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1235a6963596SDaniel Vetter .write_entry = i965_write_entry, 123622533b49SDaniel Vetter .dma_mask_size = 36, 1237450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12381b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12391a997ff2SDaniel Vetter }; 12401a997ff2SDaniel Vetter static const struct intel_gtt_driver g4x_gtt_driver = { 12411a997ff2SDaniel Vetter .gen = 5, 12422d2430cfSDaniel Vetter .setup = i9xx_setup, 1243ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1244a6963596SDaniel Vetter .write_entry = i965_write_entry, 124522533b49SDaniel Vetter .dma_mask_size = 36, 1246450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12471b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12481a997ff2SDaniel Vetter }; 12491a997ff2SDaniel Vetter static const struct intel_gtt_driver ironlake_gtt_driver = { 12501a997ff2SDaniel Vetter .gen = 5, 12511a997ff2SDaniel Vetter .is_ironlake = 1, 12522d2430cfSDaniel Vetter .setup = i9xx_setup, 1253ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1254a6963596SDaniel Vetter .write_entry = i965_write_entry, 125522533b49SDaniel Vetter .dma_mask_size = 36, 1256450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12571b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12581a997ff2SDaniel Vetter }; 12591a997ff2SDaniel Vetter 126002c026ceSDaniel Vetter /* Table to describe Intel GMCH and AGP/PCIE GART drivers. At least one of 126102c026ceSDaniel Vetter * driver and gmch_driver must be non-null, and find_gmch will determine 126202c026ceSDaniel Vetter * which one should be used if a gmch_chip_id is present. 126302c026ceSDaniel Vetter */ 126402c026ceSDaniel Vetter static const struct intel_gtt_driver_description { 126502c026ceSDaniel Vetter unsigned int gmch_chip_id; 126602c026ceSDaniel Vetter char *name; 12671a997ff2SDaniel Vetter const struct intel_gtt_driver *gtt_driver; 126802c026ceSDaniel Vetter } intel_gtt_chipsets[] = { 1269ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", 1270bdd30729SDaniel Vetter &i81x_gtt_driver}, 1271ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", 1272bdd30729SDaniel Vetter &i81x_gtt_driver}, 1273ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", 1274bdd30729SDaniel Vetter &i81x_gtt_driver}, 1275ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", 1276bdd30729SDaniel Vetter &i81x_gtt_driver}, 12771a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", 1278ff26860fSDaniel Vetter &i8xx_gtt_driver}, 127953371edaSOswald Buddenhagen { PCI_DEVICE_ID_INTEL_82845G_IG, "845G", 1280ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12811a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82854_IG, "854", 1282ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12831a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", 1284ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12851a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82865_IG, "865", 1286ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12871a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", 1288ff26860fSDaniel Vetter &i915_gtt_driver }, 12891a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", 1290ff26860fSDaniel Vetter &i915_gtt_driver }, 12911a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", 1292ff26860fSDaniel Vetter &i915_gtt_driver }, 12931a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", 1294ff26860fSDaniel Vetter &i915_gtt_driver }, 12951a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", 1296ff26860fSDaniel Vetter &i915_gtt_driver }, 12971a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", 1298ff26860fSDaniel Vetter &i915_gtt_driver }, 12991a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", 1300ff26860fSDaniel Vetter &i965_gtt_driver }, 13011a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", 1302ff26860fSDaniel Vetter &i965_gtt_driver }, 13031a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", 1304ff26860fSDaniel Vetter &i965_gtt_driver }, 13051a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", 1306ff26860fSDaniel Vetter &i965_gtt_driver }, 13071a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", 1308ff26860fSDaniel Vetter &i965_gtt_driver }, 13091a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", 1310ff26860fSDaniel Vetter &i965_gtt_driver }, 13111a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G33_IG, "G33", 1312ff26860fSDaniel Vetter &g33_gtt_driver }, 13131a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", 1314ff26860fSDaniel Vetter &g33_gtt_driver }, 13151a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", 1316ff26860fSDaniel Vetter &g33_gtt_driver }, 13171a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", 1318ff26860fSDaniel Vetter &pineview_gtt_driver }, 13191a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", 1320ff26860fSDaniel Vetter &pineview_gtt_driver }, 13211a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", 1322ff26860fSDaniel Vetter &g4x_gtt_driver }, 13231a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", 1324ff26860fSDaniel Vetter &g4x_gtt_driver }, 13251a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", 1326ff26860fSDaniel Vetter &g4x_gtt_driver }, 13271a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", 1328ff26860fSDaniel Vetter &g4x_gtt_driver }, 13291a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_B43_IG, "B43", 1330ff26860fSDaniel Vetter &g4x_gtt_driver }, 1331e9e5f8e8SChris Wilson { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", 1332ff26860fSDaniel Vetter &g4x_gtt_driver }, 13331a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G41_IG, "G41", 1334ff26860fSDaniel Vetter &g4x_gtt_driver }, 133502c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, 1336ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 133702c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, 1338ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 133902c026ceSDaniel Vetter { 0, NULL, NULL } 134002c026ceSDaniel Vetter }; 134102c026ceSDaniel Vetter 134202c026ceSDaniel Vetter static int find_gmch(u16 device) 134302c026ceSDaniel Vetter { 134402c026ceSDaniel Vetter struct pci_dev *gmch_device; 134502c026ceSDaniel Vetter 134602c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL); 134702c026ceSDaniel Vetter if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) { 134802c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, 134902c026ceSDaniel Vetter device, gmch_device); 135002c026ceSDaniel Vetter } 135102c026ceSDaniel Vetter 135202c026ceSDaniel Vetter if (!gmch_device) 135302c026ceSDaniel Vetter return 0; 135402c026ceSDaniel Vetter 135502c026ceSDaniel Vetter intel_private.pcidev = gmch_device; 135602c026ceSDaniel Vetter return 1; 135702c026ceSDaniel Vetter } 135802c026ceSDaniel Vetter 135914be93ddSDaniel Vetter int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev, 136002c026ceSDaniel Vetter struct agp_bridge_data *bridge) 136102c026ceSDaniel Vetter { 136202c026ceSDaniel Vetter int i, mask; 136314be93ddSDaniel Vetter 136402c026ceSDaniel Vetter for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { 136514be93ddSDaniel Vetter if (gpu_pdev) { 136614be93ddSDaniel Vetter if (gpu_pdev->device == 136714be93ddSDaniel Vetter intel_gtt_chipsets[i].gmch_chip_id) { 136814be93ddSDaniel Vetter intel_private.pcidev = pci_dev_get(gpu_pdev); 136914be93ddSDaniel Vetter intel_private.driver = 137014be93ddSDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 137114be93ddSDaniel Vetter 137214be93ddSDaniel Vetter break; 137314be93ddSDaniel Vetter } 137414be93ddSDaniel Vetter } else if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { 13751a997ff2SDaniel Vetter intel_private.driver = 13761a997ff2SDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 137702c026ceSDaniel Vetter break; 137802c026ceSDaniel Vetter } 137902c026ceSDaniel Vetter } 138002c026ceSDaniel Vetter 1381ff26860fSDaniel Vetter if (!intel_private.driver) 138202c026ceSDaniel Vetter return 0; 138302c026ceSDaniel Vetter 138400fe639aSVille Syrjälä #if IS_ENABLED(CONFIG_AGP_INTEL) 13857e8f6306SDaniel Vetter if (bridge) { 1386ebb7c78dSDaniel Vetter if (INTEL_GTT_GEN > 1) 1387ebb7c78dSDaniel Vetter return 0; 1388ebb7c78dSDaniel Vetter 1389ff26860fSDaniel Vetter bridge->driver = &intel_fake_agp_driver; 139002c026ceSDaniel Vetter bridge->dev_private_data = &intel_private; 139114be93ddSDaniel Vetter bridge->dev = bridge_pdev; 13927e8f6306SDaniel Vetter } 139300fe639aSVille Syrjälä #endif 139402c026ceSDaniel Vetter 1395ebb7c78dSDaniel Vetter 1396ebb7c78dSDaniel Vetter /* 1397ebb7c78dSDaniel Vetter * Can be called from the fake agp driver but also directly from 1398ebb7c78dSDaniel Vetter * drm/i915.ko. Hence we need to check whether everything is set up 1399ebb7c78dSDaniel Vetter * already. 1400ebb7c78dSDaniel Vetter */ 1401ebb7c78dSDaniel Vetter if (intel_private.refcount++) 1402ebb7c78dSDaniel Vetter return 1; 1403ebb7c78dSDaniel Vetter 140414be93ddSDaniel Vetter intel_private.bridge_dev = pci_dev_get(bridge_pdev); 1405d7cca2f7SDaniel Vetter 140614be93ddSDaniel Vetter dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name); 140702c026ceSDaniel Vetter 140822533b49SDaniel Vetter mask = intel_private.driver->dma_mask_size; 140902c026ceSDaniel Vetter if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask))) 141002c026ceSDaniel Vetter dev_err(&intel_private.pcidev->dev, 141102c026ceSDaniel Vetter "set gfx device dma mask %d-bit failed!\n", mask); 141202c026ceSDaniel Vetter else 141302c026ceSDaniel Vetter pci_set_consistent_dma_mask(intel_private.pcidev, 141402c026ceSDaniel Vetter DMA_BIT_MASK(mask)); 141502c026ceSDaniel Vetter 141614be93ddSDaniel Vetter if (intel_gtt_init() != 0) { 141714be93ddSDaniel Vetter intel_gmch_remove(); 141814be93ddSDaniel Vetter 14193b15a9d7SDaniel Vetter return 0; 142014be93ddSDaniel Vetter } 14211784a5fbSDaniel Vetter 142202c026ceSDaniel Vetter return 1; 142302c026ceSDaniel Vetter } 1424e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_probe); 142502c026ceSDaniel Vetter 1426edd1f2feSChris Wilson void intel_gtt_get(u64 *gtt_total, 1427edd1f2feSChris Wilson phys_addr_t *mappable_base, 1428*b7128ef1SMatthew Auld resource_size_t *mappable_end) 142919966754SDaniel Vetter { 1430a54c0c27SBen Widawsky *gtt_total = intel_private.gtt_total_entries << PAGE_SHIFT; 143141907ddcSBen Widawsky *mappable_base = intel_private.gma_bus_addr; 143241907ddcSBen Widawsky *mappable_end = intel_private.gtt_mappable_entries << PAGE_SHIFT; 143319966754SDaniel Vetter } 143419966754SDaniel Vetter EXPORT_SYMBOL(intel_gtt_get); 143519966754SDaniel Vetter 143640ce6575SDaniel Vetter void intel_gtt_chipset_flush(void) 143740ce6575SDaniel Vetter { 143840ce6575SDaniel Vetter if (intel_private.driver->chipset_flush) 143940ce6575SDaniel Vetter intel_private.driver->chipset_flush(); 144040ce6575SDaniel Vetter } 144140ce6575SDaniel Vetter EXPORT_SYMBOL(intel_gtt_chipset_flush); 144240ce6575SDaniel Vetter 144314be93ddSDaniel Vetter void intel_gmch_remove(void) 144402c026ceSDaniel Vetter { 144514be93ddSDaniel Vetter if (--intel_private.refcount) 144614be93ddSDaniel Vetter return; 144714be93ddSDaniel Vetter 14489f5ac8edSDaniel Vetter if (intel_private.scratch_page) 14499f5ac8edSDaniel Vetter intel_gtt_teardown_scratch_page(); 145002c026ceSDaniel Vetter if (intel_private.pcidev) 145102c026ceSDaniel Vetter pci_dev_put(intel_private.pcidev); 1452d7cca2f7SDaniel Vetter if (intel_private.bridge_dev) 1453d7cca2f7SDaniel Vetter pci_dev_put(intel_private.bridge_dev); 145414be93ddSDaniel Vetter intel_private.driver = NULL; 145502c026ceSDaniel Vetter } 1456e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_remove); 1457e2404e7cSDaniel Vetter 1458bd8136d3SDave Jones MODULE_AUTHOR("Dave Jones, Various @Intel"); 1459e2404e7cSDaniel Vetter MODULE_LICENSE("GPL and additional rights"); 1460