1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm.h> /* for drm_legacy.h! */ 12 #include <drm/drm_cache.h> 13 #include <drm/drm_legacy.h> /* for drm_pci.h! */ 14 #include <drm/drm_pci.h> 15 16 #include "gt/intel_gt.h" 17 #include "i915_drv.h" 18 #include "i915_gem_object.h" 19 #include "i915_scatterlist.h" 20 21 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 22 { 23 struct address_space *mapping = obj->base.filp->f_mapping; 24 struct drm_dma_handle *phys; 25 struct sg_table *st; 26 struct scatterlist *sg; 27 char *vaddr; 28 int i; 29 int err; 30 31 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 32 return -EINVAL; 33 34 /* Always aligning to the object size, allows a single allocation 35 * to handle all possible callers, and given typical object sizes, 36 * the alignment of the buddy allocation will naturally match. 37 */ 38 phys = drm_pci_alloc(obj->base.dev, 39 roundup_pow_of_two(obj->base.size), 40 roundup_pow_of_two(obj->base.size)); 41 if (!phys) 42 return -ENOMEM; 43 44 vaddr = phys->vaddr; 45 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 46 struct page *page; 47 char *src; 48 49 page = shmem_read_mapping_page(mapping, i); 50 if (IS_ERR(page)) { 51 err = PTR_ERR(page); 52 goto err_phys; 53 } 54 55 src = kmap_atomic(page); 56 memcpy(vaddr, src, PAGE_SIZE); 57 drm_clflush_virt_range(vaddr, PAGE_SIZE); 58 kunmap_atomic(src); 59 60 put_page(page); 61 vaddr += PAGE_SIZE; 62 } 63 64 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); 65 66 st = kmalloc(sizeof(*st), GFP_KERNEL); 67 if (!st) { 68 err = -ENOMEM; 69 goto err_phys; 70 } 71 72 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 73 kfree(st); 74 err = -ENOMEM; 75 goto err_phys; 76 } 77 78 sg = st->sgl; 79 sg->offset = 0; 80 sg->length = obj->base.size; 81 82 sg_dma_address(sg) = phys->busaddr; 83 sg_dma_len(sg) = obj->base.size; 84 85 obj->phys_handle = phys; 86 87 __i915_gem_object_set_pages(obj, st, sg->length); 88 89 return 0; 90 91 err_phys: 92 drm_pci_free(obj->base.dev, phys); 93 94 return err; 95 } 96 97 static void 98 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 99 struct sg_table *pages) 100 { 101 __i915_gem_object_release_shmem(obj, pages, false); 102 103 if (obj->mm.dirty) { 104 struct address_space *mapping = obj->base.filp->f_mapping; 105 char *vaddr = obj->phys_handle->vaddr; 106 int i; 107 108 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 109 struct page *page; 110 char *dst; 111 112 page = shmem_read_mapping_page(mapping, i); 113 if (IS_ERR(page)) 114 continue; 115 116 dst = kmap_atomic(page); 117 drm_clflush_virt_range(vaddr, PAGE_SIZE); 118 memcpy(dst, vaddr, PAGE_SIZE); 119 kunmap_atomic(dst); 120 121 set_page_dirty(page); 122 if (obj->mm.madv == I915_MADV_WILLNEED) 123 mark_page_accessed(page); 124 put_page(page); 125 vaddr += PAGE_SIZE; 126 } 127 obj->mm.dirty = false; 128 } 129 130 sg_free_table(pages); 131 kfree(pages); 132 133 drm_pci_free(obj->base.dev, obj->phys_handle); 134 } 135 136 static void phys_release(struct drm_i915_gem_object *obj) 137 { 138 fput(obj->base.filp); 139 } 140 141 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 142 .get_pages = i915_gem_object_get_pages_phys, 143 .put_pages = i915_gem_object_put_pages_phys, 144 145 .release = phys_release, 146 }; 147 148 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 149 { 150 struct sg_table *pages; 151 int err; 152 153 if (align > obj->base.size) 154 return -EINVAL; 155 156 if (obj->ops == &i915_gem_phys_ops) 157 return 0; 158 159 if (obj->ops != &i915_gem_shmem_ops) 160 return -EINVAL; 161 162 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 163 if (err) 164 return err; 165 166 mutex_lock(&obj->mm.lock); 167 168 if (obj->mm.madv != I915_MADV_WILLNEED) { 169 err = -EFAULT; 170 goto err_unlock; 171 } 172 173 if (obj->mm.quirked) { 174 err = -EFAULT; 175 goto err_unlock; 176 } 177 178 if (obj->mm.mapping) { 179 err = -EBUSY; 180 goto err_unlock; 181 } 182 183 pages = __i915_gem_object_unset_pages(obj); 184 185 obj->ops = &i915_gem_phys_ops; 186 187 err = ____i915_gem_object_get_pages(obj); 188 if (err) 189 goto err_xfer; 190 191 /* Perma-pin (until release) the physical set of pages */ 192 __i915_gem_object_pin_pages(obj); 193 194 if (!IS_ERR_OR_NULL(pages)) 195 i915_gem_shmem_ops.put_pages(obj, pages); 196 mutex_unlock(&obj->mm.lock); 197 return 0; 198 199 err_xfer: 200 obj->ops = &i915_gem_shmem_ops; 201 if (!IS_ERR_OR_NULL(pages)) { 202 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl); 203 204 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 205 } 206 err_unlock: 207 mutex_unlock(&obj->mm.lock); 208 return err; 209 } 210 211 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 212 #include "selftests/i915_gem_phys.c" 213 #endif 214