1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm_cache.h> 12 13 #include "gt/intel_gt.h" 14 #include "i915_drv.h" 15 #include "i915_gem_object.h" 16 #include "i915_gem_region.h" 17 #include "i915_gem_tiling.h" 18 #include "i915_scatterlist.h" 19 20 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 21 { 22 struct address_space *mapping = obj->base.filp->f_mapping; 23 struct drm_i915_private *i915 = to_i915(obj->base.dev); 24 struct scatterlist *sg; 25 struct sg_table *st; 26 dma_addr_t dma; 27 void *vaddr; 28 void *dst; 29 int i; 30 31 /* Contiguous chunk, with a single scatterlist element */ 32 if (overflows_type(obj->base.size, sg->length)) 33 return -E2BIG; 34 35 if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 36 return -EINVAL; 37 38 /* 39 * Always aligning to the object size, allows a single allocation 40 * to handle all possible callers, and given typical object sizes, 41 * the alignment of the buddy allocation will naturally match. 42 */ 43 vaddr = dma_alloc_coherent(obj->base.dev->dev, 44 roundup_pow_of_two(obj->base.size), 45 &dma, GFP_KERNEL); 46 if (!vaddr) 47 return -ENOMEM; 48 49 st = kmalloc(sizeof(*st), GFP_KERNEL); 50 if (!st) 51 goto err_pci; 52 53 if (sg_alloc_table(st, 1, GFP_KERNEL)) 54 goto err_st; 55 56 sg = st->sgl; 57 sg->offset = 0; 58 sg->length = obj->base.size; 59 60 sg_assign_page(sg, (struct page *)vaddr); 61 sg_dma_address(sg) = dma; 62 sg_dma_len(sg) = obj->base.size; 63 64 dst = vaddr; 65 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 66 struct page *page; 67 void *src; 68 69 page = shmem_read_mapping_page(mapping, i); 70 if (IS_ERR(page)) 71 goto err_st; 72 73 src = kmap_atomic(page); 74 memcpy(dst, src, PAGE_SIZE); 75 drm_clflush_virt_range(dst, PAGE_SIZE); 76 kunmap_atomic(src); 77 78 put_page(page); 79 dst += PAGE_SIZE; 80 } 81 82 intel_gt_chipset_flush(to_gt(i915)); 83 84 /* We're no longer struct page backed */ 85 obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE; 86 __i915_gem_object_set_pages(obj, st); 87 88 return 0; 89 90 err_st: 91 kfree(st); 92 err_pci: 93 dma_free_coherent(obj->base.dev->dev, 94 roundup_pow_of_two(obj->base.size), 95 vaddr, dma); 96 return -ENOMEM; 97 } 98 99 void 100 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 101 struct sg_table *pages) 102 { 103 dma_addr_t dma = sg_dma_address(pages->sgl); 104 void *vaddr = sg_page(pages->sgl); 105 106 __i915_gem_object_release_shmem(obj, pages, false); 107 108 if (obj->mm.dirty) { 109 struct address_space *mapping = obj->base.filp->f_mapping; 110 void *src = vaddr; 111 int i; 112 113 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 114 struct page *page; 115 char *dst; 116 117 page = shmem_read_mapping_page(mapping, i); 118 if (IS_ERR(page)) 119 continue; 120 121 dst = kmap_atomic(page); 122 drm_clflush_virt_range(src, PAGE_SIZE); 123 memcpy(dst, src, PAGE_SIZE); 124 kunmap_atomic(dst); 125 126 set_page_dirty(page); 127 if (obj->mm.madv == I915_MADV_WILLNEED) 128 mark_page_accessed(page); 129 put_page(page); 130 131 src += PAGE_SIZE; 132 } 133 obj->mm.dirty = false; 134 } 135 136 sg_free_table(pages); 137 kfree(pages); 138 139 dma_free_coherent(obj->base.dev->dev, 140 roundup_pow_of_two(obj->base.size), 141 vaddr, dma); 142 } 143 144 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 145 const struct drm_i915_gem_pwrite *args) 146 { 147 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; 148 char __user *user_data = u64_to_user_ptr(args->data_ptr); 149 struct drm_i915_private *i915 = to_i915(obj->base.dev); 150 int err; 151 152 err = i915_gem_object_wait(obj, 153 I915_WAIT_INTERRUPTIBLE | 154 I915_WAIT_ALL, 155 MAX_SCHEDULE_TIMEOUT); 156 if (err) 157 return err; 158 159 /* 160 * We manually control the domain here and pretend that it 161 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 162 */ 163 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 164 165 if (copy_from_user(vaddr, user_data, args->size)) 166 return -EFAULT; 167 168 drm_clflush_virt_range(vaddr, args->size); 169 intel_gt_chipset_flush(to_gt(i915)); 170 171 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 172 return 0; 173 } 174 175 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 176 const struct drm_i915_gem_pread *args) 177 { 178 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; 179 char __user *user_data = u64_to_user_ptr(args->data_ptr); 180 int err; 181 182 err = i915_gem_object_wait(obj, 183 I915_WAIT_INTERRUPTIBLE, 184 MAX_SCHEDULE_TIMEOUT); 185 if (err) 186 return err; 187 188 drm_clflush_virt_range(vaddr, args->size); 189 if (copy_to_user(user_data, vaddr, args->size)) 190 return -EFAULT; 191 192 return 0; 193 } 194 195 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj) 196 { 197 struct sg_table *pages; 198 int err; 199 200 pages = __i915_gem_object_unset_pages(obj); 201 202 err = i915_gem_object_get_pages_phys(obj); 203 if (err) 204 goto err_xfer; 205 206 /* Perma-pin (until release) the physical set of pages */ 207 __i915_gem_object_pin_pages(obj); 208 209 if (!IS_ERR_OR_NULL(pages)) 210 i915_gem_object_put_pages_shmem(obj, pages); 211 212 i915_gem_object_release_memory_region(obj); 213 return 0; 214 215 err_xfer: 216 if (!IS_ERR_OR_NULL(pages)) 217 __i915_gem_object_set_pages(obj, pages); 218 return err; 219 } 220 221 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 222 { 223 int err; 224 225 assert_object_held(obj); 226 227 if (align > obj->base.size) 228 return -EINVAL; 229 230 if (!i915_gem_object_is_shmem(obj)) 231 return -EINVAL; 232 233 if (!i915_gem_object_has_struct_page(obj)) 234 return 0; 235 236 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 237 if (err) 238 return err; 239 240 if (obj->mm.madv != I915_MADV_WILLNEED) 241 return -EFAULT; 242 243 if (i915_gem_object_has_tiling_quirk(obj)) 244 return -EFAULT; 245 246 if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj)) 247 return -EBUSY; 248 249 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 250 drm_dbg(obj->base.dev, 251 "Attempting to obtain a purgeable object\n"); 252 return -EFAULT; 253 } 254 255 return i915_gem_object_shmem_to_phys(obj); 256 } 257 258 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 259 #include "selftests/i915_gem_phys.c" 260 #endif 261