1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2016 Intel Corporation 4 */ 5 6 #include <linux/highmem.h> 7 #include <linux/shmem_fs.h> 8 #include <linux/swap.h> 9 10 #include <drm/drm_cache.h> 11 #include <drm/drm_print.h> 12 13 #include "gt/intel_gt.h" 14 #include "i915_drv.h" 15 #include "i915_gem_object.h" 16 #include "i915_gem_object_frontbuffer.h" 17 #include "i915_gem_region.h" 18 #include "i915_gem_tiling.h" 19 #include "i915_scatterlist.h" 20 21 /* Abuse scatterlist to store pointer instead of struct page. */ 22 static inline void __set_phys_vaddr(struct scatterlist *sg, void *vaddr) 23 { 24 sg_assign_page(sg, (struct page *)vaddr); 25 } 26 27 static inline void *__get_phys_vaddr(struct scatterlist *sg) 28 { 29 return (void *)sg_page(sg); 30 } 31 32 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 33 { 34 struct address_space *mapping = obj->base.filp->f_mapping; 35 struct drm_i915_private *i915 = to_i915(obj->base.dev); 36 struct scatterlist *sg; 37 struct sg_table *st; 38 dma_addr_t dma; 39 void *vaddr; 40 void *dst; 41 int i; 42 43 /* Contiguous chunk, with a single scatterlist element */ 44 if (overflows_type(obj->base.size, sg->length)) 45 return -E2BIG; 46 47 if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 48 return -EINVAL; 49 50 /* 51 * Always aligning to the object size, allows a single allocation 52 * to handle all possible callers, and given typical object sizes, 53 * the alignment of the buddy allocation will naturally match. 54 */ 55 vaddr = dma_alloc_coherent(obj->base.dev->dev, 56 roundup_pow_of_two(obj->base.size), 57 &dma, GFP_KERNEL); 58 if (!vaddr) 59 return -ENOMEM; 60 61 st = kmalloc_obj(*st); 62 if (!st) 63 goto err_pci; 64 65 if (sg_alloc_table(st, 1, GFP_KERNEL)) 66 goto err_st; 67 68 sg = st->sgl; 69 sg->offset = 0; 70 sg->length = obj->base.size; 71 72 __set_phys_vaddr(sg, vaddr); 73 sg_dma_address(sg) = dma; 74 sg_dma_len(sg) = obj->base.size; 75 76 dst = vaddr; 77 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 78 struct page *page; 79 80 page = shmem_read_mapping_page(mapping, i); 81 if (IS_ERR(page)) 82 goto err_st; 83 84 memcpy_from_page(dst, page, 0, PAGE_SIZE); 85 drm_clflush_virt_range(dst, PAGE_SIZE); 86 87 put_page(page); 88 dst += PAGE_SIZE; 89 } 90 91 intel_gt_chipset_flush(to_gt(i915)); 92 93 /* We're no longer struct page backed */ 94 obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE; 95 __i915_gem_object_set_pages(obj, st); 96 97 return 0; 98 99 err_st: 100 kfree(st); 101 err_pci: 102 dma_free_coherent(obj->base.dev->dev, 103 roundup_pow_of_two(obj->base.size), 104 vaddr, dma); 105 return -ENOMEM; 106 } 107 108 void 109 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 110 struct sg_table *pages) 111 { 112 dma_addr_t dma = sg_dma_address(pages->sgl); 113 void *vaddr = __get_phys_vaddr(pages->sgl); 114 115 __i915_gem_object_release_shmem(obj, pages, false); 116 117 if (obj->mm.dirty) { 118 struct address_space *mapping = obj->base.filp->f_mapping; 119 void *src = vaddr; 120 int i; 121 122 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 123 struct page *page; 124 125 page = shmem_read_mapping_page(mapping, i); 126 if (IS_ERR(page)) 127 continue; 128 129 drm_clflush_virt_range(src, PAGE_SIZE); 130 memcpy_to_page(page, 0, src, PAGE_SIZE); 131 132 set_page_dirty(page); 133 if (obj->mm.madv == I915_MADV_WILLNEED) 134 mark_page_accessed(page); 135 put_page(page); 136 137 src += PAGE_SIZE; 138 } 139 obj->mm.dirty = false; 140 } 141 142 sg_free_table(pages); 143 kfree(pages); 144 145 dma_free_coherent(obj->base.dev->dev, 146 roundup_pow_of_two(obj->base.size), 147 vaddr, dma); 148 } 149 150 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 151 const struct drm_i915_gem_pwrite *args) 152 { 153 void *vaddr = __get_phys_vaddr(obj->mm.pages->sgl) + args->offset; 154 char __user *user_data = u64_to_user_ptr(args->data_ptr); 155 struct drm_i915_private *i915 = to_i915(obj->base.dev); 156 int err; 157 158 err = i915_gem_object_wait(obj, 159 I915_WAIT_INTERRUPTIBLE | 160 I915_WAIT_ALL, 161 MAX_SCHEDULE_TIMEOUT); 162 if (err) 163 return err; 164 165 /* 166 * We manually control the domain here and pretend that it 167 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 168 */ 169 i915_gem_object_frontbuffer_invalidate(obj, ORIGIN_CPU); 170 171 if (copy_from_user(vaddr, user_data, args->size)) 172 return -EFAULT; 173 174 drm_clflush_virt_range(vaddr, args->size); 175 intel_gt_chipset_flush(to_gt(i915)); 176 177 i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU); 178 return 0; 179 } 180 181 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 182 const struct drm_i915_gem_pread *args) 183 { 184 void *vaddr = __get_phys_vaddr(obj->mm.pages->sgl) + args->offset; 185 char __user *user_data = u64_to_user_ptr(args->data_ptr); 186 int err; 187 188 err = i915_gem_object_wait(obj, 189 I915_WAIT_INTERRUPTIBLE, 190 MAX_SCHEDULE_TIMEOUT); 191 if (err) 192 return err; 193 194 drm_clflush_virt_range(vaddr, args->size); 195 if (copy_to_user(user_data, vaddr, args->size)) 196 return -EFAULT; 197 198 return 0; 199 } 200 201 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj) 202 { 203 struct sg_table *pages; 204 int err; 205 206 pages = __i915_gem_object_unset_pages(obj); 207 208 err = i915_gem_object_get_pages_phys(obj); 209 if (err) 210 goto err_xfer; 211 212 /* Perma-pin (until release) the physical set of pages */ 213 __i915_gem_object_pin_pages(obj); 214 215 if (!IS_ERR_OR_NULL(pages)) 216 i915_gem_object_put_pages_shmem(obj, pages); 217 218 i915_gem_object_release_memory_region(obj); 219 return 0; 220 221 err_xfer: 222 if (!IS_ERR_OR_NULL(pages)) 223 __i915_gem_object_set_pages(obj, pages); 224 return err; 225 } 226 227 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 228 { 229 int err; 230 231 assert_object_held(obj); 232 233 if (align > obj->base.size) 234 return -EINVAL; 235 236 if (!i915_gem_object_is_shmem(obj)) 237 return -EINVAL; 238 239 if (!i915_gem_object_has_struct_page(obj)) 240 return 0; 241 242 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 243 if (err) 244 return err; 245 246 if (obj->mm.madv != I915_MADV_WILLNEED) 247 return -EFAULT; 248 249 if (i915_gem_object_has_tiling_quirk(obj)) 250 return -EFAULT; 251 252 if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj)) 253 return -EBUSY; 254 255 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 256 drm_dbg(obj->base.dev, 257 "Attempting to obtain a purgeable object\n"); 258 return -EFAULT; 259 } 260 261 return i915_gem_object_shmem_to_phys(obj); 262 } 263 264 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 265 #include "selftests/i915_gem_phys.c" 266 #endif 267