1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4 #include "pvr_device.h"
5 #include "pvr_gem.h"
6 #include "pvr_vm.h"
7
8 #include <drm/drm_gem.h>
9 #include <drm/drm_prime.h>
10
11 #include <linux/compiler.h>
12 #include <linux/compiler_attributes.h>
13 #include <linux/dma-buf.h>
14 #include <linux/dma-direction.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/err.h>
17 #include <linux/gfp.h>
18 #include <linux/iosys-map.h>
19 #include <linux/log2.h>
20 #include <linux/mutex.h>
21 #include <linux/pagemap.h>
22 #include <linux/refcount.h>
23 #include <linux/scatterlist.h>
24
pvr_gem_object_free(struct drm_gem_object * obj)25 static void pvr_gem_object_free(struct drm_gem_object *obj)
26 {
27 drm_gem_shmem_object_free(obj);
28 }
29
pvr_gem_mmap(struct drm_gem_object * gem_obj,struct vm_area_struct * vma)30 static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma)
31 {
32 struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj);
33 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
34
35 if (!(pvr_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS))
36 return -EINVAL;
37
38 return drm_gem_shmem_mmap(shmem_obj, vma);
39 }
40
41 static const struct drm_gem_object_funcs pvr_gem_object_funcs = {
42 .free = pvr_gem_object_free,
43 .print_info = drm_gem_shmem_object_print_info,
44 .pin = drm_gem_shmem_object_pin,
45 .unpin = drm_gem_shmem_object_unpin,
46 .get_sg_table = drm_gem_shmem_object_get_sg_table,
47 .vmap = drm_gem_shmem_object_vmap,
48 .vunmap = drm_gem_shmem_object_vunmap,
49 .mmap = pvr_gem_mmap,
50 .vm_ops = &drm_gem_shmem_vm_ops,
51 };
52
53 /**
54 * pvr_gem_object_flags_validate() - Verify that a collection of PowerVR GEM
55 * mapping and/or creation flags form a valid combination.
56 * @flags: PowerVR GEM mapping/creation flags to validate.
57 *
58 * This function explicitly allows kernel-only flags. All ioctl entrypoints
59 * should do their own validation as well as relying on this function.
60 *
61 * Return:
62 * * %true if @flags contains valid mapping and/or creation flags, or
63 * * %false otherwise.
64 */
65 static bool
pvr_gem_object_flags_validate(u64 flags)66 pvr_gem_object_flags_validate(u64 flags)
67 {
68 static const u64 invalid_combinations[] = {
69 /*
70 * Memory flagged as PM/FW-protected cannot be mapped to
71 * userspace. To make this explicit, we require that the two
72 * flags allowing each of these respective features are never
73 * specified together.
74 */
75 (DRM_PVR_BO_PM_FW_PROTECT |
76 DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS),
77 };
78
79 int i;
80
81 /*
82 * Check for bits set in undefined regions. Reserved regions refer to
83 * options that can only be set by the kernel. These are explicitly
84 * allowed in most cases, and must be checked specifically in IOCTL
85 * callback code.
86 */
87 if ((flags & PVR_BO_UNDEFINED_MASK) != 0)
88 return false;
89
90 /*
91 * Check for all combinations of flags marked as invalid in the array
92 * above.
93 */
94 for (i = 0; i < ARRAY_SIZE(invalid_combinations); ++i) {
95 u64 combo = invalid_combinations[i];
96
97 if ((flags & combo) == combo)
98 return false;
99 }
100
101 return true;
102 }
103
104 /**
105 * pvr_gem_object_into_handle() - Convert a reference to an object into a
106 * userspace-accessible handle.
107 * @pvr_obj: [IN] Target PowerVR-specific object.
108 * @pvr_file: [IN] File to associate the handle with.
109 * @handle: [OUT] Pointer to store the created handle in. Remains unmodified if
110 * an error is encountered.
111 *
112 * If an error is encountered, ownership of @pvr_obj will not have been
113 * transferred. If this function succeeds, however, further use of @pvr_obj is
114 * considered undefined behaviour unless another reference to it is explicitly
115 * held.
116 *
117 * Return:
118 * * 0 on success, or
119 * * Any error encountered while attempting to allocate a handle on @pvr_file.
120 */
121 int
pvr_gem_object_into_handle(struct pvr_gem_object * pvr_obj,struct pvr_file * pvr_file,u32 * handle)122 pvr_gem_object_into_handle(struct pvr_gem_object *pvr_obj,
123 struct pvr_file *pvr_file, u32 *handle)
124 {
125 struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);
126 struct drm_file *file = from_pvr_file(pvr_file);
127
128 u32 new_handle;
129 int err;
130
131 err = drm_gem_handle_create(file, gem_obj, &new_handle);
132 if (err)
133 return err;
134
135 /*
136 * Release our reference to @pvr_obj, effectively transferring
137 * ownership to the handle.
138 */
139 pvr_gem_object_put(pvr_obj);
140
141 /*
142 * Do not store the new handle in @handle until no more errors can
143 * occur.
144 */
145 *handle = new_handle;
146
147 return 0;
148 }
149
150 /**
151 * pvr_gem_object_from_handle() - Obtain a reference to an object from a
152 * userspace handle.
153 * @pvr_file: PowerVR-specific file to which @handle is associated.
154 * @handle: Userspace handle referencing the target object.
155 *
156 * On return, @handle always maintains its reference to the requested object
157 * (if it had one in the first place). If this function succeeds, the returned
158 * object will hold an additional reference. When the caller is finished with
159 * the returned object, they should call pvr_gem_object_put() on it to release
160 * this reference.
161 *
162 * Return:
163 * * A pointer to the requested PowerVR-specific object on success, or
164 * * %NULL otherwise.
165 */
166 struct pvr_gem_object *
pvr_gem_object_from_handle(struct pvr_file * pvr_file,u32 handle)167 pvr_gem_object_from_handle(struct pvr_file *pvr_file, u32 handle)
168 {
169 struct drm_file *file = from_pvr_file(pvr_file);
170 struct drm_gem_object *gem_obj;
171
172 gem_obj = drm_gem_object_lookup(file, handle);
173 if (!gem_obj)
174 return NULL;
175
176 return gem_to_pvr_gem(gem_obj);
177 }
178
179 /**
180 * pvr_gem_object_vmap() - Map a PowerVR GEM object into CPU virtual address
181 * space.
182 * @pvr_obj: Target PowerVR GEM object.
183 *
184 * Once the caller is finished with the CPU mapping, they must call
185 * pvr_gem_object_vunmap() on @pvr_obj.
186 *
187 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_cpu() is called to make
188 * sure the CPU mapping is consistent.
189 *
190 * Return:
191 * * A pointer to the CPU mapping on success,
192 * * -%ENOMEM if the mapping fails, or
193 * * Any error encountered while attempting to acquire a reference to the
194 * backing pages for @pvr_obj.
195 */
196 void *
pvr_gem_object_vmap(struct pvr_gem_object * pvr_obj)197 pvr_gem_object_vmap(struct pvr_gem_object *pvr_obj)
198 {
199 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
200 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
201 struct iosys_map map;
202 int err;
203
204 dma_resv_lock(obj->resv, NULL);
205
206 err = drm_gem_shmem_vmap(shmem_obj, &map);
207 if (err)
208 goto err_unlock;
209
210 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {
211 struct device *dev = shmem_obj->base.dev->dev;
212
213 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped
214 * in GPU space yet.
215 */
216 if (shmem_obj->sgt)
217 dma_sync_sgtable_for_cpu(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL);
218 }
219
220 dma_resv_unlock(obj->resv);
221
222 return map.vaddr;
223
224 err_unlock:
225 dma_resv_unlock(obj->resv);
226
227 return ERR_PTR(err);
228 }
229
230 /**
231 * pvr_gem_object_vunmap() - Unmap a PowerVR memory object from CPU virtual
232 * address space.
233 * @pvr_obj: Target PowerVR GEM object.
234 *
235 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_device() is called to make
236 * sure the GPU mapping is consistent.
237 */
238 void
pvr_gem_object_vunmap(struct pvr_gem_object * pvr_obj)239 pvr_gem_object_vunmap(struct pvr_gem_object *pvr_obj)
240 {
241 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
242 struct iosys_map map = IOSYS_MAP_INIT_VADDR(shmem_obj->vaddr);
243 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
244
245 if (WARN_ON(!map.vaddr))
246 return;
247
248 dma_resv_lock(obj->resv, NULL);
249
250 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {
251 struct device *dev = shmem_obj->base.dev->dev;
252
253 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped
254 * in GPU space yet.
255 */
256 if (shmem_obj->sgt)
257 dma_sync_sgtable_for_device(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL);
258 }
259
260 drm_gem_shmem_vunmap(shmem_obj, &map);
261
262 dma_resv_unlock(obj->resv);
263 }
264
265 /**
266 * pvr_gem_object_zero() - Zeroes the physical memory behind an object.
267 * @pvr_obj: Target PowerVR GEM object.
268 *
269 * Return:
270 * * 0 on success, or
271 * * Any error encountered while attempting to map @pvr_obj to the CPU (see
272 * pvr_gem_object_vmap()).
273 */
274 static int
pvr_gem_object_zero(struct pvr_gem_object * pvr_obj)275 pvr_gem_object_zero(struct pvr_gem_object *pvr_obj)
276 {
277 void *cpu_ptr;
278
279 cpu_ptr = pvr_gem_object_vmap(pvr_obj);
280 if (IS_ERR(cpu_ptr))
281 return PTR_ERR(cpu_ptr);
282
283 memset(cpu_ptr, 0, pvr_gem_object_size(pvr_obj));
284
285 /* Make sure the zero-ing is done before vumap-ing the object. */
286 wmb();
287
288 pvr_gem_object_vunmap(pvr_obj);
289
290 return 0;
291 }
292
293 /**
294 * pvr_gem_create_object() - Allocate and pre-initializes a pvr_gem_object
295 * @drm_dev: DRM device creating this object.
296 * @size: Size of the object to allocate in bytes.
297 *
298 * Return:
299 * * The new pre-initialized GEM object on success,
300 * * -ENOMEM if the allocation failed.
301 */
pvr_gem_create_object(struct drm_device * drm_dev,size_t size)302 struct drm_gem_object *pvr_gem_create_object(struct drm_device *drm_dev, size_t size)
303 {
304 struct drm_gem_object *gem_obj;
305 struct pvr_gem_object *pvr_obj;
306
307 pvr_obj = kzalloc(sizeof(*pvr_obj), GFP_KERNEL);
308 if (!pvr_obj)
309 return ERR_PTR(-ENOMEM);
310
311 gem_obj = gem_from_pvr_gem(pvr_obj);
312 gem_obj->funcs = &pvr_gem_object_funcs;
313
314 return gem_obj;
315 }
316
317 /**
318 * pvr_gem_object_create() - Creates a PowerVR-specific buffer object.
319 * @pvr_dev: Target PowerVR device.
320 * @size: Size of the object to allocate in bytes. Must be greater than zero.
321 * Any value which is not an exact multiple of the system page size will be
322 * rounded up to satisfy this condition.
323 * @flags: Options which affect both this operation and future mapping
324 * operations performed on the returned object. Must be a combination of
325 * DRM_PVR_BO_* and/or PVR_BO_* flags.
326 *
327 * The created object may be larger than @size, but can never be smaller. To
328 * get the exact size, call pvr_gem_object_size() on the returned pointer.
329 *
330 * Return:
331 * * The newly-minted PowerVR-specific buffer object on success,
332 * * -%EINVAL if @size is zero or @flags is not valid,
333 * * -%ENOMEM if sufficient physical memory cannot be allocated, or
334 * * Any other error returned by drm_gem_create_mmap_offset().
335 */
336 struct pvr_gem_object *
pvr_gem_object_create(struct pvr_device * pvr_dev,size_t size,u64 flags)337 pvr_gem_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags)
338 {
339 struct drm_gem_shmem_object *shmem_obj;
340 struct pvr_gem_object *pvr_obj;
341 struct sg_table *sgt;
342 int err;
343
344 /* Verify @size and @flags before continuing. */
345 if (size == 0 || !pvr_gem_object_flags_validate(flags))
346 return ERR_PTR(-EINVAL);
347
348 shmem_obj = drm_gem_shmem_create(from_pvr_device(pvr_dev), size);
349 if (IS_ERR(shmem_obj))
350 return ERR_CAST(shmem_obj);
351
352 shmem_obj->pages_mark_dirty_on_put = true;
353 shmem_obj->map_wc = !(flags & PVR_BO_CPU_CACHED);
354 pvr_obj = shmem_gem_to_pvr_gem(shmem_obj);
355 pvr_obj->flags = flags;
356
357 sgt = drm_gem_shmem_get_pages_sgt(shmem_obj);
358 if (IS_ERR(sgt)) {
359 err = PTR_ERR(sgt);
360 goto err_shmem_object_free;
361 }
362
363 dma_sync_sgtable_for_device(shmem_obj->base.dev->dev, sgt,
364 DMA_BIDIRECTIONAL);
365
366 /*
367 * Do this last because pvr_gem_object_zero() requires a fully
368 * configured instance of struct pvr_gem_object.
369 */
370 pvr_gem_object_zero(pvr_obj);
371
372 return pvr_obj;
373
374 err_shmem_object_free:
375 drm_gem_shmem_free(shmem_obj);
376
377 return ERR_PTR(err);
378 }
379
380 /**
381 * pvr_gem_get_dma_addr() - Get DMA address for given offset in object
382 * @pvr_obj: Pointer to object to lookup address in.
383 * @offset: Offset within object to lookup address at.
384 * @dma_addr_out: Pointer to location to store DMA address.
385 *
386 * Returns:
387 * * 0 on success, or
388 * * -%EINVAL if object is not currently backed, or if @offset is out of valid
389 * range for this object.
390 */
391 int
pvr_gem_get_dma_addr(struct pvr_gem_object * pvr_obj,u32 offset,dma_addr_t * dma_addr_out)392 pvr_gem_get_dma_addr(struct pvr_gem_object *pvr_obj, u32 offset,
393 dma_addr_t *dma_addr_out)
394 {
395 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
396 u32 accumulated_offset = 0;
397 struct scatterlist *sgl;
398 unsigned int sgt_idx;
399
400 WARN_ON(!shmem_obj->sgt);
401 for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, sgt_idx) {
402 u32 new_offset = accumulated_offset + sg_dma_len(sgl);
403
404 if (offset >= accumulated_offset && offset < new_offset) {
405 *dma_addr_out = sg_dma_address(sgl) +
406 (offset - accumulated_offset);
407 return 0;
408 }
409
410 accumulated_offset = new_offset;
411 }
412
413 return -EINVAL;
414 }
415