1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include "xe_mmio_gem.h"
7
8 #include <linux/dma-resv.h>
9 #include <drm/drm_drv.h>
10 #include <drm/drm_gem.h>
11
12 #include "xe_device_types.h"
13
14 /**
15 * DOC: Exposing MMIO regions to userspace
16 *
17 * In certain cases, the driver may allow userspace to mmap a portion of the hardware registers.
18 *
19 * This can be done as follows:
20 * 1. Call xe_mmio_gem_create() to create a GEM object with an mmap-able fake offset.
21 * 2. Use xe_mmio_gem_mmap_offset() on the created GEM object to retrieve the fake offset.
22 * 3. Provide the fake offset to userspace.
23 * 4. Userspace can call mmap with the fake offset. The length provided to mmap
24 * must match the size of the GEM object.
25 * 5. When the region is no longer needed, call xe_mmio_gem_destroy() to release the GEM object.
26 *
27 * NOTE: The exposed MMIO region must be page-aligned with regards to its BAR offset and size.
28 *
29 * WARNING: Exposing MMIO regions to userspace can have security and stability implications.
30 * Make sure not to expose any sensitive registers.
31 */
32
33 static void xe_mmio_gem_free(struct drm_gem_object *);
34 static int xe_mmio_gem_mmap(struct drm_gem_object *, struct vm_area_struct *);
35 static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *);
36
37 struct xe_mmio_gem {
38 struct drm_gem_object base;
39 phys_addr_t phys_addr;
40 struct page *dummy_page; /* protected by the GEM's dma_resv */
41 bool destroyed; /* protected by the GEM's dma_resv */
42 };
43
xe_mmio_gem_vm_may_split(struct vm_area_struct * area,unsigned long addr)44 static int xe_mmio_gem_vm_may_split(struct vm_area_struct *area, unsigned long addr)
45 {
46 /*
47 * Forbid splitting. Together with VM_DONTEXPAND, this keeps the VMA
48 * matching the GEM object exactly.
49 */
50 return -EINVAL;
51 }
52
53 static const struct vm_operations_struct vm_ops = {
54 .open = drm_gem_vm_open,
55 .close = drm_gem_vm_close,
56 .fault = xe_mmio_gem_vm_fault,
57 .may_split = xe_mmio_gem_vm_may_split,
58 };
59
60 static const struct drm_gem_object_funcs xe_mmio_gem_funcs = {
61 .free = xe_mmio_gem_free,
62 .mmap = xe_mmio_gem_mmap,
63 .vm_ops = &vm_ops,
64 };
65
to_xe_mmio_gem(struct drm_gem_object * obj)66 static inline struct xe_mmio_gem *to_xe_mmio_gem(struct drm_gem_object *obj)
67 {
68 return container_of(obj, struct xe_mmio_gem, base);
69 }
70
71 /**
72 * xe_mmio_gem_create - Expose an MMIO region to userspace
73 * @xe: The xe device
74 * @file: DRM file descriptor
75 * @phys_addr: Start of the exposed MMIO region
76 * @size: The size of the exposed MMIO region
77 *
78 * This function creates a GEM object that exposes an MMIO region with an mmap-able
79 * fake offset.
80 *
81 * See: "Exposing MMIO regions to userspace"
82 */
xe_mmio_gem_create(struct xe_device * xe,struct drm_file * file,phys_addr_t phys_addr,size_t size)83 struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file,
84 phys_addr_t phys_addr, size_t size)
85 {
86 struct xe_mmio_gem *obj;
87 struct drm_gem_object *base;
88 int err;
89
90 if ((phys_addr % PAGE_SIZE != 0) || (size % PAGE_SIZE != 0))
91 return ERR_PTR(-EINVAL);
92
93 obj = kzalloc_obj(*obj);
94 if (!obj)
95 return ERR_PTR(-ENOMEM);
96
97 base = &obj->base;
98 base->funcs = &xe_mmio_gem_funcs;
99 obj->phys_addr = phys_addr;
100
101 drm_gem_private_object_init(&xe->drm, base, size);
102
103 err = drm_gem_create_mmap_offset(base);
104 if (err)
105 goto free_gem;
106
107 err = drm_vma_node_allow(&base->vma_node, file);
108 if (err)
109 goto free_gem;
110
111 return obj;
112
113 free_gem:
114 xe_mmio_gem_free(base);
115 return ERR_PTR(err);
116 }
117
118 /**
119 * xe_mmio_gem_mmap_offset - Return the mmap-able fake offset
120 * @gem: the GEM object created with xe_mmio_gem_create()
121 *
122 * This function returns the mmap-able fake offset allocated during
123 * xe_mmio_gem_create().
124 *
125 * See: "Exposing MMIO regions to userspace"
126 */
xe_mmio_gem_mmap_offset(struct xe_mmio_gem * gem)127 u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem)
128 {
129 return drm_vma_node_offset_addr(&gem->base.vma_node);
130 }
131
xe_mmio_gem_free(struct drm_gem_object * base)132 static void xe_mmio_gem_free(struct drm_gem_object *base)
133 {
134 struct xe_mmio_gem *obj = to_xe_mmio_gem(base);
135
136 if (obj->dummy_page)
137 __free_page(obj->dummy_page);
138 drm_gem_object_release(base);
139 kfree(obj);
140 }
141
142 /**
143 * xe_mmio_gem_destroy - Destroy the GEM object that exposes an MMIO region
144 * @gem: the GEM object to destroy
145 * @file: DRM file descriptor previously passed to xe_mmio_gem_create()
146 *
147 * This function releases resources associated with the GEM object created by
148 * xe_mmio_gem_create().
149 *
150 * See: "Exposing MMIO regions to userspace"
151 */
xe_mmio_gem_destroy(struct xe_mmio_gem * gem,struct drm_file * file)152 void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file)
153 {
154 struct drm_gem_object *base = &gem->base;
155 struct drm_device *dev = base->dev;
156
157 drm_vma_node_revoke(&base->vma_node, file);
158
159 dma_resv_lock(base->resv, NULL);
160 gem->destroyed = true;
161 dma_resv_unlock(base->resv);
162 /*
163 * Setting 'destroyed' under lock takes care of the subsequent faults.
164 * Zap the existing PTEs to cut off access to the real MMIO through
165 * currently mapped pages.
166 */
167 drm_vma_node_unmap(&base->vma_node, dev->anon_inode->i_mapping);
168
169 drm_gem_object_put(base);
170 }
171
xe_mmio_gem_mmap(struct drm_gem_object * base,struct vm_area_struct * vma)172 static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *vma)
173 {
174 if (vma->vm_end - vma->vm_start != base->size)
175 return -EINVAL;
176
177 if ((vma->vm_flags & VM_SHARED) == 0)
178 return -EINVAL;
179
180 vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma));
181 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
182 VM_DONTCOPY | VM_NORESERVE);
183
184 /* Defer actual mapping to the fault handler. */
185 return 0;
186 }
187
alloc_dummy_page_if_needed(struct drm_gem_object * base)188 static int alloc_dummy_page_if_needed(struct drm_gem_object *base)
189 {
190 struct xe_mmio_gem *obj = to_xe_mmio_gem(base);
191
192 dma_resv_assert_held(base->resv);
193 if (!obj->dummy_page)
194 obj->dummy_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
195
196 return obj->dummy_page ? 0 : -ENOMEM;
197 }
198
xe_mmio_gem_vm_fault_dummy_page(struct vm_fault * vmf)199 static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf)
200 {
201 struct vm_area_struct *vma = vmf->vma;
202 struct drm_gem_object *base = vma->vm_private_data;
203 struct xe_mmio_gem *obj = to_xe_mmio_gem(base);
204 unsigned long pfn;
205
206 if (alloc_dummy_page_if_needed(base))
207 return VM_FAULT_OOM;
208
209 pfn = page_to_pfn(obj->dummy_page);
210
211 return vmf_insert_pfn_prot(vma, vmf->address, pfn,
212 vm_get_page_prot(vma->vm_flags));
213 }
214
xe_mmio_gem_vm_fault_locked(struct vm_fault * vmf)215 static vm_fault_t xe_mmio_gem_vm_fault_locked(struct vm_fault *vmf)
216 {
217 struct vm_area_struct *vma = vmf->vma;
218 struct drm_gem_object *base = vma->vm_private_data;
219 struct xe_mmio_gem *obj = to_xe_mmio_gem(base);
220 struct drm_device *dev = base->dev;
221 vm_fault_t ret = VM_FAULT_NOPAGE;
222 unsigned long addr, pfn;
223 int idx;
224
225 dma_resv_assert_held(base->resv);
226 if (obj->destroyed)
227 return VM_FAULT_SIGBUS;
228
229 if (!drm_dev_enter(dev, &idx)) {
230 /*
231 * Provide a dummy page to avoid SIGBUS for events such as hot-unplug.
232 * This gives the userspace the option to recover instead of crashing.
233 * It is assumed the userspace will receive the notification via some
234 * other channel (e.g. drm uevent).
235 */
236 return xe_mmio_gem_vm_fault_dummy_page(vmf);
237 }
238
239 pfn = PHYS_PFN(obj->phys_addr);
240 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
241 ret = vmf_insert_pfn(vma, addr, pfn);
242 if (ret & VM_FAULT_ERROR)
243 break;
244
245 pfn++;
246 }
247
248 drm_dev_exit(idx);
249 return ret;
250 }
251
xe_mmio_gem_vm_fault(struct vm_fault * vmf)252 static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf)
253 {
254 struct vm_area_struct *vma = vmf->vma;
255 struct drm_gem_object *base = vma->vm_private_data;
256 vm_fault_t ret;
257
258 dma_resv_lock(base->resv, NULL);
259 ret = xe_mmio_gem_vm_fault_locked(vmf);
260 dma_resv_unlock(base->resv);
261 return ret;
262 }
263