1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3 *
4 * Copyright (c) 2019-2025 Broadcom. All Rights Reserved. The term
5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
6 *
7 **************************************************************************/
8 #include "vmwgfx_bo.h"
9 #include "vmwgfx_drv.h"
10
11 /*
12 * Different methods for tracking dirty:
13 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
14 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
15 * accesses in the VM mkwrite() callback
16 */
17 enum vmw_bo_dirty_method {
18 VMW_BO_DIRTY_PAGETABLE,
19 VMW_BO_DIRTY_MKWRITE,
20 };
21
22 /*
23 * No dirtied pages at scan trigger a transition to the _MKWRITE method,
24 * similarly a certain percentage of dirty pages trigger a transition to
25 * the _PAGETABLE method. How many triggers should we wait for before
26 * changing method?
27 */
28 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
29
30 /* Percentage to trigger a transition to the _PAGETABLE method */
31 #define VMW_DIRTY_PERCENTAGE 10
32
33 /**
34 * struct vmw_bo_dirty - Dirty information for buffer objects
35 * @ref_count: Reference count for this structure. Must be first member!
36 * @start: First currently dirty bit
37 * @end: Last currently dirty bit + 1
38 * @method: The currently used dirty method
39 * @change_count: Number of consecutive method change triggers
40 * @bitmap_size: The size of the bitmap in bits. Typically equal to the
41 * nuber of pages in the bo.
42 * @bitmap: A bitmap where each bit represents a page. A set bit means a
43 * dirty page.
44 */
45 struct vmw_bo_dirty {
46 struct kref ref_count;
47 unsigned long start;
48 unsigned long end;
49 enum vmw_bo_dirty_method method;
50 unsigned int change_count;
51 unsigned long bitmap_size;
52 unsigned long bitmap[];
53 };
54
vmw_bo_is_dirty(struct vmw_bo * vbo)55 bool vmw_bo_is_dirty(struct vmw_bo *vbo)
56 {
57 return vbo->dirty && (vbo->dirty->start < vbo->dirty->end);
58 }
59
60 /**
61 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
62 * @vbo: The buffer object to scan
63 *
64 * Scans the pagetable for dirty bits. Clear those bits and modify the
65 * dirty structure with the results. This function may change the
66 * dirty-tracking method.
67 */
vmw_bo_dirty_scan_pagetable(struct vmw_bo * vbo)68 static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)
69 {
70 struct vmw_bo_dirty *dirty = vbo->dirty;
71 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
72 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
73 pgoff_t num_marked;
74
75 num_marked = clean_record_shared_mapping_range
76 (mapping,
77 offset, dirty->bitmap_size,
78 offset, &dirty->bitmap[0],
79 &dirty->start, &dirty->end);
80 if (num_marked == 0)
81 dirty->change_count++;
82 else
83 dirty->change_count = 0;
84
85 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
86 dirty->change_count = 0;
87 dirty->method = VMW_BO_DIRTY_MKWRITE;
88 wp_shared_mapping_range(mapping,
89 offset, dirty->bitmap_size);
90 clean_record_shared_mapping_range(mapping,
91 offset, dirty->bitmap_size,
92 offset, &dirty->bitmap[0],
93 &dirty->start, &dirty->end);
94 }
95 }
96
97 /**
98 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
99 * @vbo: The buffer object to scan
100 *
101 * Write-protect pages written to so that consecutive write accesses will
102 * trigger a call to mkwrite.
103 *
104 * This function may change the dirty-tracking method.
105 */
vmw_bo_dirty_scan_mkwrite(struct vmw_bo * vbo)106 static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)
107 {
108 struct vmw_bo_dirty *dirty = vbo->dirty;
109 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
110 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
111 pgoff_t num_marked;
112
113 if (dirty->end <= dirty->start)
114 return;
115
116 num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,
117 dirty->start + offset,
118 dirty->end - dirty->start);
119
120 if (100UL * num_marked / dirty->bitmap_size >
121 VMW_DIRTY_PERCENTAGE)
122 dirty->change_count++;
123 else
124 dirty->change_count = 0;
125
126 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
127 pgoff_t start = 0;
128 pgoff_t end = dirty->bitmap_size;
129
130 dirty->method = VMW_BO_DIRTY_PAGETABLE;
131 clean_record_shared_mapping_range(mapping, offset, end, offset,
132 &dirty->bitmap[0],
133 &start, &end);
134 bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
135 if (dirty->start < dirty->end)
136 bitmap_set(&dirty->bitmap[0], dirty->start,
137 dirty->end - dirty->start);
138 dirty->change_count = 0;
139 }
140 }
141
142 /**
143 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
144 * tracking structure
145 * @vbo: The buffer object to scan
146 *
147 * This function may change the dirty tracking method.
148 */
vmw_bo_dirty_scan(struct vmw_bo * vbo)149 void vmw_bo_dirty_scan(struct vmw_bo *vbo)
150 {
151 struct vmw_bo_dirty *dirty = vbo->dirty;
152
153 if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
154 vmw_bo_dirty_scan_pagetable(vbo);
155 else
156 vmw_bo_dirty_scan_mkwrite(vbo);
157 }
158
159 /**
160 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
161 * an unmap_mapping_range operation.
162 * @vbo: The buffer object,
163 * @start: First page of the range within the buffer object.
164 * @end: Last page of the range within the buffer object + 1.
165 *
166 * If we're using the _PAGETABLE scan method, we may leak dirty pages
167 * when calling unmap_mapping_range(). This function makes sure we pick
168 * up all dirty pages.
169 */
vmw_bo_dirty_pre_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)170 static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,
171 pgoff_t start, pgoff_t end)
172 {
173 struct vmw_bo_dirty *dirty = vbo->dirty;
174 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
175 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
176
177 if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
178 return;
179
180 wp_shared_mapping_range(mapping, start + offset, end - start);
181 clean_record_shared_mapping_range(mapping, start + offset,
182 end - start, offset,
183 &dirty->bitmap[0], &dirty->start,
184 &dirty->end);
185 }
186
187 /**
188 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
189 * @vbo: The buffer object,
190 * @start: First page of the range within the buffer object.
191 * @end: Last page of the range within the buffer object + 1.
192 *
193 * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
194 */
vmw_bo_dirty_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)195 void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
196 pgoff_t start, pgoff_t end)
197 {
198 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
199 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
200
201 vmw_bo_dirty_pre_unmap(vbo, start, end);
202 unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
203 (loff_t) (end - start) << PAGE_SHIFT);
204 }
205
206 /**
207 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
208 * @vbo: The buffer object
209 *
210 * This function registers a dirty-tracking user to a buffer object.
211 * A user can be for example a resource or a vma in a special user-space
212 * mapping.
213 *
214 * Return: Zero on success, -ENOMEM on memory allocation failure.
215 */
vmw_bo_dirty_add(struct vmw_bo * vbo)216 int vmw_bo_dirty_add(struct vmw_bo *vbo)
217 {
218 struct vmw_bo_dirty *dirty = vbo->dirty;
219 pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);
220 size_t size;
221 int ret;
222
223 if (dirty) {
224 kref_get(&dirty->ref_count);
225 return 0;
226 }
227
228 size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
229 dirty = kvzalloc(size, GFP_KERNEL);
230 if (!dirty) {
231 ret = -ENOMEM;
232 goto out_no_dirty;
233 }
234
235 dirty->bitmap_size = num_pages;
236 dirty->start = dirty->bitmap_size;
237 dirty->end = 0;
238 kref_init(&dirty->ref_count);
239 if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
240 dirty->method = VMW_BO_DIRTY_PAGETABLE;
241 } else {
242 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
243 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
244
245 dirty->method = VMW_BO_DIRTY_MKWRITE;
246
247 /* Write-protect and then pick up already dirty bits */
248 wp_shared_mapping_range(mapping, offset, num_pages);
249 clean_record_shared_mapping_range(mapping, offset, num_pages,
250 offset,
251 &dirty->bitmap[0],
252 &dirty->start, &dirty->end);
253 }
254
255 vbo->dirty = dirty;
256
257 return 0;
258
259 out_no_dirty:
260 return ret;
261 }
262
vmw_bo_dirty_free(struct kref * kref)263 static void vmw_bo_dirty_free(struct kref *kref)
264 {
265 struct vmw_bo_dirty *dirty = container_of(kref, struct vmw_bo_dirty, ref_count);
266
267 kvfree(dirty);
268 }
269
270 /**
271 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
272 * @vbo: The buffer object
273 *
274 * This function releases a dirty-tracking user from a buffer object.
275 * If the reference count reaches zero, then the dirty-tracking object is
276 * freed and the pointer to it cleared.
277 *
278 * Return: Zero on success, -ENOMEM on memory allocation failure.
279 */
vmw_bo_dirty_release(struct vmw_bo * vbo)280 void vmw_bo_dirty_release(struct vmw_bo *vbo)
281 {
282 struct vmw_bo_dirty *dirty = vbo->dirty;
283
284 if (dirty && kref_put(&dirty->ref_count, vmw_bo_dirty_free))
285 vbo->dirty = NULL;
286 }
287
288 /**
289 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
290 * its backing mob.
291 * @res: The resource
292 *
293 * This function will pick up all dirty ranges affecting the resource from
294 * it's backup mob, and call vmw_resource_dirty_update() once for each
295 * range. The transferred ranges will be cleared from the backing mob's
296 * dirty tracking.
297 */
vmw_bo_dirty_transfer_to_res(struct vmw_resource * res)298 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
299 {
300 struct vmw_bo *vbo = res->guest_memory_bo;
301 struct vmw_bo_dirty *dirty = vbo->dirty;
302 pgoff_t start, cur, end;
303 unsigned long res_start = res->guest_memory_offset;
304 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
305
306 WARN_ON_ONCE(res_start & ~PAGE_MASK);
307 res_start >>= PAGE_SHIFT;
308 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
309
310 if (res_start >= dirty->end || res_end <= dirty->start)
311 return;
312
313 cur = max(res_start, dirty->start);
314 res_end = max(res_end, dirty->end);
315 while (cur < res_end) {
316 unsigned long num;
317
318 start = find_next_bit(&dirty->bitmap[0], res_end, cur);
319 if (start >= res_end)
320 break;
321
322 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
323 cur = end + 1;
324 num = end - start;
325 bitmap_clear(&dirty->bitmap[0], start, num);
326 vmw_resource_dirty_update(res, start, end);
327 }
328
329 if (res_start <= dirty->start && res_end > dirty->start)
330 dirty->start = res_end;
331 if (res_start < dirty->end && res_end >= dirty->end)
332 dirty->end = res_start;
333 }
334
vmw_bo_dirty_clear(struct vmw_bo * vbo)335 void vmw_bo_dirty_clear(struct vmw_bo *vbo)
336 {
337 struct vmw_bo_dirty *dirty = vbo->dirty;
338 pgoff_t start, cur, end;
339 unsigned long res_start = 0;
340 unsigned long res_end = vbo->tbo.base.size;
341
342 WARN_ON_ONCE(res_start & ~PAGE_MASK);
343 res_start >>= PAGE_SHIFT;
344 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
345
346 if (res_start >= dirty->end || res_end <= dirty->start)
347 return;
348
349 cur = max(res_start, dirty->start);
350 res_end = max(res_end, dirty->end);
351 while (cur < res_end) {
352 unsigned long num;
353
354 start = find_next_bit(&dirty->bitmap[0], res_end, cur);
355 if (start >= res_end)
356 break;
357
358 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
359 cur = end + 1;
360 num = end - start;
361 bitmap_clear(&dirty->bitmap[0], start, num);
362 }
363
364 if (res_start <= dirty->start && res_end > dirty->start)
365 dirty->start = res_end;
366 if (res_start < dirty->end && res_end >= dirty->end)
367 dirty->end = res_start;
368 }
369
370 /**
371 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
372 * its backing mob.
373 * @res: The resource
374 *
375 * This function will clear all dirty ranges affecting the resource from
376 * it's backup mob's dirty tracking.
377 */
vmw_bo_dirty_clear_res(struct vmw_resource * res)378 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
379 {
380 unsigned long res_start = res->guest_memory_offset;
381 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
382 struct vmw_bo *vbo = res->guest_memory_bo;
383 struct vmw_bo_dirty *dirty = vbo->dirty;
384
385 res_start >>= PAGE_SHIFT;
386 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
387
388 if (res_start >= dirty->end || res_end <= dirty->start)
389 return;
390
391 res_start = max(res_start, dirty->start);
392 res_end = min(res_end, dirty->end);
393 bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
394
395 if (res_start <= dirty->start && res_end > dirty->start)
396 dirty->start = res_end;
397 if (res_start < dirty->end && res_end >= dirty->end)
398 dirty->end = res_start;
399 }
400
vmw_bo_vm_mkwrite(struct vm_fault * vmf)401 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
402 {
403 struct vm_area_struct *vma = vmf->vma;
404 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
405 vma->vm_private_data;
406 vm_fault_t ret;
407 unsigned long page_offset;
408 unsigned int save_flags;
409 struct vmw_bo *vbo = to_vmw_bo(&bo->base);
410
411 /*
412 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
413 * So make sure the TTM helpers are aware.
414 */
415 save_flags = vmf->flags;
416 vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
417 ret = ttm_bo_vm_reserve(bo, vmf);
418 vmf->flags = save_flags;
419 if (ret)
420 return ret;
421
422 page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
423 if (unlikely(page_offset >= PFN_UP(bo->resource->size))) {
424 ret = VM_FAULT_SIGBUS;
425 goto out_unlock;
426 }
427
428 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
429 !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
430 struct vmw_bo_dirty *dirty = vbo->dirty;
431
432 __set_bit(page_offset, &dirty->bitmap[0]);
433 dirty->start = min(dirty->start, page_offset);
434 dirty->end = max(dirty->end, page_offset + 1);
435 }
436
437 out_unlock:
438 dma_resv_unlock(bo->base.resv);
439 return ret;
440 }
441
vmw_bo_vm_fault(struct vm_fault * vmf)442 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
443 {
444 struct vm_area_struct *vma = vmf->vma;
445 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
446 vma->vm_private_data;
447 struct vmw_bo *vbo = to_vmw_bo(&bo->base);
448 pgoff_t num_prefault;
449 pgprot_t prot;
450 vm_fault_t ret;
451
452 ret = ttm_bo_vm_reserve(bo, vmf);
453 if (ret)
454 return ret;
455
456 num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
457 TTM_BO_VM_NUM_PREFAULT;
458
459 if (vbo->dirty) {
460 pgoff_t allowed_prefault;
461 unsigned long page_offset;
462
463 page_offset = vmf->pgoff -
464 drm_vma_node_start(&bo->base.vma_node);
465 if (page_offset >= PFN_UP(bo->resource->size) ||
466 vmw_resources_clean(vbo, page_offset,
467 page_offset + PAGE_SIZE,
468 &allowed_prefault)) {
469 ret = VM_FAULT_SIGBUS;
470 goto out_unlock;
471 }
472
473 num_prefault = min(num_prefault, allowed_prefault);
474 }
475
476 /*
477 * If we don't track dirty using the MKWRITE method, make sure
478 * sure the page protection is write-enabled so we don't get
479 * a lot of unnecessary write faults.
480 */
481 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
482 prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
483 else
484 prot = vm_get_page_prot(vma->vm_flags);
485
486 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
487 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
488 return ret;
489
490 out_unlock:
491 dma_resv_unlock(bo->base.resv);
492
493 return ret;
494 }
495