1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2024 Intel Corporation
4 */
5
6 #include <drm/ttm/ttm_backup.h>
7
8 #include <linux/export.h>
9 #include <linux/swap.h>
10
11 #include "ttm_pool_internal.h"
12
13 /*
14 * Need to map shmem indices to handle since a handle value
15 * of 0 means error, following the swp_entry_t convention.
16 */
ttm_backup_shmem_idx_to_handle(pgoff_t idx)17 static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx)
18 {
19 return (unsigned long)idx + 1;
20 }
21
ttm_backup_handle_to_shmem_idx(pgoff_t handle)22 static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle)
23 {
24 return handle - 1;
25 }
26
27 /**
28 * ttm_backup_drop() - release memory associated with a handle
29 * @backup: The struct backup pointer used to obtain the handle
30 * @handle: The handle obtained from the @backup_page function.
31 */
ttm_backup_drop(struct file * backup,pgoff_t handle)32 void ttm_backup_drop(struct file *backup, pgoff_t handle)
33 {
34 loff_t start = ttm_backup_handle_to_shmem_idx(handle);
35
36 start <<= PAGE_SHIFT;
37 shmem_truncate_range(file_inode(backup), start,
38 start + PAGE_SIZE - 1);
39 }
40
41 /**
42 * ttm_backup_copy_page() - Copy the contents of a previously backed
43 * up page
44 * @backup: The struct backup pointer used to back up the page.
45 * @dst: The struct page to copy into.
46 * @handle: The handle returned when the page was backed up.
47 * @intr: Try to perform waits interruptible or at least killable.
48 * @additional_gfp: GFP mask to add to the default GFP mask if any.
49 *
50 * Return: 0 on success, Negative error code on failure, notably
51 * -EINTR if @intr was set to true and a signal is pending.
52 */
ttm_backup_copy_page(struct file * backup,struct page * dst,pgoff_t handle,bool intr,gfp_t additional_gfp)53 int ttm_backup_copy_page(struct file *backup, struct page *dst,
54 pgoff_t handle, bool intr, gfp_t additional_gfp)
55 {
56 struct address_space *mapping = backup->f_mapping;
57 struct folio *from_folio;
58 pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle);
59
60 from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping)
61 | additional_gfp);
62 if (IS_ERR(from_folio))
63 return PTR_ERR(from_folio);
64
65 copy_highpage(dst, folio_file_page(from_folio, idx));
66 folio_put(from_folio);
67
68 return 0;
69 }
70
71 /**
72 * ttm_backup_backup_folio() - Backup a folio
73 * @backup: The struct backup pointer to use.
74 * @folio: The folio to back up.
75 * @order: The allocation order of @folio. Since TTM allocates higher-order
76 * pages without __GFP_COMP, folio_nr_pages(@folio) would always
77 * return 1; the caller must pass the true order explicitly.
78 * @writeback: Whether to perform immediate writeback of the folio's pages.
79 * This may have performance implications.
80 * @idx: A unique integer for the first page of the folio and each struct backup.
81 * This allows the backup implementation to avoid managing
82 * its address space separately.
83 * @folio_gfp: The gfp value used when the folio was allocated.
84 * Currently unused.
85 * @alloc_gfp: The gfp to be used when allocating memory.
86 * @nr_pages_backed: Output. On a successful return, set to the number of
87 * pages actually backed up, which may be less than (1 << @order)
88 * if an -ENOMEM was encountered mid-folio.
89 *
90 * Context: If called from reclaim context, the caller needs to
91 * assert that the shrinker gfp has __GFP_FS set, to avoid
92 * deadlocking on lock_page(). If @writeback is set to true and
93 * called from reclaim context, the caller also needs to assert
94 * that the shrinker gfp has __GFP_IO set, since without it,
95 * we're not allowed to start backup IO.
96 *
97 * Return: A handle for the first backed-up page on success (handles for
98 * subsequent pages follow sequentially). -ENOMEM if no pages could be backed
99 * up. Any other negative error code if a non-ENOMEM failure occurred; in that
100 * case any pages backed up so far are truncated before returning.
101 */
102 s64
ttm_backup_backup_folio(struct file * backup,struct folio * folio,unsigned int order,bool writeback,pgoff_t idx,gfp_t folio_gfp,gfp_t alloc_gfp,pgoff_t * nr_pages_backed)103 ttm_backup_backup_folio(struct file *backup, struct folio *folio,
104 unsigned int order, bool writeback, pgoff_t idx,
105 gfp_t folio_gfp, gfp_t alloc_gfp,
106 pgoff_t *nr_pages_backed)
107 {
108 struct address_space *mapping = backup->f_mapping;
109 int nr_pages = 1 << order;
110 struct folio *to_folio;
111 int ret, i;
112
113 *nr_pages_backed = 0;
114
115 for (i = 0; i < nr_pages; ) {
116 int to_nr, j;
117
118 /*
119 * Only inject past the first subpage so *nr_pages_backed is
120 * always > 0 here, matching a genuine mid-compound -ENOMEM
121 * and driving the caller's reactive split fallback instead
122 * of an early, no-progress failure.
123 */
124 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && i &&
125 ttm_backup_fault_inject_folio())
126 to_folio = ERR_PTR(-ENOMEM);
127 else
128 to_folio = shmem_read_folio_gfp(mapping, idx + i, alloc_gfp);
129 if (IS_ERR(to_folio)) {
130 int err = PTR_ERR(to_folio);
131
132 if (err == -ENOMEM && *nr_pages_backed)
133 return ttm_backup_shmem_idx_to_handle(idx);
134
135 if (*nr_pages_backed) {
136 shmem_truncate_range(file_inode(backup),
137 (loff_t)idx << PAGE_SHIFT,
138 ((loff_t)(idx + i) << PAGE_SHIFT) - 1);
139 /*
140 * The pages just truncated are no longer
141 * backed up; don't let the caller mistake
142 * them for valid handles.
143 */
144 *nr_pages_backed = 0;
145 }
146 return err;
147 }
148
149 to_nr = min_t(int, nr_pages - i,
150 folio_next_index(to_folio) - (idx + i));
151
152 folio_mark_accessed(to_folio);
153 folio_lock(to_folio);
154 folio_mark_dirty(to_folio);
155
156 for (j = 0; j < to_nr; j++)
157 copy_highpage(folio_file_page(to_folio, idx + i + j),
158 folio_page(folio, i + j));
159
160 if (writeback && !folio_mapped(to_folio) &&
161 folio_clear_dirty_for_io(to_folio)) {
162 folio_set_reclaim(to_folio);
163 ret = shmem_writeout(to_folio, NULL, NULL);
164 if (!folio_test_writeback(to_folio))
165 folio_clear_reclaim(to_folio);
166 if (ret == AOP_WRITEPAGE_ACTIVATE)
167 folio_unlock(to_folio);
168 } else {
169 folio_unlock(to_folio);
170 }
171
172 folio_put(to_folio);
173 i += to_nr;
174 *nr_pages_backed = i;
175 }
176
177 return ttm_backup_shmem_idx_to_handle(idx);
178 }
179
180 /**
181 * ttm_backup_fini() - Free the struct backup resources after last use.
182 * @backup: Pointer to the struct backup whose resources to free.
183 *
184 * After a call to this function, it's illegal to use the @backup pointer.
185 */
ttm_backup_fini(struct file * backup)186 void ttm_backup_fini(struct file *backup)
187 {
188 fput(backup);
189 }
190
191 /**
192 * ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space
193 * left for backup.
194 *
195 * This function is intended also for driver use to indicate whether a
196 * backup attempt is meaningful.
197 *
198 * Return: An approximate size of backup space available.
199 */
ttm_backup_bytes_avail(void)200 u64 ttm_backup_bytes_avail(void)
201 {
202 /*
203 * The idea behind backing up to shmem is that shmem objects may
204 * eventually be swapped out. So no point swapping out if there
205 * is no or low swap-space available. But the accuracy of this
206 * number also depends on shmem actually swapping out backed-up
207 * shmem objects without too much buffering.
208 */
209 return (u64)get_nr_swap_pages() << PAGE_SHIFT;
210 }
211 EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail);
212
213 /**
214 * ttm_backup_shmem_create() - Create a shmem-based struct backup.
215 * @size: The maximum size (in bytes) to back up.
216 *
217 * Create a backup utilizing shmem objects.
218 *
219 * Return: A pointer to a struct file on success,
220 * an error pointer on error.
221 */
ttm_backup_shmem_create(loff_t size)222 struct file *ttm_backup_shmem_create(loff_t size)
223 {
224 return shmem_file_setup("ttm shmem backup", size,
225 EMPTY_VMA_FLAGS);
226 }
227