1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_backup.h> 7 8 #include <linux/export.h> 9 #include <linux/swap.h> 10 11 #include "ttm_pool_internal.h" 12 13 /* 14 * Need to map shmem indices to handle since a handle value 15 * of 0 means error, following the swp_entry_t convention. 16 */ 17 static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx) 18 { 19 return (unsigned long)idx + 1; 20 } 21 22 static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle) 23 { 24 return handle - 1; 25 } 26 27 /** 28 * ttm_backup_drop() - release memory associated with a handle 29 * @backup: The struct backup pointer used to obtain the handle 30 * @handle: The handle obtained from the @backup_page function. 31 */ 32 void ttm_backup_drop(struct file *backup, pgoff_t handle) 33 { 34 loff_t start = ttm_backup_handle_to_shmem_idx(handle); 35 36 start <<= PAGE_SHIFT; 37 shmem_truncate_range(file_inode(backup), start, 38 start + PAGE_SIZE - 1); 39 } 40 41 /** 42 * ttm_backup_copy_page() - Copy the contents of a previously backed 43 * up page 44 * @backup: The struct backup pointer used to back up the page. 45 * @dst: The struct page to copy into. 46 * @handle: The handle returned when the page was backed up. 47 * @intr: Try to perform waits interruptible or at least killable. 48 * @additional_gfp: GFP mask to add to the default GFP mask if any. 49 * 50 * Return: 0 on success, Negative error code on failure, notably 51 * -EINTR if @intr was set to true and a signal is pending. 52 */ 53 int ttm_backup_copy_page(struct file *backup, struct page *dst, 54 pgoff_t handle, bool intr, gfp_t additional_gfp) 55 { 56 struct address_space *mapping = backup->f_mapping; 57 struct folio *from_folio; 58 pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle); 59 60 from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping) 61 | additional_gfp); 62 if (IS_ERR(from_folio)) 63 return PTR_ERR(from_folio); 64 65 copy_highpage(dst, folio_file_page(from_folio, idx)); 66 folio_put(from_folio); 67 68 return 0; 69 } 70 71 /** 72 * ttm_backup_backup_folio() - Backup a folio 73 * @backup: The struct backup pointer to use. 74 * @folio: The folio to back up. 75 * @order: The allocation order of @folio. Since TTM allocates higher-order 76 * pages without __GFP_COMP, folio_nr_pages(@folio) would always 77 * return 1; the caller must pass the true order explicitly. 78 * @writeback: Whether to perform immediate writeback of the folio's pages. 79 * This may have performance implications. 80 * @idx: A unique integer for the first page of the folio and each struct backup. 81 * This allows the backup implementation to avoid managing 82 * its address space separately. 83 * @folio_gfp: The gfp value used when the folio was allocated. 84 * Currently unused. 85 * @alloc_gfp: The gfp to be used when allocating memory. 86 * @nr_pages_backed: Output. On a successful return, set to the number of 87 * pages actually backed up, which may be less than (1 << @order) 88 * if an -ENOMEM was encountered mid-folio. 89 * 90 * Context: If called from reclaim context, the caller needs to 91 * assert that the shrinker gfp has __GFP_FS set, to avoid 92 * deadlocking on lock_page(). If @writeback is set to true and 93 * called from reclaim context, the caller also needs to assert 94 * that the shrinker gfp has __GFP_IO set, since without it, 95 * we're not allowed to start backup IO. 96 * 97 * Return: A handle for the first backed-up page on success (handles for 98 * subsequent pages follow sequentially). -ENOMEM if no pages could be backed 99 * up. Any other negative error code if a non-ENOMEM failure occurred; in that 100 * case any pages backed up so far are truncated before returning. 101 */ 102 s64 103 ttm_backup_backup_folio(struct file *backup, struct folio *folio, 104 unsigned int order, bool writeback, pgoff_t idx, 105 gfp_t folio_gfp, gfp_t alloc_gfp, 106 pgoff_t *nr_pages_backed) 107 { 108 struct address_space *mapping = backup->f_mapping; 109 int nr_pages = 1 << order; 110 struct folio *to_folio; 111 int ret, i; 112 113 *nr_pages_backed = 0; 114 115 for (i = 0; i < nr_pages; ) { 116 int to_nr, j; 117 118 /* 119 * Only inject past the first subpage so *nr_pages_backed is 120 * always > 0 here, matching a genuine mid-compound -ENOMEM 121 * and driving the caller's reactive split fallback instead 122 * of an early, no-progress failure. 123 */ 124 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && i && 125 ttm_backup_fault_inject_folio()) 126 to_folio = ERR_PTR(-ENOMEM); 127 else 128 to_folio = shmem_read_folio_gfp(mapping, idx + i, alloc_gfp); 129 if (IS_ERR(to_folio)) { 130 int err = PTR_ERR(to_folio); 131 132 if (err == -ENOMEM && *nr_pages_backed) 133 return ttm_backup_shmem_idx_to_handle(idx); 134 135 if (*nr_pages_backed) { 136 shmem_truncate_range(file_inode(backup), 137 (loff_t)idx << PAGE_SHIFT, 138 ((loff_t)(idx + i) << PAGE_SHIFT) - 1); 139 /* 140 * The pages just truncated are no longer 141 * backed up; don't let the caller mistake 142 * them for valid handles. 143 */ 144 *nr_pages_backed = 0; 145 } 146 return err; 147 } 148 149 to_nr = min_t(int, nr_pages - i, 150 folio_next_index(to_folio) - (idx + i)); 151 152 folio_mark_accessed(to_folio); 153 folio_lock(to_folio); 154 folio_mark_dirty(to_folio); 155 156 for (j = 0; j < to_nr; j++) 157 copy_highpage(folio_file_page(to_folio, idx + i + j), 158 folio_page(folio, i + j)); 159 160 if (writeback && !folio_mapped(to_folio) && 161 folio_clear_dirty_for_io(to_folio)) { 162 folio_set_reclaim(to_folio); 163 ret = shmem_writeout(to_folio, NULL, NULL); 164 if (!folio_test_writeback(to_folio)) 165 folio_clear_reclaim(to_folio); 166 if (ret == AOP_WRITEPAGE_ACTIVATE) 167 folio_unlock(to_folio); 168 } else { 169 folio_unlock(to_folio); 170 } 171 172 folio_put(to_folio); 173 i += to_nr; 174 *nr_pages_backed = i; 175 } 176 177 return ttm_backup_shmem_idx_to_handle(idx); 178 } 179 180 /** 181 * ttm_backup_fini() - Free the struct backup resources after last use. 182 * @backup: Pointer to the struct backup whose resources to free. 183 * 184 * After a call to this function, it's illegal to use the @backup pointer. 185 */ 186 void ttm_backup_fini(struct file *backup) 187 { 188 fput(backup); 189 } 190 191 /** 192 * ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space 193 * left for backup. 194 * 195 * This function is intended also for driver use to indicate whether a 196 * backup attempt is meaningful. 197 * 198 * Return: An approximate size of backup space available. 199 */ 200 u64 ttm_backup_bytes_avail(void) 201 { 202 /* 203 * The idea behind backing up to shmem is that shmem objects may 204 * eventually be swapped out. So no point swapping out if there 205 * is no or low swap-space available. But the accuracy of this 206 * number also depends on shmem actually swapping out backed-up 207 * shmem objects without too much buffering. 208 */ 209 return (u64)get_nr_swap_pages() << PAGE_SHIFT; 210 } 211 EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail); 212 213 /** 214 * ttm_backup_shmem_create() - Create a shmem-based struct backup. 215 * @size: The maximum size (in bytes) to back up. 216 * 217 * Create a backup utilizing shmem objects. 218 * 219 * Return: A pointer to a struct file on success, 220 * an error pointer on error. 221 */ 222 struct file *ttm_backup_shmem_create(loff_t size) 223 { 224 return shmem_file_setup("ttm shmem backup", size, 225 EMPTY_VMA_FLAGS); 226 } 227