1 /* 2 * linux/mm/page_io.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 * 6 * Swap reorganised 29.12.95, 7 * Asynchronous swapping added 30.12.95. Stephen Tweedie 8 * Removed race in async swapping. 14.4.1996. Bruno Haible 9 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie 10 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman 11 */ 12 13 #include <linux/mm.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/gfp.h> 16 #include <linux/pagemap.h> 17 #include <linux/swap.h> 18 #include <linux/bio.h> 19 #include <linux/swapops.h> 20 #include <linux/buffer_head.h> 21 #include <linux/writeback.h> 22 #include <linux/frontswap.h> 23 #include <linux/aio.h> 24 #include <linux/blkdev.h> 25 #include <asm/pgtable.h> 26 27 static struct bio *get_swap_bio(gfp_t gfp_flags, 28 struct page *page, bio_end_io_t end_io) 29 { 30 struct bio *bio; 31 32 bio = bio_alloc(gfp_flags, 1); 33 if (bio) { 34 bio->bi_sector = map_swap_page(page, &bio->bi_bdev); 35 bio->bi_sector <<= PAGE_SHIFT - 9; 36 bio->bi_io_vec[0].bv_page = page; 37 bio->bi_io_vec[0].bv_len = PAGE_SIZE; 38 bio->bi_io_vec[0].bv_offset = 0; 39 bio->bi_vcnt = 1; 40 bio->bi_size = PAGE_SIZE; 41 bio->bi_end_io = end_io; 42 } 43 return bio; 44 } 45 46 void end_swap_bio_write(struct bio *bio, int err) 47 { 48 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 49 struct page *page = bio->bi_io_vec[0].bv_page; 50 51 if (!uptodate) { 52 SetPageError(page); 53 /* 54 * We failed to write the page out to swap-space. 55 * Re-dirty the page in order to avoid it being reclaimed. 56 * Also print a dire warning that things will go BAD (tm) 57 * very quickly. 58 * 59 * Also clear PG_reclaim to avoid rotate_reclaimable_page() 60 */ 61 set_page_dirty(page); 62 printk(KERN_ALERT "Write-error on swap-device (%u:%u:%Lu)\n", 63 imajor(bio->bi_bdev->bd_inode), 64 iminor(bio->bi_bdev->bd_inode), 65 (unsigned long long)bio->bi_sector); 66 ClearPageReclaim(page); 67 } 68 end_page_writeback(page); 69 bio_put(bio); 70 } 71 72 void end_swap_bio_read(struct bio *bio, int err) 73 { 74 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 75 struct page *page = bio->bi_io_vec[0].bv_page; 76 77 if (!uptodate) { 78 SetPageError(page); 79 ClearPageUptodate(page); 80 printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", 81 imajor(bio->bi_bdev->bd_inode), 82 iminor(bio->bi_bdev->bd_inode), 83 (unsigned long long)bio->bi_sector); 84 goto out; 85 } 86 87 SetPageUptodate(page); 88 89 /* 90 * There is no guarantee that the page is in swap cache - the software 91 * suspend code (at least) uses end_swap_bio_read() against a non- 92 * swapcache page. So we must check PG_swapcache before proceeding with 93 * this optimization. 94 */ 95 if (likely(PageSwapCache(page))) { 96 struct swap_info_struct *sis; 97 98 sis = page_swap_info(page); 99 if (sis->flags & SWP_BLKDEV) { 100 /* 101 * The swap subsystem performs lazy swap slot freeing, 102 * expecting that the page will be swapped out again. 103 * So we can avoid an unnecessary write if the page 104 * isn't redirtied. 105 * This is good for real swap storage because we can 106 * reduce unnecessary I/O and enhance wear-leveling 107 * if an SSD is used as the as swap device. 108 * But if in-memory swap device (eg zram) is used, 109 * this causes a duplicated copy between uncompressed 110 * data in VM-owned memory and compressed data in 111 * zram-owned memory. So let's free zram-owned memory 112 * and make the VM-owned decompressed page *dirty*, 113 * so the page should be swapped out somewhere again if 114 * we again wish to reclaim it. 115 */ 116 struct gendisk *disk = sis->bdev->bd_disk; 117 if (disk->fops->swap_slot_free_notify) { 118 swp_entry_t entry; 119 unsigned long offset; 120 121 entry.val = page_private(page); 122 offset = swp_offset(entry); 123 124 SetPageDirty(page); 125 disk->fops->swap_slot_free_notify(sis->bdev, 126 offset); 127 } 128 } 129 } 130 131 out: 132 unlock_page(page); 133 bio_put(bio); 134 } 135 136 int generic_swapfile_activate(struct swap_info_struct *sis, 137 struct file *swap_file, 138 sector_t *span) 139 { 140 struct address_space *mapping = swap_file->f_mapping; 141 struct inode *inode = mapping->host; 142 unsigned blocks_per_page; 143 unsigned long page_no; 144 unsigned blkbits; 145 sector_t probe_block; 146 sector_t last_block; 147 sector_t lowest_block = -1; 148 sector_t highest_block = 0; 149 int nr_extents = 0; 150 int ret; 151 152 blkbits = inode->i_blkbits; 153 blocks_per_page = PAGE_SIZE >> blkbits; 154 155 /* 156 * Map all the blocks into the extent list. This code doesn't try 157 * to be very smart. 158 */ 159 probe_block = 0; 160 page_no = 0; 161 last_block = i_size_read(inode) >> blkbits; 162 while ((probe_block + blocks_per_page) <= last_block && 163 page_no < sis->max) { 164 unsigned block_in_page; 165 sector_t first_block; 166 167 first_block = bmap(inode, probe_block); 168 if (first_block == 0) 169 goto bad_bmap; 170 171 /* 172 * It must be PAGE_SIZE aligned on-disk 173 */ 174 if (first_block & (blocks_per_page - 1)) { 175 probe_block++; 176 goto reprobe; 177 } 178 179 for (block_in_page = 1; block_in_page < blocks_per_page; 180 block_in_page++) { 181 sector_t block; 182 183 block = bmap(inode, probe_block + block_in_page); 184 if (block == 0) 185 goto bad_bmap; 186 if (block != first_block + block_in_page) { 187 /* Discontiguity */ 188 probe_block++; 189 goto reprobe; 190 } 191 } 192 193 first_block >>= (PAGE_SHIFT - blkbits); 194 if (page_no) { /* exclude the header page */ 195 if (first_block < lowest_block) 196 lowest_block = first_block; 197 if (first_block > highest_block) 198 highest_block = first_block; 199 } 200 201 /* 202 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 203 */ 204 ret = add_swap_extent(sis, page_no, 1, first_block); 205 if (ret < 0) 206 goto out; 207 nr_extents += ret; 208 page_no++; 209 probe_block += blocks_per_page; 210 reprobe: 211 continue; 212 } 213 ret = nr_extents; 214 *span = 1 + highest_block - lowest_block; 215 if (page_no == 0) 216 page_no = 1; /* force Empty message */ 217 sis->max = page_no; 218 sis->pages = page_no - 1; 219 sis->highest_bit = page_no - 1; 220 out: 221 return ret; 222 bad_bmap: 223 printk(KERN_ERR "swapon: swapfile has holes\n"); 224 ret = -EINVAL; 225 goto out; 226 } 227 228 /* 229 * We may have stale swap cache pages in memory: notice 230 * them here and get rid of the unnecessary final write. 231 */ 232 int swap_writepage(struct page *page, struct writeback_control *wbc) 233 { 234 int ret = 0; 235 236 if (try_to_free_swap(page)) { 237 unlock_page(page); 238 goto out; 239 } 240 if (frontswap_store(page) == 0) { 241 set_page_writeback(page); 242 unlock_page(page); 243 end_page_writeback(page); 244 goto out; 245 } 246 ret = __swap_writepage(page, wbc, end_swap_bio_write); 247 out: 248 return ret; 249 } 250 251 int __swap_writepage(struct page *page, struct writeback_control *wbc, 252 void (*end_write_func)(struct bio *, int)) 253 { 254 struct bio *bio; 255 int ret = 0, rw = WRITE; 256 struct swap_info_struct *sis = page_swap_info(page); 257 258 if (sis->flags & SWP_FILE) { 259 struct kiocb kiocb; 260 struct file *swap_file = sis->swap_file; 261 struct address_space *mapping = swap_file->f_mapping; 262 struct iovec iov = { 263 .iov_base = kmap(page), 264 .iov_len = PAGE_SIZE, 265 }; 266 267 init_sync_kiocb(&kiocb, swap_file); 268 kiocb.ki_pos = page_file_offset(page); 269 kiocb.ki_left = PAGE_SIZE; 270 kiocb.ki_nbytes = PAGE_SIZE; 271 272 set_page_writeback(page); 273 unlock_page(page); 274 ret = mapping->a_ops->direct_IO(KERNEL_WRITE, 275 &kiocb, &iov, 276 kiocb.ki_pos, 1); 277 kunmap(page); 278 if (ret == PAGE_SIZE) { 279 count_vm_event(PSWPOUT); 280 ret = 0; 281 } else { 282 /* 283 * In the case of swap-over-nfs, this can be a 284 * temporary failure if the system has limited 285 * memory for allocating transmit buffers. 286 * Mark the page dirty and avoid 287 * rotate_reclaimable_page but rate-limit the 288 * messages but do not flag PageError like 289 * the normal direct-to-bio case as it could 290 * be temporary. 291 */ 292 set_page_dirty(page); 293 ClearPageReclaim(page); 294 pr_err_ratelimited("Write error on dio swapfile (%Lu)\n", 295 page_file_offset(page)); 296 } 297 end_page_writeback(page); 298 return ret; 299 } 300 301 bio = get_swap_bio(GFP_NOIO, page, end_write_func); 302 if (bio == NULL) { 303 set_page_dirty(page); 304 unlock_page(page); 305 ret = -ENOMEM; 306 goto out; 307 } 308 if (wbc->sync_mode == WB_SYNC_ALL) 309 rw |= REQ_SYNC; 310 count_vm_event(PSWPOUT); 311 set_page_writeback(page); 312 unlock_page(page); 313 submit_bio(rw, bio); 314 out: 315 return ret; 316 } 317 318 int swap_readpage(struct page *page) 319 { 320 struct bio *bio; 321 int ret = 0; 322 struct swap_info_struct *sis = page_swap_info(page); 323 324 VM_BUG_ON(!PageLocked(page)); 325 VM_BUG_ON(PageUptodate(page)); 326 if (frontswap_load(page) == 0) { 327 SetPageUptodate(page); 328 unlock_page(page); 329 goto out; 330 } 331 332 if (sis->flags & SWP_FILE) { 333 struct file *swap_file = sis->swap_file; 334 struct address_space *mapping = swap_file->f_mapping; 335 336 ret = mapping->a_ops->readpage(swap_file, page); 337 if (!ret) 338 count_vm_event(PSWPIN); 339 return ret; 340 } 341 342 bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read); 343 if (bio == NULL) { 344 unlock_page(page); 345 ret = -ENOMEM; 346 goto out; 347 } 348 count_vm_event(PSWPIN); 349 submit_bio(READ, bio); 350 out: 351 return ret; 352 } 353 354 int swap_set_page_dirty(struct page *page) 355 { 356 struct swap_info_struct *sis = page_swap_info(page); 357 358 if (sis->flags & SWP_FILE) { 359 struct address_space *mapping = sis->swap_file->f_mapping; 360 return mapping->a_ops->set_page_dirty(page); 361 } else { 362 return __set_page_dirty_no_writeback(page); 363 } 364 } 365