1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/mm/page_io.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 *
7 * Swap reorganised 29.12.95,
8 * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 * Removed race in async swapping. 14.4.1996. Bruno Haible
10 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
12 */
13
14 #include <linux/mm.h>
15 #include <linux/kernel_stat.h>
16 #include <linux/gfp.h>
17 #include <linux/pagemap.h>
18 #include <linux/swap.h>
19 #include <linux/bio.h>
20 #include <linux/swapops.h>
21 #include <linux/writeback.h>
22 #include <linux/blkdev.h>
23 #include <linux/psi.h>
24 #include <linux/uio.h>
25 #include <linux/sched/task.h>
26 #include <linux/delayacct.h>
27 #include <linux/zswap.h>
28 #include <linux/swap_ops.h>
29 #include "swap.h"
30 #include "swap_table.h"
31
generic_swapfile_activate(struct swap_info_struct * sis,struct file * swap_file,sector_t * span)32 int generic_swapfile_activate(struct swap_info_struct *sis,
33 struct file *swap_file,
34 sector_t *span)
35 {
36 struct address_space *mapping = swap_file->f_mapping;
37 struct inode *inode = mapping->host;
38 unsigned blocks_per_page;
39 unsigned long page_no;
40 unsigned blkbits;
41 sector_t probe_block;
42 sector_t last_block;
43 sector_t lowest_block = -1;
44 sector_t highest_block = 0;
45 int nr_extents = 0;
46 int ret;
47
48 blkbits = inode->i_blkbits;
49 blocks_per_page = PAGE_SIZE >> blkbits;
50
51 /*
52 * Map all the blocks into the extent tree. This code doesn't try
53 * to be very smart.
54 */
55 probe_block = 0;
56 page_no = 0;
57 last_block = i_size_read(inode) >> blkbits;
58 while ((probe_block + blocks_per_page) <= last_block &&
59 page_no < sis->max) {
60 unsigned block_in_page;
61 sector_t first_block;
62
63 cond_resched();
64
65 first_block = probe_block;
66 ret = bmap(inode, &first_block);
67 if (ret || !first_block)
68 goto bad_bmap;
69
70 /*
71 * It must be PAGE_SIZE aligned on-disk
72 */
73 if (first_block & (blocks_per_page - 1)) {
74 probe_block++;
75 goto reprobe;
76 }
77
78 for (block_in_page = 1; block_in_page < blocks_per_page;
79 block_in_page++) {
80 sector_t block;
81
82 block = probe_block + block_in_page;
83 ret = bmap(inode, &block);
84 if (ret || !block)
85 goto bad_bmap;
86
87 if (block != first_block + block_in_page) {
88 /* Discontiguity */
89 probe_block++;
90 goto reprobe;
91 }
92 }
93
94 first_block >>= (PAGE_SHIFT - blkbits);
95 if (page_no) { /* exclude the header page */
96 if (first_block < lowest_block)
97 lowest_block = first_block;
98 if (first_block > highest_block)
99 highest_block = first_block;
100 }
101
102 /*
103 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
104 */
105 ret = add_swap_extent(sis, page_no, 1, first_block);
106 if (ret < 0)
107 goto out;
108 nr_extents += ret;
109 page_no++;
110 probe_block += blocks_per_page;
111 reprobe:
112 continue;
113 }
114 ret = nr_extents;
115 *span = 1 + highest_block - lowest_block;
116 if (page_no == 0)
117 page_no = 1; /* force Empty message */
118 sis->max = page_no;
119 sis->pages = page_no - 1;
120 out:
121 return ret;
122 bad_bmap:
123 pr_err("swapon: swapfile has holes\n");
124 ret = -EINVAL;
125 goto out;
126 }
127
is_folio_zero_filled(struct folio * folio)128 static bool is_folio_zero_filled(struct folio *folio)
129 {
130 unsigned int pos, last_pos;
131 unsigned long *data;
132 unsigned int i;
133
134 last_pos = PAGE_SIZE / sizeof(*data) - 1;
135 for (i = 0; i < folio_nr_pages(folio); i++) {
136 data = kmap_local_folio(folio, i * PAGE_SIZE);
137 /*
138 * Check last word first, incase the page is zero-filled at
139 * the start and has non-zero data at the end, which is common
140 * in real-world workloads.
141 */
142 if (data[last_pos]) {
143 kunmap_local(data);
144 return false;
145 }
146 for (pos = 0; pos < last_pos; pos++) {
147 if (data[pos]) {
148 kunmap_local(data);
149 return false;
150 }
151 }
152 kunmap_local(data);
153 }
154
155 return true;
156 }
157
swap_zeromap_folio_set(struct folio * folio)158 static void swap_zeromap_folio_set(struct folio *folio)
159 {
160 struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
161 int nr_pages = folio_nr_pages(folio);
162 struct swap_cluster_info *ci;
163 swp_entry_t entry;
164 unsigned int i;
165
166 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
167 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
168
169 ci = swap_cluster_get_and_lock(folio);
170 for (i = 0; i < folio_nr_pages(folio); i++) {
171 entry = page_swap_entry(folio_page(folio, i));
172 __swap_table_set_zero(ci, swp_cluster_offset(entry));
173 }
174 swap_cluster_unlock(ci);
175
176 count_vm_events(SWPOUT_ZERO, nr_pages);
177 if (objcg) {
178 count_objcg_events(objcg, SWPOUT_ZERO, nr_pages);
179 obj_cgroup_put(objcg);
180 }
181 }
182
swap_zeromap_folio_clear(struct folio * folio)183 static void swap_zeromap_folio_clear(struct folio *folio)
184 {
185 struct swap_cluster_info *ci;
186 swp_entry_t entry;
187 unsigned int i;
188
189 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
190 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
191
192 ci = swap_cluster_get_and_lock(folio);
193 for (i = 0; i < folio_nr_pages(folio); i++) {
194 entry = page_swap_entry(folio_page(folio, i));
195 __swap_table_clear_zero(ci, swp_cluster_offset(entry));
196 }
197 swap_cluster_unlock(ci);
198 }
199
200 /*
201 * We may have stale swap cache pages in memory: notice
202 * them here and get rid of the unnecessary final write.
203 */
swap_writeout(struct swap_io_ctx * ctx,struct folio * folio)204 int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
205 {
206 int ret = 0;
207
208 if (folio_free_swap(folio))
209 goto out_unlock;
210
211 /*
212 * Arch code may have to preserve more data than just the page
213 * contents, e.g. memory tags.
214 */
215 ret = arch_prepare_to_swap(folio);
216 if (ret) {
217 folio_mark_dirty(folio);
218 goto out_unlock;
219 }
220
221 /*
222 * Use the swap table zero mark to avoid doing IO for zero-filled
223 * pages. The zero mark is protected by the cluster lock, which is
224 * acquired internally by swap_zeromap_folio_set/clear.
225 */
226 if (is_folio_zero_filled(folio)) {
227 swap_zeromap_folio_set(folio);
228 goto out_unlock;
229 }
230
231 /*
232 * Clear bits this folio occupies in the zeromap to prevent zero data
233 * being read in from any previous zero writes that occupied the same
234 * swap entries.
235 */
236 swap_zeromap_folio_clear(folio);
237
238 if (zswap_store(folio)) {
239 count_mthp_stat(folio_order(folio), MTHP_STAT_ZSWPOUT);
240 goto out_unlock;
241 }
242
243 rcu_read_lock();
244 if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) {
245 rcu_read_unlock();
246 folio_mark_dirty(folio);
247 return AOP_WRITEPAGE_ACTIVATE;
248 }
249 rcu_read_unlock();
250
251 __swap_writepage(ctx, folio);
252 return 0;
253 out_unlock:
254 folio_unlock(folio);
255 return ret;
256 }
257
258 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
folio_memcg_blkg_css(struct folio * folio)259 static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio)
260 {
261 return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys);
262 }
263
folio_blkg_can_merge(struct folio * folio,struct folio * prev_folio)264 static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
265 {
266 bool can_merge = true;
267
268 if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio))
269 return false;
270 if (folio_memcg_charged(folio)) {
271 rcu_read_lock();
272 if (folio_memcg_blkg_css(folio) !=
273 folio_memcg_blkg_css(prev_folio))
274 can_merge = false;
275 rcu_read_unlock();
276 }
277 return can_merge;
278 }
279
bio_associate_blkg_from_page(struct bio * bio,struct folio * folio)280 static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
281 {
282 struct cgroup_subsys_state *css;
283
284 if (!folio_memcg_charged(folio))
285 return;
286 rcu_read_lock();
287 css = folio_memcg_blkg_css(folio);
288 if (css && !css_tryget(css))
289 css = NULL;
290 rcu_read_unlock();
291
292 bio_associate_blkg_from_css(bio, css);
293 if (css)
294 css_put(css);
295 }
296 #else
folio_blkg_can_merge(struct folio * folio,struct folio * prev_folio)297 static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
298 {
299 return true;
300 }
301 #define bio_associate_blkg_from_page(bio, folio) do { } while (0)
302 #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
303
304 static mempool_t *sio_pool;
305
sio_pool_init(void)306 int sio_pool_init(void)
307 {
308 if (!sio_pool) {
309 mempool_t *pool = mempool_create_kmalloc_pool(
310 SWAP_CLUSTER_MAX, sizeof(struct swap_iocb));
311 if (cmpxchg(&sio_pool, NULL, pool))
312 mempool_destroy(pool);
313 }
314 if (!sio_pool)
315 return -ENOMEM;
316 return 0;
317 }
318
swap_can_merge(struct swap_io_ctx * ctx,struct folio * folio,int rw)319 static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio,
320 int rw)
321 {
322 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
323 struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1];
324 struct folio *prev_folio = bvec_folio(last_bv);
325 size_t prev_folio_size = folio_size(prev_folio);
326
327 if (ctx->sis != sis)
328 return false;
329 return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw);
330 }
331
swap_add_folio(struct swap_io_ctx * ctx,struct folio * folio,int rw)332 static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw)
333 {
334 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
335 struct swap_iocb *sio = ctx->sio;
336
337 if (sio && !swap_can_merge(ctx, folio, rw)) {
338 if (rw == WRITE)
339 swap_write_submit(ctx);
340 else
341 swap_read_submit(ctx);
342 sio = ctx->sio;
343 }
344
345 if (!sio) {
346 ctx->sis = sis;
347 ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO);
348 sio->nr_bvecs = 0;
349 sio->len = 0;
350 }
351 bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
352 sio->len += folio_size(folio);
353
354 /*
355 * Write out the iocb if we filled it, or if the device is synchronous.
356 *
357 * The latter is to work around expectations in the classic LRU code
358 * which make synchronous clearing of the folio writeback flag in the
359 * reclaim path beneficial.
360 */
361 if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) ||
362 (rw == WRITE && (sis->flags & SWP_SYNCHRONOUS_IO))) {
363 if (rw == WRITE)
364 swap_write_submit(ctx);
365 else
366 swap_read_submit(ctx);
367 }
368 }
369
__swap_writepage(struct swap_io_ctx * ctx,struct folio * folio)370 void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
371 {
372 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
373
374 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
375 if (unlikely(folio_test_pmd_mappable(folio))) {
376 count_memcg_folio_events(folio, THP_SWPOUT, 1);
377 count_vm_event(THP_SWPOUT);
378 }
379 #endif
380 count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
381 count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
382 count_vm_events(PSWPOUT, folio_nr_pages(folio));
383
384 folio_start_writeback(folio);
385 folio_unlock(folio);
386 swap_add_folio(ctx, folio, WRITE);
387 }
388
389 /*
390 * Return the count of contiguous swap entries that share the same
391 * zeromap status as the starting entry. If is_zerop is not NULL,
392 * it will return the zeromap status of the starting entry.
393 *
394 * Context: Caller must ensure the cluster containing the entries
395 * that will be checked won't be freed.
396 */
swap_zeromap_batch(swp_entry_t entry,int max_nr,bool * is_zerop)397 static int swap_zeromap_batch(swp_entry_t entry, int max_nr,
398 bool *is_zerop)
399 {
400 int i;
401 bool is_zero;
402 unsigned int ci_start = swp_cluster_offset(entry);
403 struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
404
405 VM_WARN_ON_ONCE(ci_start + max_nr > SWAPFILE_CLUSTER);
406
407 rcu_read_lock();
408 is_zero = __swap_table_test_zero(ci, ci_start);
409 for (i = 1; i < max_nr; i++)
410 if (is_zero != __swap_table_test_zero(ci, ci_start + i))
411 break;
412 rcu_read_unlock();
413 if (is_zerop)
414 *is_zerop = is_zero;
415
416 return i;
417 }
418
swap_read_folio_zeromap(struct folio * folio)419 static bool swap_read_folio_zeromap(struct folio *folio)
420 {
421 int nr_pages = folio_nr_pages(folio);
422 struct obj_cgroup *objcg;
423 bool is_zeromap;
424
425 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
426
427 /*
428 * Swapping in a large folio that is partially in the zeromap is not
429 * currently handled. Return true without marking the folio uptodate so
430 * that an IO error is emitted (e.g. do_swap_page() will sigbus).
431 * Folio lock stabilizes the cluster and map, so the check is safe.
432 */
433 if (WARN_ON_ONCE(swap_zeromap_batch(folio->swap, nr_pages,
434 &is_zeromap) != nr_pages))
435 return true;
436
437 if (!is_zeromap)
438 return false;
439
440 objcg = get_obj_cgroup_from_folio(folio);
441 count_vm_events(SWPIN_ZERO, nr_pages);
442 if (objcg) {
443 count_objcg_events(objcg, SWPIN_ZERO, nr_pages);
444 obj_cgroup_put(objcg);
445 }
446
447 folio_zero_range(folio, 0, folio_size(folio));
448 folio_mark_uptodate(folio);
449 return true;
450 }
451
swap_read_folio(struct swap_io_ctx * ctx,struct folio * folio)452 void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
453 {
454 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
455 bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
456 bool workingset = folio_test_workingset(folio);
457 unsigned long pflags;
458 bool in_thrashing;
459
460 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio) && !synchronous, folio);
461 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
462 VM_BUG_ON_FOLIO(folio_test_uptodate(folio), folio);
463
464 /*
465 * Count submission time as memory stall and delay. When the device
466 * is congested, or the submitting cgroup IO-throttled, submission
467 * can be a significant part of overall IO time.
468 */
469 if (workingset) {
470 delayacct_thrashing_start(&in_thrashing);
471 psi_memstall_enter(&pflags);
472 }
473 delayacct_swapin_start();
474
475 if (swap_read_folio_zeromap(folio)) {
476 folio_unlock(folio);
477 goto finish;
478 }
479
480 if (zswap_load(folio) != -ENOENT)
481 goto finish;
482
483 /* We have to read from slower devices. Increase zswap protection. */
484 zswap_folio_swapin(folio);
485 swap_add_folio(ctx, folio, READ);
486
487 finish:
488 if (workingset) {
489 delayacct_thrashing_end(&in_thrashing);
490 psi_memstall_leave(&pflags);
491 }
492 delayacct_swapin_end();
493 }
494
swap_write_end(struct swap_iocb * sio,bool failed)495 static void swap_write_end(struct swap_iocb *sio, bool failed)
496 {
497 int p;
498
499 for (p = 0; p < sio->nr_bvecs; p++) {
500 struct page *page = sio->bvecs[p].bv_page;
501
502 if (failed) {
503 set_page_dirty(page);
504 ClearPageReclaim(page);
505 }
506 end_page_writeback(page);
507 }
508 mempool_free(sio, sio_pool);
509 }
510
swap_fs_write_complete(struct kiocb * iocb,long ret)511 static void swap_fs_write_complete(struct kiocb *iocb, long ret)
512 {
513 struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
514 bool failed = ret != sio->len;
515
516 if (failed) {
517 struct page *page = sio->bvecs[0].bv_page;
518
519 /*
520 * In the case of swap-over-nfs, this can be a temporary failure
521 * if the system has limited memory for allocating transmit
522 * buffers. Mark the page dirty and avoid
523 * folio_rotate_reclaimable but rate-limit the messages.
524 */
525 pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
526 ret, swap_dev_pos(page_swap_entry(page)));
527 }
528
529 swap_write_end(sio, failed);
530 }
531
end_swap_bio_write(struct bio * bio)532 static void end_swap_bio_write(struct bio *bio)
533 {
534 struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
535 bool failed = !!bio->bi_status;
536
537 if (failed)
538 pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
539 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
540 (unsigned long long)bio->bi_iter.bi_sector);
541 bio_uninit(bio);
542 swap_write_end(sio, failed);
543 }
544
swap_read_end(struct swap_iocb * sio,bool failed)545 static void swap_read_end(struct swap_iocb *sio, bool failed)
546 {
547 int p;
548
549 for (p = 0; p < sio->nr_bvecs; p++) {
550 struct folio *folio = bvec_folio(&sio->bvecs[p]);
551
552 if (!failed) {
553 count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
554 count_memcg_folio_events(folio, PSWPIN,
555 folio_nr_pages(folio));
556 folio_mark_uptodate(folio);
557 }
558 folio_unlock(folio);
559 }
560
561 if (!failed)
562 count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
563
564 mempool_free(sio, sio_pool);
565 }
566
swap_fs_read_complete(struct kiocb * iocb,long ret)567 static void swap_fs_read_complete(struct kiocb *iocb, long ret)
568 {
569 struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
570 bool failed = ret != sio->len;
571
572 if (failed)
573 pr_alert_ratelimited("Read-error on swap-device\n");
574 swap_read_end(sio, failed);
575 }
576
swap_bio_read_end_io(struct bio * bio)577 static void swap_bio_read_end_io(struct bio *bio)
578 {
579 struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
580 bool failed = !!bio->bi_status;
581
582 if (failed)
583 pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
584 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
585 (unsigned long long)bio->bi_iter.bi_sector);
586 bio_uninit(bio);
587 swap_read_end(sio, failed);
588 }
589
swap_bdev_submit_write(struct swap_io_ctx * ctx)590 static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
591 {
592 struct swap_iocb *sio = ctx->sio;
593 struct bio *bio = &sio->bio;
594
595 bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
596 REQ_OP_WRITE | REQ_SWAP);
597 bio->bi_iter.bi_size = sio->len;
598 bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
599 bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
600
601 if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
602 submit_bio_wait(bio);
603 end_swap_bio_write(bio);
604 } else {
605 bio->bi_end_io = end_swap_bio_write;
606 submit_bio(bio);
607 }
608 }
609
swap_bdev_submit_read(struct swap_io_ctx * ctx)610 static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
611 {
612 struct swap_iocb *sio = ctx->sio;
613 struct bio *bio = &sio->bio;
614
615 bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
616 REQ_OP_READ);
617 bio->bi_iter.bi_size = sio->len;
618 bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
619
620 if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
621 /*
622 * Keep this task valid during swap readpage because the oom
623 * killer may attempt to access it in the page fault retry
624 * time check.
625 */
626 get_task_struct(current);
627 submit_bio_wait(bio);
628 swap_bio_read_end_io(bio);
629 put_task_struct(current);
630 } else {
631 bio->bi_end_io = swap_bio_read_end_io;
632 submit_bio(bio);
633 }
634 }
635
swap_bdev_can_merge(struct folio * folio,struct folio * prev_folio,size_t prev_folio_size,int rw)636 static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio,
637 size_t prev_folio_size, int rw)
638 {
639 if (swap_folio_sector(folio) !=
640 swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT))
641 return false;
642 if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio))
643 return false;
644 return true;
645 }
646
647 const struct swap_ops swap_bdev_ops = {
648 .submit_write = swap_bdev_submit_write,
649 .submit_read = swap_bdev_submit_read,
650 .can_merge = swap_bdev_can_merge,
651 };
652
swap_fs_prepare_rw(struct swap_io_ctx * ctx,int rw,struct iov_iter * iter)653 void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter)
654 {
655 struct swap_iocb *sio = ctx->sio;
656
657 init_sync_kiocb(&sio->iocb, ctx->sis->swap_file);
658 sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap);
659 if (rw == WRITE)
660 sio->iocb.ki_complete = swap_fs_write_complete;
661 else
662 sio->iocb.ki_complete = swap_fs_read_complete;
663
664 iov_iter_bvec(iter, rw == WRITE ? ITER_SOURCE : ITER_DEST,
665 sio->bvecs, sio->nr_bvecs, sio->len);
666 }
667 EXPORT_SYMBOL_GPL(swap_fs_prepare_rw);
668
swap_fs_can_merge(struct folio * folio,struct folio * prev_folio,size_t prev_folio_size,int rw)669 bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
670 size_t prev_folio_size, int rw)
671 {
672 return swap_dev_pos(folio->swap) ==
673 swap_dev_pos(prev_folio->swap) + prev_folio_size;
674 }
675 EXPORT_SYMBOL_GPL(swap_fs_can_merge);
676
swap_fs_activate(struct swap_info_struct * sis,const struct swap_ops * ops)677 int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops)
678 {
679 sis->ops = ops;
680 return add_swap_extent(sis, 0, sis->max, 0);
681 }
682 EXPORT_SYMBOL_GPL(swap_fs_activate);
683
swap_write_submit(struct swap_io_ctx * ctx)684 void swap_write_submit(struct swap_io_ctx *ctx)
685 {
686 if (!ctx->sio)
687 return;
688 count_vm_events(NRSWPOUT, 1);
689 ctx->sis->ops->submit_write(ctx);
690 ctx->sio = NULL;
691 ctx->sis = NULL;
692 }
693
swap_read_submit(struct swap_io_ctx * ctx)694 void swap_read_submit(struct swap_io_ctx *ctx)
695 {
696 if (!ctx->sio)
697 return;
698 count_vm_events(NRSWPIN, 1);
699 ctx->sis->ops->submit_read(ctx);
700 ctx->sio = NULL;
701 ctx->sis = NULL;
702 }
703