346 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_write_complete()  local
347 	struct page *page = sio->bvec[0].bv_page;  in sio_write_complete()
350 	if (ret != sio->len) {  in sio_write_complete()
361 		for (p = 0; p < sio->pages; p++) {  in sio_write_complete()
362 			page = sio->bvec[p].bv_page;  in sio_write_complete()
368 	for (p = 0; p < sio->pages; p++)  in sio_write_complete()
369 		end_page_writeback(sio->bvec[p].bv_page);  in sio_write_complete()
371 	mempool_free(sio, sio_pool);  in sio_write_complete()
376 	struct swap_iocb *sio = NULL;  in swap_writepage_fs()  local
385 		sio = *wbc->swap_plug;  in swap_writepage_fs()
386 	if (sio) {  in swap_writepage_fs()
387 		if (sio->iocb.ki_filp != swap_file ||  in swap_writepage_fs()
388 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_writepage_fs()
389 			swap_write_unplug(sio);  in swap_writepage_fs()
390 			sio = NULL;  in swap_writepage_fs()
393 	if (!sio) {  in swap_writepage_fs()
394 		sio = mempool_alloc(sio_pool, GFP_NOIO);  in swap_writepage_fs()
395 		init_sync_kiocb(&sio->iocb, swap_file);  in swap_writepage_fs()
396 		sio->iocb.ki_complete = sio_write_complete;  in swap_writepage_fs()
397 		sio->iocb.ki_pos = pos;  in swap_writepage_fs()
398 		sio->pages = 0;  in swap_writepage_fs()
399 		sio->len = 0;  in swap_writepage_fs()
401 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_writepage_fs()
402 	sio->len += folio_size(folio);  in swap_writepage_fs()
403 	sio->pages += 1;  in swap_writepage_fs()
404 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) {  in swap_writepage_fs()
405 		swap_write_unplug(sio);  in swap_writepage_fs()
406 		sio = NULL;  in swap_writepage_fs()
409 		*wbc->swap_plug = sio;  in swap_writepage_fs()
475 void swap_write_unplug(struct swap_iocb *sio)  in swap_write_unplug()  argument
478 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in swap_write_unplug()
481 	iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);  in swap_write_unplug()
482 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in swap_write_unplug()
484 		sio_write_complete(&sio->iocb, ret);  in swap_write_unplug()
489 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_read_complete()  local
492 	if (ret == sio->len) {  in sio_read_complete()
493 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
494 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
499 		count_vm_events(PSWPIN, sio->pages);  in sio_read_complete()
501 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
502 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
508 	mempool_free(sio, sio_pool);  in sio_read_complete()
544 	struct swap_iocb *sio = NULL;  in swap_read_folio_fs()  local
548 		sio = *plug;  in swap_read_folio_fs()
549 	if (sio) {  in swap_read_folio_fs()
550 		if (sio->iocb.ki_filp != sis->swap_file ||  in swap_read_folio_fs()
551 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_read_folio_fs()
552 			swap_read_unplug(sio);  in swap_read_folio_fs()
553 			sio = NULL;  in swap_read_folio_fs()
556 	if (!sio) {  in swap_read_folio_fs()
557 		sio = mempool_alloc(sio_pool, GFP_KERNEL);  in swap_read_folio_fs()
558 		init_sync_kiocb(&sio->iocb, sis->swap_file);  in swap_read_folio_fs()
559 		sio->iocb.ki_pos = pos;  in swap_read_folio_fs()
560 		sio->iocb.ki_complete = sio_read_complete;  in swap_read_folio_fs()
561 		sio->pages = 0;  in swap_read_folio_fs()
562 		sio->len = 0;  in swap_read_folio_fs()
564 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_read_folio_fs()
565 	sio->len += folio_size(folio);  in swap_read_folio_fs()
566 	sio->pages += 1;  in swap_read_folio_fs()
567 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {  in swap_read_folio_fs()
568 		swap_read_unplug(sio);  in swap_read_folio_fs()
569 		sio = NULL;  in swap_read_folio_fs()
572 		*plug = sio;  in swap_read_folio_fs()
658 void __swap_read_unplug(struct swap_iocb *sio)  in __swap_read_unplug()  argument
661 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in __swap_read_unplug()
664 	iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);  in __swap_read_unplug()
665 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in __swap_read_unplug()
667 		sio_read_complete(&sio->iocb, ret);  in __swap_read_unplug()