1 // SPDX-License-Identifier: GPL-2.0-only
44 	i->req = req;  in nfs_page_iter_page_init()
45 	i->count = 0;  in nfs_page_iter_page_init()
50 	const struct nfs_page *req = i->req;  in nfs_page_iter_page_advance()
51 	size_t tmp = i->count + sz;  in nfs_page_iter_page_advance()
53 	i->count = (tmp < req->wb_bytes) ? tmp : req->wb_bytes;  in nfs_page_iter_page_advance()
58 	const struct nfs_page *req = i->req;  in nfs_page_iter_page_get()
61 	if (i->count != req->wb_bytes) {  in nfs_page_iter_page_get()
62 		size_t base = i->count + req->wb_pgbase;  in nfs_page_iter_page_get()
63 		size_t len = PAGE_SIZE - offset_in_page(base);  in nfs_page_iter_page_get()
75 	if (desc->pg_ops->pg_get_mirror)  in nfs_pgio_get_mirror()
76 		return desc->pg_ops->pg_get_mirror(desc, idx);  in nfs_pgio_get_mirror()
77 	return &desc->pg_mirrors[0];  in nfs_pgio_get_mirror()
83 	return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);  in nfs_pgio_current_mirror()
90 	if (desc->pg_ops->pg_set_mirror)  in nfs_pgio_set_current_mirror()
91 		return desc->pg_ops->pg_set_mirror(desc, idx);  in nfs_pgio_set_current_mirror()
92 	return desc->pg_mirror_idx;  in nfs_pgio_set_current_mirror()
99 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pgheader_init()  local
102 	hdr->req = nfs_list_entry(mirror->pg_list.next);  in nfs_pgheader_init()
103 	hdr->inode = desc->pg_inode;  in nfs_pgheader_init()
104 	hdr->cred = nfs_req_openctx(hdr->req)->cred;  in nfs_pgheader_init()
105 	hdr->io_start = req_offset(hdr->req);  in nfs_pgheader_init()
106 	hdr->good_bytes = mirror->pg_count;  in nfs_pgheader_init()
107 	hdr->io_completion = desc->pg_io_completion;  in nfs_pgheader_init()
108 	hdr->dreq = desc->pg_dreq;  in nfs_pgheader_init()
110 	hdr->release = release;  in nfs_pgheader_init()
111 	hdr->completion_ops = desc->pg_completion_ops;  in nfs_pgheader_init()
112 	if (hdr->completion_ops->init_hdr)  in nfs_pgheader_init()
113 		hdr->completion_ops->init_hdr(hdr);  in nfs_pgheader_init()
115 	hdr->pgio_mirror_idx = desc->pg_mirror_idx;  in nfs_pgheader_init()
121 	unsigned int new = pos - hdr->io_start;  in nfs_set_pgio_error()
124 	if (hdr->good_bytes > new) {  in nfs_set_pgio_error()
125 		hdr->good_bytes = new;  in nfs_set_pgio_error()
126 		clear_bit(NFS_IOHDR_EOF, &hdr->flags);  in nfs_set_pgio_error()
127 		if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags))  in nfs_set_pgio_error()
128 			hdr->error = error;  in nfs_set_pgio_error()
137 		INIT_LIST_HEAD(&p->wb_list);  in nfs_page_alloc()
148  * nfs_iocounter_wait - wait for i/o to complete
151  * returns -ERESTARTSYS if interrupted by a fatal signal.
157 	return wait_var_event_killable(&l_ctx->io_count,  in nfs_iocounter_wait()
158 				       !atomic_read(&l_ctx->io_count));  in nfs_iocounter_wait()
162  * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
173 	struct inode *inode = d_inode(l_ctx->open_context->dentry);  in nfs_async_iocounter_wait()
176 	if (atomic_read(&l_ctx->io_count) > 0) {  in nfs_async_iocounter_wait()
177 		rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL);  in nfs_async_iocounter_wait()
181 	if (atomic_read(&l_ctx->io_count) == 0) {  in nfs_async_iocounter_wait()
182 		rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task);  in nfs_async_iocounter_wait()
191  * nfs_page_set_headlock - set the request PG_HEADLOCK
194  * this lock must be held when modifying req->wb_head
201 	if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags))  in nfs_page_set_headlock()
204 	set_bit(PG_CONTENDED1, &req->wb_flags);  in nfs_page_set_headlock()
206 	return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK,  in nfs_page_set_headlock()
211  * nfs_page_clear_headlock - clear the request PG_HEADLOCK
217 	clear_bit_unlock(PG_HEADLOCK, &req->wb_flags);  in nfs_page_clear_headlock()
219 	if (!test_bit(PG_CONTENDED1, &req->wb_flags))  in nfs_page_clear_headlock()
221 	wake_up_bit(&req->wb_flags, PG_HEADLOCK);  in nfs_page_clear_headlock()
225  * nfs_page_group_lock - lock the head of the page group
239 	if (ret || req->wb_head == req)  in nfs_page_group_lock()
241 	return nfs_page_set_headlock(req->wb_head);  in nfs_page_group_lock()
245  * nfs_page_group_unlock - unlock the head of the page group
251 	if (req != req->wb_head)  in nfs_page_group_unlock()
252 		nfs_page_clear_headlock(req->wb_head);  in nfs_page_group_unlock()
257  * nfs_page_group_sync_on_bit_locked - Test if all requests have @bit set
265 	struct nfs_page *head = req->wb_head;  in nfs_page_group_sync_on_bit_locked()
268 	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));  in nfs_page_group_sync_on_bit_locked()
269 	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));  in nfs_page_group_sync_on_bit_locked()
271 	tmp = req->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
273 		if (!test_bit(bit, &tmp->wb_flags))  in nfs_page_group_sync_on_bit_locked()
275 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
281 		clear_bit(bit, &tmp->wb_flags);  in nfs_page_group_sync_on_bit_locked()
282 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
289  * nfs_page_group_sync_on_bit - set bit on current request, but only
291  * @req - request in page group
292  * @bit - PG_* bit that is used to sync page group
306  * nfs_page_group_init - Initialize the page group linkage for @req
307  * @req - a new nfs request
308  * @prev - the previous request in page group, or NULL if @req is the first
319 		req->wb_head = req;  in nfs_page_group_init()
320 		req->wb_this_page = req;  in nfs_page_group_init()
323 		WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);  in nfs_page_group_init()
324 		WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));  in nfs_page_group_init()
325 		req->wb_head = prev->wb_head;  in nfs_page_group_init()
326 		req->wb_this_page = prev->wb_this_page;  in nfs_page_group_init()
327 		prev->wb_this_page = req;  in nfs_page_group_init()
331 		kref_get(&req->wb_head->wb_kref);  in nfs_page_group_init()
336 		if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {  in nfs_page_group_init()
338 			set_bit(PG_INODE_REF, &req->wb_flags);  in nfs_page_group_init()
339 			kref_get(&req->wb_kref);  in nfs_page_group_init()
340 			atomic_long_inc(&NFS_I(inode)->nrequests);  in nfs_page_group_init()
346  * nfs_page_group_destroy - sync the destruction of page groups
347  * @req - request that no longer needs the page group
356 	struct nfs_page *head = req->wb_head;  in nfs_page_group_destroy()
364 		next = tmp->wb_this_page;  in nfs_page_group_destroy()
366 		tmp->wb_this_page = tmp;  in nfs_page_group_destroy()
367 		tmp->wb_head = tmp;  in nfs_page_group_destroy()
382 	struct nfs_open_context *ctx = l_ctx->open_context;  in nfs_page_create()
384 	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))  in nfs_page_create()
385 		return ERR_PTR(-EBADF);  in nfs_page_create()
389 		return ERR_PTR(-ENOMEM);  in nfs_page_create()
391 	req->wb_lock_context = l_ctx;  in nfs_page_create()
392 	refcount_inc(&l_ctx->count);  in nfs_page_create()
393 	atomic_inc(&l_ctx->io_count);  in nfs_page_create()
396 	 * long write-back delay. This will be adjusted in  in nfs_page_create()
398 	req->wb_pgbase = pgbase;  in nfs_page_create()
399 	req->wb_index = index;  in nfs_page_create()
400 	req->wb_offset = offset;  in nfs_page_create()
401 	req->wb_bytes = count;  in nfs_page_create()
402 	kref_init(&req->wb_kref);  in nfs_page_create()
403 	req->wb_nio = 0;  in nfs_page_create()
410 		req->wb_folio = folio;  in nfs_page_assign_folio()
412 		set_bit(PG_FOLIO, &req->wb_flags);  in nfs_page_assign_folio()
419 		req->wb_page = page;  in nfs_page_assign_page()
425  * nfs_page_create_from_page - Create an NFS read/write request.
457  * nfs_page_create_from_folio - Create an NFS read/write request.
477 	ret = nfs_page_create(l_ctx, offset, folio->index, offset, count);  in nfs_page_create_from_folio()
497 	ret = nfs_page_create(req->wb_lock_context, pgbase, req->wb_index,  in nfs_create_subreq()
505 		for (last = req->wb_head;  in nfs_create_subreq()
506 		     last->wb_this_page != req->wb_head;  in nfs_create_subreq()
507 		     last = last->wb_this_page)  in nfs_create_subreq()
512 		ret->wb_nio = req->wb_nio;  in nfs_create_subreq()
518  * nfs_unlock_request - Unlock request and wake up sleepers.
523 	clear_bit_unlock(PG_BUSY, &req->wb_flags);  in nfs_unlock_request()
525 	if (!test_bit(PG_CONTENDED2, &req->wb_flags))  in nfs_unlock_request()
527 	wake_up_bit(&req->wb_flags, PG_BUSY);  in nfs_unlock_request()
531  * nfs_unlock_and_release_request - Unlock request and release the nfs_page
541  * nfs_clear_request - Free up all resources allocated to the request
550 	struct page *page = req->wb_page;  in nfs_clear_request()
551 	struct nfs_lock_context *l_ctx = req->wb_lock_context;  in nfs_clear_request()
556 		req->wb_folio = NULL;  in nfs_clear_request()
557 		clear_bit(PG_FOLIO, &req->wb_flags);  in nfs_clear_request()
560 		req->wb_page = NULL;  in nfs_clear_request()
563 		if (atomic_dec_and_test(&l_ctx->io_count)) {  in nfs_clear_request()
564 			wake_up_var(&l_ctx->io_count);  in nfs_clear_request()
565 			ctx = l_ctx->open_context;  in nfs_clear_request()
566 			if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))  in nfs_clear_request()
567 				rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);  in nfs_clear_request()
570 		req->wb_lock_context = NULL;  in nfs_clear_request()
575  * nfs_free_request - Release the count on an NFS read/write request
582 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_free_request()
585 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_free_request()
586 	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));  in nfs_free_request()
587 	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));  in nfs_free_request()
588 	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));  in nfs_free_request()
589 	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));  in nfs_free_request()
598 	kref_put(&req->wb_kref, nfs_page_group_destroy);  in nfs_release_request()
603  * nfs_generic_pg_test - determine if requests can be coalesced
614 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pg_test()  local
617 	if (mirror->pg_count > mirror->pg_bsize) {  in nfs_generic_pg_test()
627 	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *  in nfs_generic_pg_test()
631 	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);  in nfs_generic_pg_test()
637 	struct nfs_pgio_header *hdr = ops->rw_alloc_header();  in nfs_pgio_header_alloc()
640 		INIT_LIST_HEAD(&hdr->pages);  in nfs_pgio_header_alloc()
641 		hdr->rw_ops = ops;  in nfs_pgio_header_alloc()
648  * nfs_pgio_data_destroy - make @hdr suitable for reuse
657 	if (hdr->args.context)  in nfs_pgio_data_destroy()
658 		put_nfs_open_context(hdr->args.context);  in nfs_pgio_data_destroy()
659 	if (hdr->page_array.pagevec != hdr->page_array.page_array)  in nfs_pgio_data_destroy()
660 		kfree(hdr->page_array.pagevec);  in nfs_pgio_data_destroy()
664  * nfs_pgio_header_free - Free a read or write header
670 	hdr->rw_ops->rw_free_header(hdr);  in nfs_pgio_header_free()
675  * nfs_pgio_rpcsetup - Set up arguments for a pageio call
679  * @how: How to commit data (writes only)
686 	struct nfs_page *req = hdr->req;  in nfs_pgio_rpcsetup()
689 	 * NB: take care not to mess about with hdr->commit et al. */  in nfs_pgio_rpcsetup()
691 	hdr->args.fh     = NFS_FH(hdr->inode);  in nfs_pgio_rpcsetup()
692 	hdr->args.offset = req_offset(req);  in nfs_pgio_rpcsetup()
694 	hdr->mds_offset = hdr->args.offset;  in nfs_pgio_rpcsetup()
695 	hdr->args.pgbase = pgbase;  in nfs_pgio_rpcsetup()
696 	hdr->args.pages  = hdr->page_array.pagevec;  in nfs_pgio_rpcsetup()
697 	hdr->args.count  = count;  in nfs_pgio_rpcsetup()
698 	hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));  in nfs_pgio_rpcsetup()
699 	hdr->args.lock_context = req->wb_lock_context;  in nfs_pgio_rpcsetup()
700 	hdr->args.stable  = NFS_UNSTABLE;  in nfs_pgio_rpcsetup()
709 		hdr->args.stable = NFS_FILE_SYNC;  in nfs_pgio_rpcsetup()
712 	hdr->res.fattr   = &hdr->fattr;  in nfs_pgio_rpcsetup()
713 	hdr->res.count   = 0;  in nfs_pgio_rpcsetup()
714 	hdr->res.eof     = 0;  in nfs_pgio_rpcsetup()
715 	hdr->res.verf    = &hdr->verf;  in nfs_pgio_rpcsetup()
716 	nfs_fattr_init(&hdr->fattr);  in nfs_pgio_rpcsetup()
720  * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
728 	err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);  in nfs_pgio_prepare()
740 		.rpc_argp = &hdr->args,  in nfs_initiate_pgio()
741 		.rpc_resp = &hdr->res,  in nfs_initiate_pgio()
746 		.task = &hdr->task,  in nfs_initiate_pgio()
754 	if (nfs_server_capable(hdr->inode, NFS_CAP_MOVEABLE))  in nfs_initiate_pgio()
757 	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);  in nfs_initiate_pgio()
761 		hdr->inode->i_sb->s_id,  in nfs_initiate_pgio()
762 		(unsigned long long)NFS_FILEID(hdr->inode),  in nfs_initiate_pgio()
763 		hdr->args.count,  in nfs_initiate_pgio()
764 		(unsigned long long)hdr->args.offset);  in nfs_initiate_pgio()
767 		return nfs_local_doio(NFS_SERVER(hdr->inode)->nfs_client,  in nfs_initiate_pgio()
779  * nfs_pgio_error - Clean up from a pageio error
784 	set_bit(NFS_IOHDR_REDO, &hdr->flags);  in nfs_pgio_error()
785 	hdr->completion_ops->completion(hdr);  in nfs_pgio_error()
789  * nfs_pgio_release - Release pageio data
795 	hdr->completion_ops->completion(hdr);  in nfs_pgio_release()
798 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,  in nfs_pageio_mirror_init()  argument
801 	INIT_LIST_HEAD(&mirror->pg_list);  in nfs_pageio_mirror_init()
802 	mirror->pg_bytes_written = 0;  in nfs_pageio_mirror_init()
803 	mirror->pg_count = 0;  in nfs_pageio_mirror_init()
804 	mirror->pg_bsize = bsize;  in nfs_pageio_mirror_init()
805 	mirror->pg_base = 0;  in nfs_pageio_mirror_init()
806 	mirror->pg_recoalesce = 0;  in nfs_pageio_mirror_init()
810  * nfs_pageio_init - initialise a page io descriptor
827 	desc->pg_moreio = 0;  in nfs_pageio_init()
828 	desc->pg_inode = inode;  in nfs_pageio_init()
829 	desc->pg_ops = pg_ops;  in nfs_pageio_init()
830 	desc->pg_completion_ops = compl_ops;  in nfs_pageio_init()
831 	desc->pg_rw_ops = rw_ops;  in nfs_pageio_init()
832 	desc->pg_ioflags = io_flags;  in nfs_pageio_init()
833 	desc->pg_error = 0;  in nfs_pageio_init()
834 	desc->pg_lseg = NULL;  in nfs_pageio_init()
835 	desc->pg_io_completion = NULL;  in nfs_pageio_init()
836 	desc->pg_dreq = NULL;  in nfs_pageio_init()
838 	desc->pg_bsize = bsize;  in nfs_pageio_init()
840 	desc->pg_mirror_count = 1;  in nfs_pageio_init()
841 	desc->pg_mirror_idx = 0;  in nfs_pageio_init()
843 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_init()
844 	desc->pg_mirrors = desc->pg_mirrors_static;  in nfs_pageio_init()
845 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);  in nfs_pageio_init()
846 	desc->pg_maxretrans = 0;  in nfs_pageio_init()
850  * nfs_pgio_result - Basic pageio error handling
857 	struct inode *inode = hdr->inode;  in nfs_pgio_result()
859 	if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)  in nfs_pgio_result()
861 	if (task->tk_status < 0)  in nfs_pgio_result()
862 		nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);  in nfs_pgio_result()
864 		hdr->rw_ops->rw_result(task, hdr);  in nfs_pgio_result()
878 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pgio()  local
883 	struct list_head *head = &mirror->pg_list;  in nfs_generic_pgio()
885 	struct nfs_page_array *pg_array = &hdr->page_array;  in nfs_generic_pgio()
887 	unsigned int pg_base = offset_in_page(mirror->pg_base);  in nfs_generic_pgio()
890 	pagecount = nfs_page_array_len(pg_base, mirror->pg_count);  in nfs_generic_pgio()
891 	pg_array->npages = pagecount;  in nfs_generic_pgio()
893 	if (pagecount <= ARRAY_SIZE(pg_array->page_array))  in nfs_generic_pgio()
894 		pg_array->pagevec = pg_array->page_array;  in nfs_generic_pgio()
896 		pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags);  in nfs_generic_pgio()
897 		if (!pg_array->pagevec) {  in nfs_generic_pgio()
898 			pg_array->npages = 0;  in nfs_generic_pgio()
900 			desc->pg_error = -ENOMEM;  in nfs_generic_pgio()
901 			return desc->pg_error;  in nfs_generic_pgio()
905 	nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);  in nfs_generic_pgio()
906 	pages = hdr->page_array.pagevec;  in nfs_generic_pgio()
913 		req = nfs_list_entry(head->next);  in nfs_generic_pgio()
914 		nfs_list_move_request(req, &hdr->pages);  in nfs_generic_pgio()
916 		if (req->wb_pgbase == 0)  in nfs_generic_pgio()
932 		desc->pg_error = -EINVAL;  in nfs_generic_pgio()
933 		return desc->pg_error;  in nfs_generic_pgio()
936 	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&  in nfs_generic_pgio()
937 	    (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))  in nfs_generic_pgio()
938 		desc->pg_ioflags &= ~FLUSH_COND_STABLE;  in nfs_generic_pgio()
941 	nfs_pgio_rpcsetup(hdr, pg_base, mirror->pg_count, desc->pg_ioflags,  in nfs_generic_pgio()
943 	desc->pg_rpc_callops = &nfs_pgio_common_ops;  in nfs_generic_pgio()
954 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);  in nfs_generic_pg_pgios()
956 		desc->pg_error = -ENOMEM;  in nfs_generic_pg_pgios()
957 		return desc->pg_error;  in nfs_generic_pg_pgios()
962 		struct nfs_client *clp = NFS_SERVER(hdr->inode)->nfs_client;  in nfs_generic_pg_pgios()
965 			nfs_local_open_fh(clp, hdr->cred, hdr->args.fh,  in nfs_generic_pg_pgios()
966 					  &hdr->args.context->nfl,  in nfs_generic_pg_pgios()
967 					  hdr->args.context->mode);  in nfs_generic_pg_pgios()
969 		if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion)  in nfs_generic_pg_pgios()
971 		ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),  in nfs_generic_pg_pgios()
973 					hdr->cred,  in nfs_generic_pg_pgios()
974 					NFS_PROTO(hdr->inode),  in nfs_generic_pg_pgios()
975 					desc->pg_rpc_callops,  in nfs_generic_pg_pgios()
976 					desc->pg_ioflags,  in nfs_generic_pg_pgios()
990 	kfree(desc->pg_mirrors_dynamic);  in nfs_pageio_alloc_mirrors()
991 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_alloc_mirrors()
993 		return desc->pg_mirrors_static;  in nfs_pageio_alloc_mirrors()
997 			nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);  in nfs_pageio_alloc_mirrors()
998 		desc->pg_mirrors_dynamic = ret;  in nfs_pageio_alloc_mirrors()
1004  * nfs_pageio_setup_mirroring - determine if mirroring is to be used
1012 	if (pgio->pg_ops->pg_get_mirror_count)  in nfs_pageio_setup_mirroring()
1013 		mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);  in nfs_pageio_setup_mirroring()
1014 	if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0)  in nfs_pageio_setup_mirroring()
1018 		pgio->pg_error = -EINVAL;  in nfs_pageio_setup_mirroring()
1022 	pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count);  in nfs_pageio_setup_mirroring()
1023 	if (pgio->pg_mirrors == NULL) {  in nfs_pageio_setup_mirroring()
1024 		pgio->pg_error = -ENOMEM;  in nfs_pageio_setup_mirroring()
1025 		pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_setup_mirroring()
1028 	pgio->pg_mirror_count = mirror_count;  in nfs_pageio_setup_mirroring()
1033 	pgio->pg_mirror_count = 1;  in nfs_pageio_cleanup_mirroring()
1034 	pgio->pg_mirror_idx = 0;  in nfs_pageio_cleanup_mirroring()
1035 	pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_cleanup_mirroring()
1036 	kfree(pgio->pg_mirrors_dynamic);  in nfs_pageio_cleanup_mirroring()
1037 	pgio->pg_mirrors_dynamic = NULL;  in nfs_pageio_cleanup_mirroring()
1043 	return l1->lockowner == l2->lockowner;  in nfs_match_lock_context()
1049 	size_t prev_end = prev->wb_pgbase + prev->wb_bytes;  in nfs_page_is_contiguous()
1051 	if (req_offset(req) != req_offset(prev) + prev->wb_bytes)  in nfs_page_is_contiguous()
1053 	if (req->wb_pgbase == 0)  in nfs_page_is_contiguous()
1055 	if (req->wb_pgbase == prev_end) {  in nfs_page_is_contiguous()
1059 		return req->wb_page == prev->wb_page;  in nfs_page_is_contiguous()
1065  * nfs_coalesce_size - test two requests for compatibility
1071  * page data area they describe is contiguous, and that their RPC
1085 		flctx = locks_inode_context(d_inode(nfs_req_openctx(req)->dentry));  in nfs_coalesce_size()
1087 		    !(list_empty_careful(&flctx->flc_posix) &&  in nfs_coalesce_size()
1088 		      list_empty_careful(&flctx->flc_flock)) &&  in nfs_coalesce_size()
1089 		    !nfs_match_lock_context(req->wb_lock_context,  in nfs_coalesce_size()
1090 					    prev->wb_lock_context))  in nfs_coalesce_size()
1095 	return pgio->pg_ops->pg_test(pgio, prev, req);  in nfs_coalesce_size()
1099  * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1110 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_do_add_request()  local
1114 	if (list_empty(&mirror->pg_list)) {  in nfs_pageio_do_add_request()
1115 		if (desc->pg_ops->pg_init)  in nfs_pageio_do_add_request()
1116 			desc->pg_ops->pg_init(desc, req);  in nfs_pageio_do_add_request()
1117 		if (desc->pg_error < 0)  in nfs_pageio_do_add_request()
1119 		mirror->pg_base = req->wb_pgbase;  in nfs_pageio_do_add_request()
1120 		mirror->pg_count = 0;  in nfs_pageio_do_add_request()
1121 		mirror->pg_recoalesce = 0;  in nfs_pageio_do_add_request()
1123 		prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_do_add_request()
1125 	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {  in nfs_pageio_do_add_request()
1126 		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)  in nfs_pageio_do_add_request()
1127 			desc->pg_error = -ETIMEDOUT;  in nfs_pageio_do_add_request()
1129 			desc->pg_error = -EIO;  in nfs_pageio_do_add_request()
1134 	if (size < req->wb_bytes)  in nfs_pageio_do_add_request()
1136 	nfs_list_move_request(req, &mirror->pg_list);  in nfs_pageio_do_add_request()
1137 	mirror->pg_count += req->wb_bytes;  in nfs_pageio_do_add_request()
1138 	return req->wb_bytes;  in nfs_pageio_do_add_request()
1146 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_doio()  local
1148 	if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_doio()
1149 		int error = desc->pg_ops->pg_doio(desc);  in nfs_pageio_doio()
1151 			desc->pg_error = error;  in nfs_pageio_doio()
1152 		if (list_empty(&mirror->pg_list))  in nfs_pageio_doio()
1153 			mirror->pg_bytes_written += mirror->pg_count;  in nfs_pageio_doio()
1164 	desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);  in nfs_pageio_cleanup_request()
1168  * __nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1182 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in __nfs_pageio_add_request()  local
1189 	subreq_size = subreq->wb_bytes;  in __nfs_pageio_add_request()
1196 			req->wb_pgbase += size;  in __nfs_pageio_add_request()
1197 			req->wb_bytes -= size;  in __nfs_pageio_add_request()
1198 			req->wb_offset += size;  in __nfs_pageio_add_request()
1199 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1207 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1213 			desc->pg_moreio = 1;  in __nfs_pageio_add_request()
1215 			if (desc->pg_error < 0 || mirror->pg_recoalesce)  in __nfs_pageio_add_request()
1221 		subreq = nfs_create_subreq(req, req->wb_pgbase,  in __nfs_pageio_add_request()
1222 				req->wb_offset, size);  in __nfs_pageio_add_request()
1231 	desc->pg_error = PTR_ERR(subreq);  in __nfs_pageio_add_request()
1238 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_do_recoalesce()  local
1242 		list_splice_init(&mirror->pg_list, &head);  in nfs_do_recoalesce()
1243 		mirror->pg_recoalesce = 0;  in nfs_do_recoalesce()
1251 			if (desc->pg_error < 0) {  in nfs_do_recoalesce()
1252 				list_splice_tail(&head, &mirror->pg_list);  in nfs_do_recoalesce()
1253 				mirror->pg_recoalesce = 1;  in nfs_do_recoalesce()
1258 	} while (mirror->pg_recoalesce);  in nfs_do_recoalesce()
1271 		if (desc->pg_error < 0)  in nfs_pageio_add_request_mirror()
1282 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_error_cleanup()  local
1284 	if (!desc->pg_error)  in nfs_pageio_error_cleanup()
1287 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_error_cleanup()
1288 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_error_cleanup()
1289 		desc->pg_completion_ops->error_cleanup(&mirror->pg_list,  in nfs_pageio_error_cleanup()
1290 				desc->pg_error);  in nfs_pageio_error_cleanup()
1301 	pgbase = req->wb_pgbase;  in nfs_pageio_add_request()
1302 	offset = req->wb_offset;  in nfs_pageio_add_request()
1303 	bytes = req->wb_bytes;  in nfs_pageio_add_request()
1306 	if (desc->pg_error < 0)  in nfs_pageio_add_request()
1309 	/* Create the mirror instances first, and fire them off */  in nfs_pageio_add_request()
1310 	for (midx = 1; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_add_request()
1318 			desc->pg_error = PTR_ERR(dupreq);  in nfs_pageio_add_request()
1341  * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1344  * @mirror_idx: pointer to mirror index
1349 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_complete_mirror()  local
1353 	mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_complete_mirror()
1357 		if (desc->pg_error < 0 || !mirror->pg_recoalesce)  in nfs_pageio_complete_mirror()
1366  * nfs_pageio_resend - Transfer requests to new descriptor and resend
1367  * @hdr - the pgio header to move request from
1368  * @desc - the pageio descriptor to add requests to
1380 	desc->pg_io_completion = hdr->io_completion;  in nfs_pageio_resend()
1381 	desc->pg_dreq = hdr->dreq;  in nfs_pageio_resend()
1383 	list_splice_init(&hdr->pages, &pages);  in nfs_pageio_resend()
1392 		int err = desc->pg_error < 0 ? desc->pg_error : -EIO;  in nfs_pageio_resend()
1393 		hdr->completion_ops->error_cleanup(&pages, err);  in nfs_pageio_resend()
1394 		nfs_set_pgio_error(hdr, err, hdr->io_start);  in nfs_pageio_resend()
1402  * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1409 	for (midx = 0; midx < desc->pg_mirror_count; midx++)  in nfs_pageio_complete()
1412 	if (desc->pg_error < 0)  in nfs_pageio_complete()
1414 	if (desc->pg_ops->pg_cleanup)  in nfs_pageio_complete()
1415 		desc->pg_ops->pg_cleanup(desc);  in nfs_pageio_complete()
1420  * nfs_pageio_cond_complete - Conditional I/O completion
1425  * on non-contiguous ranges of pages as that might deadlock. This
1432 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_cond_complete()  local
1437 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_cond_complete()
1438 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_cond_complete()
1439 		if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_cond_complete()
1440 			prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_cond_complete()
1445 			} else if (index == prev->wb_index + 1)  in nfs_pageio_cond_complete()
1451 			desc->pg_moreio = 1;  in nfs_pageio_cond_complete()
1459  * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1473 		return -ENOMEM;  in nfs_init_nfspagecache()