xref: /linux/fs/nfs/write.c (revision 6d17d653c9f152e113043d00f3bcf00c0eb5f5a2)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * linux/fs/nfs/write.c
31da177e4SLinus Torvalds  *
47c85d900STrond Myklebust  * Write file data over NFS.
51da177e4SLinus Torvalds  *
61da177e4SLinus Torvalds  * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
71da177e4SLinus Torvalds  */
81da177e4SLinus Torvalds 
91da177e4SLinus Torvalds #include <linux/types.h>
101da177e4SLinus Torvalds #include <linux/slab.h>
111da177e4SLinus Torvalds #include <linux/mm.h>
121da177e4SLinus Torvalds #include <linux/pagemap.h>
131da177e4SLinus Torvalds #include <linux/file.h>
141da177e4SLinus Torvalds #include <linux/writeback.h>
1589a09141SPeter Zijlstra #include <linux/swap.h>
16074cc1deSTrond Myklebust #include <linux/migrate.h>
171da177e4SLinus Torvalds 
181da177e4SLinus Torvalds #include <linux/sunrpc/clnt.h>
191da177e4SLinus Torvalds #include <linux/nfs_fs.h>
201da177e4SLinus Torvalds #include <linux/nfs_mount.h>
211da177e4SLinus Torvalds #include <linux/nfs_page.h>
223fcfab16SAndrew Morton #include <linux/backing-dev.h>
23afeacc8cSPaul Gortmaker #include <linux/export.h>
24af7cf057STrond Myklebust #include <linux/freezer.h>
25af7cf057STrond Myklebust #include <linux/wait.h>
263fcfab16SAndrew Morton 
277c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
281da177e4SLinus Torvalds 
291da177e4SLinus Torvalds #include "delegation.h"
3049a70f27STrond Myklebust #include "internal.h"
3191d5b470SChuck Lever #include "iostat.h"
32def6ed7eSAndy Adamson #include "nfs4_fs.h"
33074cc1deSTrond Myklebust #include "fscache.h"
3494ad1c80SFred Isaman #include "pnfs.h"
351da177e4SLinus Torvalds 
36f4ce1299STrond Myklebust #include "nfstrace.h"
37f4ce1299STrond Myklebust 
381da177e4SLinus Torvalds #define NFSDBG_FACILITY		NFSDBG_PAGECACHE
391da177e4SLinus Torvalds 
401da177e4SLinus Torvalds #define MIN_POOL_WRITE		(32)
411da177e4SLinus Torvalds #define MIN_POOL_COMMIT		(4)
421da177e4SLinus Torvalds 
43919e3bd9STrond Myklebust struct nfs_io_completion {
44919e3bd9STrond Myklebust 	void (*complete)(void *data);
45919e3bd9STrond Myklebust 	void *data;
46919e3bd9STrond Myklebust 	struct kref refcount;
47919e3bd9STrond Myklebust };
48919e3bd9STrond Myklebust 
491da177e4SLinus Torvalds /*
501da177e4SLinus Torvalds  * Local function declarations
511da177e4SLinus Torvalds  */
52f8512ad0SFred Isaman static void nfs_redirty_request(struct nfs_page *req);
53788e7a89STrond Myklebust static const struct rpc_call_ops nfs_commit_ops;
54061ae2edSFred Isaman static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
55f453a54aSFred Isaman static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
564a0de55cSAnna Schumaker static const struct nfs_rw_ops nfs_rw_write_ops;
57d4581383SWeston Andros Adamson static void nfs_clear_request_commit(struct nfs_page *req);
5802d1426cSWeston Andros Adamson static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
5902d1426cSWeston Andros Adamson 				      struct inode *inode);
603a3908c8STrond Myklebust static struct nfs_page *
613a3908c8STrond Myklebust nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
623a3908c8STrond Myklebust 						struct page *page);
631da177e4SLinus Torvalds 
64e18b890bSChristoph Lameter static struct kmem_cache *nfs_wdata_cachep;
653feb2d49STrond Myklebust static mempool_t *nfs_wdata_mempool;
660b7c0153SFred Isaman static struct kmem_cache *nfs_cdata_cachep;
671da177e4SLinus Torvalds static mempool_t *nfs_commit_mempool;
681da177e4SLinus Torvalds 
69518662e0SNeilBrown struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
701da177e4SLinus Torvalds {
71518662e0SNeilBrown 	struct nfs_commit_data *p;
7240859d7eSChuck Lever 
73518662e0SNeilBrown 	if (never_fail)
74518662e0SNeilBrown 		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
75518662e0SNeilBrown 	else {
76518662e0SNeilBrown 		/* It is OK to do some reclaim, not no safe to wait
77518662e0SNeilBrown 		 * for anything to be returned to the pool.
78518662e0SNeilBrown 		 * mempool_alloc() cannot handle that particular combination,
79518662e0SNeilBrown 		 * so we need two separate attempts.
80518662e0SNeilBrown 		 */
81518662e0SNeilBrown 		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
82518662e0SNeilBrown 		if (!p)
83518662e0SNeilBrown 			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
84518662e0SNeilBrown 					     __GFP_NOWARN | __GFP_NORETRY);
85518662e0SNeilBrown 		if (!p)
86518662e0SNeilBrown 			return NULL;
87518662e0SNeilBrown 	}
88518662e0SNeilBrown 
891da177e4SLinus Torvalds 	memset(p, 0, sizeof(*p));
901da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->pages);
911da177e4SLinus Torvalds 	return p;
921da177e4SLinus Torvalds }
93e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
941da177e4SLinus Torvalds 
950b7c0153SFred Isaman void nfs_commit_free(struct nfs_commit_data *p)
961da177e4SLinus Torvalds {
971da177e4SLinus Torvalds 	mempool_free(p, nfs_commit_mempool);
981da177e4SLinus Torvalds }
99e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_commit_free);
1001da177e4SLinus Torvalds 
1011e7f3a48SWeston Andros Adamson static struct nfs_pgio_header *nfs_writehdr_alloc(void)
1023feb2d49STrond Myklebust {
1031e7f3a48SWeston Andros Adamson 	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
1043feb2d49STrond Myklebust 
105fbe77c30SBenjamin Coddington 	if (p) {
1063feb2d49STrond Myklebust 		memset(p, 0, sizeof(*p));
107fbe77c30SBenjamin Coddington 		p->rw_mode = FMODE_WRITE;
108fbe77c30SBenjamin Coddington 	}
1093feb2d49STrond Myklebust 	return p;
1103feb2d49STrond Myklebust }
1113feb2d49STrond Myklebust 
1121e7f3a48SWeston Andros Adamson static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
1136c75dc0dSFred Isaman {
1141e7f3a48SWeston Andros Adamson 	mempool_free(hdr, nfs_wdata_mempool);
1153feb2d49STrond Myklebust }
1161da177e4SLinus Torvalds 
117919e3bd9STrond Myklebust static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
118919e3bd9STrond Myklebust {
119919e3bd9STrond Myklebust 	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
120919e3bd9STrond Myklebust }
121919e3bd9STrond Myklebust 
122919e3bd9STrond Myklebust static void nfs_io_completion_init(struct nfs_io_completion *ioc,
123919e3bd9STrond Myklebust 		void (*complete)(void *), void *data)
124919e3bd9STrond Myklebust {
125919e3bd9STrond Myklebust 	ioc->complete = complete;
126919e3bd9STrond Myklebust 	ioc->data = data;
127919e3bd9STrond Myklebust 	kref_init(&ioc->refcount);
128919e3bd9STrond Myklebust }
129919e3bd9STrond Myklebust 
130919e3bd9STrond Myklebust static void nfs_io_completion_release(struct kref *kref)
131919e3bd9STrond Myklebust {
132919e3bd9STrond Myklebust 	struct nfs_io_completion *ioc = container_of(kref,
133919e3bd9STrond Myklebust 			struct nfs_io_completion, refcount);
134919e3bd9STrond Myklebust 	ioc->complete(ioc->data);
135919e3bd9STrond Myklebust 	kfree(ioc);
136919e3bd9STrond Myklebust }
137919e3bd9STrond Myklebust 
138919e3bd9STrond Myklebust static void nfs_io_completion_get(struct nfs_io_completion *ioc)
139919e3bd9STrond Myklebust {
140919e3bd9STrond Myklebust 	if (ioc != NULL)
141919e3bd9STrond Myklebust 		kref_get(&ioc->refcount);
142919e3bd9STrond Myklebust }
143919e3bd9STrond Myklebust 
144919e3bd9STrond Myklebust static void nfs_io_completion_put(struct nfs_io_completion *ioc)
145919e3bd9STrond Myklebust {
146919e3bd9STrond Myklebust 	if (ioc != NULL)
147919e3bd9STrond Myklebust 		kref_put(&ioc->refcount, nfs_io_completion_release);
148919e3bd9STrond Myklebust }
149919e3bd9STrond Myklebust 
1507b159fc1STrond Myklebust static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
1517b159fc1STrond Myklebust {
1527b159fc1STrond Myklebust 	ctx->error = error;
1537b159fc1STrond Myklebust 	smp_wmb();
1547b159fc1STrond Myklebust 	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
1557b159fc1STrond Myklebust }
1567b159fc1STrond Myklebust 
15784d3a9a9SWeston Andros Adamson /*
15884d3a9a9SWeston Andros Adamson  * nfs_page_find_head_request_locked - find head request associated with @page
15984d3a9a9SWeston Andros Adamson  *
16084d3a9a9SWeston Andros Adamson  * must be called while holding the inode lock.
16184d3a9a9SWeston Andros Adamson  *
16284d3a9a9SWeston Andros Adamson  * returns matching head request with reference held, or NULL if not found.
16384d3a9a9SWeston Andros Adamson  */
16429418aa4SMel Gorman static struct nfs_page *
16584d3a9a9SWeston Andros Adamson nfs_page_find_head_request_locked(struct nfs_inode *nfsi, struct page *page)
166277459d2STrond Myklebust {
167277459d2STrond Myklebust 	struct nfs_page *req = NULL;
168277459d2STrond Myklebust 
16929418aa4SMel Gorman 	if (PagePrivate(page))
170277459d2STrond Myklebust 		req = (struct nfs_page *)page_private(page);
17102d1426cSWeston Andros Adamson 	else if (unlikely(PageSwapCache(page)))
17202d1426cSWeston Andros Adamson 		req = nfs_page_search_commits_for_head_request_locked(nfsi,
17302d1426cSWeston Andros Adamson 			page);
17429418aa4SMel Gorman 
17584d3a9a9SWeston Andros Adamson 	if (req) {
17684d3a9a9SWeston Andros Adamson 		WARN_ON_ONCE(req->wb_head != req);
17729418aa4SMel Gorman 		kref_get(&req->wb_kref);
17884d3a9a9SWeston Andros Adamson 	}
17929418aa4SMel Gorman 
180277459d2STrond Myklebust 	return req;
181277459d2STrond Myklebust }
182277459d2STrond Myklebust 
18384d3a9a9SWeston Andros Adamson /*
18484d3a9a9SWeston Andros Adamson  * nfs_page_find_head_request - find head request associated with @page
18584d3a9a9SWeston Andros Adamson  *
18684d3a9a9SWeston Andros Adamson  * returns matching head request with reference held, or NULL if not found.
18784d3a9a9SWeston Andros Adamson  */
18884d3a9a9SWeston Andros Adamson static struct nfs_page *nfs_page_find_head_request(struct page *page)
189277459d2STrond Myklebust {
190d56b4ddfSMel Gorman 	struct inode *inode = page_file_mapping(page)->host;
191277459d2STrond Myklebust 	struct nfs_page *req = NULL;
192277459d2STrond Myklebust 
193587142f8STrond Myklebust 	spin_lock(&inode->i_lock);
19484d3a9a9SWeston Andros Adamson 	req = nfs_page_find_head_request_locked(NFS_I(inode), page);
195587142f8STrond Myklebust 	spin_unlock(&inode->i_lock);
196277459d2STrond Myklebust 	return req;
197277459d2STrond Myklebust }
198277459d2STrond Myklebust 
1991da177e4SLinus Torvalds /* Adjust the file length if we're writing beyond the end */
2001da177e4SLinus Torvalds static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
2011da177e4SLinus Torvalds {
202d56b4ddfSMel Gorman 	struct inode *inode = page_file_mapping(page)->host;
203a3d01454STrond Myklebust 	loff_t end, i_size;
204a3d01454STrond Myklebust 	pgoff_t end_index;
2051da177e4SLinus Torvalds 
206a3d01454STrond Myklebust 	spin_lock(&inode->i_lock);
207a3d01454STrond Myklebust 	i_size = i_size_read(inode);
20809cbfeafSKirill A. Shutemov 	end_index = (i_size - 1) >> PAGE_SHIFT;
2098cd79788SHuang Ying 	if (i_size > 0 && page_index(page) < end_index)
210a3d01454STrond Myklebust 		goto out;
211d56b4ddfSMel Gorman 	end = page_file_offset(page) + ((loff_t)offset+count);
2121da177e4SLinus Torvalds 	if (i_size >= end)
213a3d01454STrond Myklebust 		goto out;
2141da177e4SLinus Torvalds 	i_size_write(inode, end);
215a3d01454STrond Myklebust 	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
216a3d01454STrond Myklebust out:
217a3d01454STrond Myklebust 	spin_unlock(&inode->i_lock);
2181da177e4SLinus Torvalds }
2191da177e4SLinus Torvalds 
220a301b777STrond Myklebust /* A writeback failed: mark the page as bad, and invalidate the page cache */
221a301b777STrond Myklebust static void nfs_set_pageerror(struct page *page)
222a301b777STrond Myklebust {
223d56b4ddfSMel Gorman 	nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
224a301b777STrond Myklebust }
225a301b777STrond Myklebust 
226d72ddcbaSWeston Andros Adamson /*
227d72ddcbaSWeston Andros Adamson  * nfs_page_group_search_locked
228d72ddcbaSWeston Andros Adamson  * @head - head request of page group
229d72ddcbaSWeston Andros Adamson  * @page_offset - offset into page
230d72ddcbaSWeston Andros Adamson  *
231d72ddcbaSWeston Andros Adamson  * Search page group with head @head to find a request that contains the
232d72ddcbaSWeston Andros Adamson  * page offset @page_offset.
233d72ddcbaSWeston Andros Adamson  *
234d72ddcbaSWeston Andros Adamson  * Returns a pointer to the first matching nfs request, or NULL if no
235d72ddcbaSWeston Andros Adamson  * match is found.
236d72ddcbaSWeston Andros Adamson  *
237d72ddcbaSWeston Andros Adamson  * Must be called with the page group lock held
238d72ddcbaSWeston Andros Adamson  */
239d72ddcbaSWeston Andros Adamson static struct nfs_page *
240d72ddcbaSWeston Andros Adamson nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
241d72ddcbaSWeston Andros Adamson {
242d72ddcbaSWeston Andros Adamson 	struct nfs_page *req;
243d72ddcbaSWeston Andros Adamson 
244d72ddcbaSWeston Andros Adamson 	WARN_ON_ONCE(head != head->wb_head);
245d72ddcbaSWeston Andros Adamson 	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_head->wb_flags));
246d72ddcbaSWeston Andros Adamson 
247d72ddcbaSWeston Andros Adamson 	req = head;
248d72ddcbaSWeston Andros Adamson 	do {
249d72ddcbaSWeston Andros Adamson 		if (page_offset >= req->wb_pgbase &&
250d72ddcbaSWeston Andros Adamson 		    page_offset < (req->wb_pgbase + req->wb_bytes))
251d72ddcbaSWeston Andros Adamson 			return req;
252d72ddcbaSWeston Andros Adamson 
253d72ddcbaSWeston Andros Adamson 		req = req->wb_this_page;
254d72ddcbaSWeston Andros Adamson 	} while (req != head);
255d72ddcbaSWeston Andros Adamson 
256d72ddcbaSWeston Andros Adamson 	return NULL;
257d72ddcbaSWeston Andros Adamson }
258d72ddcbaSWeston Andros Adamson 
259d72ddcbaSWeston Andros Adamson /*
260d72ddcbaSWeston Andros Adamson  * nfs_page_group_covers_page
261d72ddcbaSWeston Andros Adamson  * @head - head request of page group
262d72ddcbaSWeston Andros Adamson  *
263d72ddcbaSWeston Andros Adamson  * Return true if the page group with head @head covers the whole page,
264d72ddcbaSWeston Andros Adamson  * returns false otherwise
265d72ddcbaSWeston Andros Adamson  */
266d72ddcbaSWeston Andros Adamson static bool nfs_page_group_covers_page(struct nfs_page *req)
267d72ddcbaSWeston Andros Adamson {
268d72ddcbaSWeston Andros Adamson 	struct nfs_page *tmp;
269d72ddcbaSWeston Andros Adamson 	unsigned int pos = 0;
270d72ddcbaSWeston Andros Adamson 	unsigned int len = nfs_page_length(req->wb_page);
271d72ddcbaSWeston Andros Adamson 
272fd2f3a06SWeston Andros Adamson 	nfs_page_group_lock(req, false);
273d72ddcbaSWeston Andros Adamson 
274d72ddcbaSWeston Andros Adamson 	do {
275d72ddcbaSWeston Andros Adamson 		tmp = nfs_page_group_search_locked(req->wb_head, pos);
276d72ddcbaSWeston Andros Adamson 		if (tmp) {
277d72ddcbaSWeston Andros Adamson 			/* no way this should happen */
278d72ddcbaSWeston Andros Adamson 			WARN_ON_ONCE(tmp->wb_pgbase != pos);
279d72ddcbaSWeston Andros Adamson 			pos += tmp->wb_bytes - (pos - tmp->wb_pgbase);
280d72ddcbaSWeston Andros Adamson 		}
281d72ddcbaSWeston Andros Adamson 	} while (tmp && pos < len);
282d72ddcbaSWeston Andros Adamson 
283d72ddcbaSWeston Andros Adamson 	nfs_page_group_unlock(req);
284d72ddcbaSWeston Andros Adamson 	WARN_ON_ONCE(pos > len);
285d72ddcbaSWeston Andros Adamson 	return pos == len;
286d72ddcbaSWeston Andros Adamson }
287d72ddcbaSWeston Andros Adamson 
2881da177e4SLinus Torvalds /* We can set the PG_uptodate flag if we see that a write request
2891da177e4SLinus Torvalds  * covers the full page.
2901da177e4SLinus Torvalds  */
291d72ddcbaSWeston Andros Adamson static void nfs_mark_uptodate(struct nfs_page *req)
2921da177e4SLinus Torvalds {
293d72ddcbaSWeston Andros Adamson 	if (PageUptodate(req->wb_page))
2941da177e4SLinus Torvalds 		return;
295d72ddcbaSWeston Andros Adamson 	if (!nfs_page_group_covers_page(req))
2961da177e4SLinus Torvalds 		return;
297d72ddcbaSWeston Andros Adamson 	SetPageUptodate(req->wb_page);
2981da177e4SLinus Torvalds }
2991da177e4SLinus Torvalds 
3001da177e4SLinus Torvalds static int wb_priority(struct writeback_control *wbc)
3011da177e4SLinus Torvalds {
302e87b4c7aSNeilBrown 	int ret = 0;
303cca588d6STrond Myklebust 
304e87b4c7aSNeilBrown 	if (wbc->sync_mode == WB_SYNC_ALL)
305e87b4c7aSNeilBrown 		ret = FLUSH_COND_STABLE;
306e87b4c7aSNeilBrown 	return ret;
3071da177e4SLinus Torvalds }
3081da177e4SLinus Torvalds 
3091da177e4SLinus Torvalds /*
31089a09141SPeter Zijlstra  * NFS congestion control
31189a09141SPeter Zijlstra  */
31289a09141SPeter Zijlstra 
31389a09141SPeter Zijlstra int nfs_congestion_kb;
31489a09141SPeter Zijlstra 
31589a09141SPeter Zijlstra #define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
31689a09141SPeter Zijlstra #define NFS_CONGESTION_OFF_THRESH	\
31789a09141SPeter Zijlstra 	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
31889a09141SPeter Zijlstra 
319deed85e7STrond Myklebust static void nfs_set_page_writeback(struct page *page)
32089a09141SPeter Zijlstra {
3210db10944SJan Kara 	struct inode *inode = page_file_mapping(page)->host;
3220db10944SJan Kara 	struct nfs_server *nfss = NFS_SERVER(inode);
3235a6d41b3STrond Myklebust 	int ret = test_set_page_writeback(page);
3245a6d41b3STrond Myklebust 
325deed85e7STrond Myklebust 	WARN_ON_ONCE(ret != 0);
32689a09141SPeter Zijlstra 
327277866a0SPeter Zijlstra 	if (atomic_long_inc_return(&nfss->writeback) >
3280db10944SJan Kara 			NFS_CONGESTION_ON_THRESH)
3290db10944SJan Kara 		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
33089a09141SPeter Zijlstra }
33189a09141SPeter Zijlstra 
33220633f04SWeston Andros Adamson static void nfs_end_page_writeback(struct nfs_page *req)
33389a09141SPeter Zijlstra {
33420633f04SWeston Andros Adamson 	struct inode *inode = page_file_mapping(req->wb_page)->host;
33589a09141SPeter Zijlstra 	struct nfs_server *nfss = NFS_SERVER(inode);
33689a09141SPeter Zijlstra 
33720633f04SWeston Andros Adamson 	if (!nfs_page_group_sync_on_bit(req, PG_WB_END))
33820633f04SWeston Andros Adamson 		return;
33920633f04SWeston Andros Adamson 
34020633f04SWeston Andros Adamson 	end_page_writeback(req->wb_page);
341c4dc4beeSPeter Zijlstra 	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
3420db10944SJan Kara 		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
34389a09141SPeter Zijlstra }
34489a09141SPeter Zijlstra 
345d4581383SWeston Andros Adamson 
346d4581383SWeston Andros Adamson /* nfs_page_group_clear_bits
347d4581383SWeston Andros Adamson  *   @req - an nfs request
348d4581383SWeston Andros Adamson  * clears all page group related bits from @req
349d4581383SWeston Andros Adamson  */
350d4581383SWeston Andros Adamson static void
351d4581383SWeston Andros Adamson nfs_page_group_clear_bits(struct nfs_page *req)
352e261f51fSTrond Myklebust {
353d4581383SWeston Andros Adamson 	clear_bit(PG_TEARDOWN, &req->wb_flags);
354d4581383SWeston Andros Adamson 	clear_bit(PG_UNLOCKPAGE, &req->wb_flags);
355d4581383SWeston Andros Adamson 	clear_bit(PG_UPTODATE, &req->wb_flags);
356d4581383SWeston Andros Adamson 	clear_bit(PG_WB_END, &req->wb_flags);
357d4581383SWeston Andros Adamson 	clear_bit(PG_REMOVE, &req->wb_flags);
358d4581383SWeston Andros Adamson }
359d4581383SWeston Andros Adamson 
360d4581383SWeston Andros Adamson 
361d4581383SWeston Andros Adamson /*
362d4581383SWeston Andros Adamson  * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
363d4581383SWeston Andros Adamson  *
364d4581383SWeston Andros Adamson  * this is a helper function for nfs_lock_and_join_requests
365d4581383SWeston Andros Adamson  *
366d4581383SWeston Andros Adamson  * @inode - inode associated with request page group, must be holding inode lock
367d4581383SWeston Andros Adamson  * @head  - head request of page group, must be holding head lock
368d4581383SWeston Andros Adamson  * @req   - request that couldn't lock and needs to wait on the req bit lock
369d4581383SWeston Andros Adamson  *
370d4581383SWeston Andros Adamson  * NOTE: this must be called holding page_group bit lock and inode spin lock
371d4581383SWeston Andros Adamson  *       and BOTH will be released before returning.
372d4581383SWeston Andros Adamson  *
373d4581383SWeston Andros Adamson  * returns 0 on success, < 0 on error.
374d4581383SWeston Andros Adamson  */
375d4581383SWeston Andros Adamson static int
376d4581383SWeston Andros Adamson nfs_unroll_locks_and_wait(struct inode *inode, struct nfs_page *head,
377*6d17d653STrond Myklebust 			  struct nfs_page *req)
378d4581383SWeston Andros Adamson 	__releases(&inode->i_lock)
379d4581383SWeston Andros Adamson {
380d4581383SWeston Andros Adamson 	struct nfs_page *tmp;
381e261f51fSTrond Myklebust 	int ret;
382e261f51fSTrond Myklebust 
383d4581383SWeston Andros Adamson 	/* relinquish all the locks successfully grabbed this run */
384d4581383SWeston Andros Adamson 	for (tmp = head ; tmp != req; tmp = tmp->wb_this_page)
385d4581383SWeston Andros Adamson 		nfs_unlock_request(tmp);
386d4581383SWeston Andros Adamson 
387d4581383SWeston Andros Adamson 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
388d4581383SWeston Andros Adamson 
389d4581383SWeston Andros Adamson 	/* grab a ref on the request that will be waited on */
390d4581383SWeston Andros Adamson 	kref_get(&req->wb_kref);
391d4581383SWeston Andros Adamson 
392d4581383SWeston Andros Adamson 	nfs_page_group_unlock(head);
393587142f8STrond Myklebust 	spin_unlock(&inode->i_lock);
394d4581383SWeston Andros Adamson 
395d4581383SWeston Andros Adamson 	/* release ref from nfs_page_find_head_request_locked */
396d4581383SWeston Andros Adamson 	nfs_release_request(head);
397d4581383SWeston Andros Adamson 
398e261f51fSTrond Myklebust 	ret = nfs_wait_on_request(req);
399e261f51fSTrond Myklebust 	nfs_release_request(req);
400d4581383SWeston Andros Adamson 
401d4581383SWeston Andros Adamson 	return ret;
402e261f51fSTrond Myklebust }
403d4581383SWeston Andros Adamson 
404d4581383SWeston Andros Adamson /*
405d4581383SWeston Andros Adamson  * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
406d4581383SWeston Andros Adamson  *
407d4581383SWeston Andros Adamson  * @destroy_list - request list (using wb_this_page) terminated by @old_head
408d4581383SWeston Andros Adamson  * @old_head - the old head of the list
409d4581383SWeston Andros Adamson  *
410d4581383SWeston Andros Adamson  * All subrequests must be locked and removed from all lists, so at this point
411d4581383SWeston Andros Adamson  * they are only "active" in this function, and possibly in nfs_wait_on_request
412d4581383SWeston Andros Adamson  * with a reference held by some other context.
413d4581383SWeston Andros Adamson  */
414d4581383SWeston Andros Adamson static void
415d4581383SWeston Andros Adamson nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
416d4581383SWeston Andros Adamson 				 struct nfs_page *old_head)
417d4581383SWeston Andros Adamson {
418d4581383SWeston Andros Adamson 	while (destroy_list) {
419d4581383SWeston Andros Adamson 		struct nfs_page *subreq = destroy_list;
420d4581383SWeston Andros Adamson 
421d4581383SWeston Andros Adamson 		destroy_list = (subreq->wb_this_page == old_head) ?
422d4581383SWeston Andros Adamson 				   NULL : subreq->wb_this_page;
423d4581383SWeston Andros Adamson 
424d4581383SWeston Andros Adamson 		WARN_ON_ONCE(old_head != subreq->wb_head);
425d4581383SWeston Andros Adamson 
426d4581383SWeston Andros Adamson 		/* make sure old group is not used */
427d4581383SWeston Andros Adamson 		subreq->wb_head = subreq;
428d4581383SWeston Andros Adamson 		subreq->wb_this_page = subreq;
429d4581383SWeston Andros Adamson 
430d4581383SWeston Andros Adamson 		/* subreq is now totally disconnected from page group or any
431d4581383SWeston Andros Adamson 		 * write / commit lists. last chance to wake any waiters */
432d4581383SWeston Andros Adamson 		nfs_unlock_request(subreq);
433d4581383SWeston Andros Adamson 
434d4581383SWeston Andros Adamson 		if (!test_bit(PG_TEARDOWN, &subreq->wb_flags)) {
435d4581383SWeston Andros Adamson 			/* release ref on old head request */
436d4581383SWeston Andros Adamson 			nfs_release_request(old_head);
437d4581383SWeston Andros Adamson 
438d4581383SWeston Andros Adamson 			nfs_page_group_clear_bits(subreq);
439d4581383SWeston Andros Adamson 
440d4581383SWeston Andros Adamson 			/* release the PG_INODE_REF reference */
441d4581383SWeston Andros Adamson 			if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags))
442d4581383SWeston Andros Adamson 				nfs_release_request(subreq);
443d4581383SWeston Andros Adamson 			else
444d4581383SWeston Andros Adamson 				WARN_ON_ONCE(1);
445d4581383SWeston Andros Adamson 		} else {
446d4581383SWeston Andros Adamson 			WARN_ON_ONCE(test_bit(PG_CLEAN, &subreq->wb_flags));
447d4581383SWeston Andros Adamson 			/* zombie requests have already released the last
448d4581383SWeston Andros Adamson 			 * reference and were waiting on the rest of the
449d4581383SWeston Andros Adamson 			 * group to complete. Since it's no longer part of a
450d4581383SWeston Andros Adamson 			 * group, simply free the request */
451d4581383SWeston Andros Adamson 			nfs_page_group_clear_bits(subreq);
452d4581383SWeston Andros Adamson 			nfs_free_request(subreq);
453d4581383SWeston Andros Adamson 		}
454d4581383SWeston Andros Adamson 	}
455d4581383SWeston Andros Adamson }
456d4581383SWeston Andros Adamson 
457d4581383SWeston Andros Adamson /*
458d4581383SWeston Andros Adamson  * nfs_lock_and_join_requests - join all subreqs to the head req and return
459d4581383SWeston Andros Adamson  *                              a locked reference, cancelling any pending
460d4581383SWeston Andros Adamson  *                              operations for this page.
461d4581383SWeston Andros Adamson  *
462d4581383SWeston Andros Adamson  * @page - the page used to lookup the "page group" of nfs_page structures
463d4581383SWeston Andros Adamson  *
464d4581383SWeston Andros Adamson  * This function joins all sub requests to the head request by first
465d4581383SWeston Andros Adamson  * locking all requests in the group, cancelling any pending operations
466d4581383SWeston Andros Adamson  * and finally updating the head request to cover the whole range covered by
467d4581383SWeston Andros Adamson  * the (former) group.  All subrequests are removed from any write or commit
468d4581383SWeston Andros Adamson  * lists, unlinked from the group and destroyed.
469d4581383SWeston Andros Adamson  *
470d4581383SWeston Andros Adamson  * Returns a locked, referenced pointer to the head request - which after
471d4581383SWeston Andros Adamson  * this call is guaranteed to be the only request associated with the page.
472d4581383SWeston Andros Adamson  * Returns NULL if no requests are found for @page, or a ERR_PTR if an
473d4581383SWeston Andros Adamson  * error was encountered.
474d4581383SWeston Andros Adamson  */
475d4581383SWeston Andros Adamson static struct nfs_page *
476*6d17d653STrond Myklebust nfs_lock_and_join_requests(struct page *page)
477d4581383SWeston Andros Adamson {
478d4581383SWeston Andros Adamson 	struct inode *inode = page_file_mapping(page)->host;
479d4581383SWeston Andros Adamson 	struct nfs_page *head, *subreq;
480d4581383SWeston Andros Adamson 	struct nfs_page *destroy_list = NULL;
481d4581383SWeston Andros Adamson 	unsigned int total_bytes;
482d4581383SWeston Andros Adamson 	int ret;
483d4581383SWeston Andros Adamson 
484d4581383SWeston Andros Adamson try_again:
485d4581383SWeston Andros Adamson 	total_bytes = 0;
486d4581383SWeston Andros Adamson 
487d4581383SWeston Andros Adamson 	WARN_ON_ONCE(destroy_list);
488d4581383SWeston Andros Adamson 
489d4581383SWeston Andros Adamson 	spin_lock(&inode->i_lock);
490d4581383SWeston Andros Adamson 
491d4581383SWeston Andros Adamson 	/*
492d4581383SWeston Andros Adamson 	 * A reference is taken only on the head request which acts as a
493d4581383SWeston Andros Adamson 	 * reference to the whole page group - the group will not be destroyed
494d4581383SWeston Andros Adamson 	 * until the head reference is released.
495d4581383SWeston Andros Adamson 	 */
496d4581383SWeston Andros Adamson 	head = nfs_page_find_head_request_locked(NFS_I(inode), page);
497d4581383SWeston Andros Adamson 
498d4581383SWeston Andros Adamson 	if (!head) {
499587142f8STrond Myklebust 		spin_unlock(&inode->i_lock);
500d4581383SWeston Andros Adamson 		return NULL;
501d4581383SWeston Andros Adamson 	}
502d4581383SWeston Andros Adamson 
5037c3af975SWeston Andros Adamson 	/* holding inode lock, so always make a non-blocking call to try the
5047c3af975SWeston Andros Adamson 	 * page group lock */
505fd2f3a06SWeston Andros Adamson 	ret = nfs_page_group_lock(head, true);
50694970014SWeston Andros Adamson 	if (ret < 0) {
50794970014SWeston Andros Adamson 		spin_unlock(&inode->i_lock);
5087c3af975SWeston Andros Adamson 
5097c3af975SWeston Andros Adamson 		nfs_page_group_lock_wait(head);
5107c3af975SWeston Andros Adamson 		nfs_release_request(head);
5117c3af975SWeston Andros Adamson 		goto try_again;
5127c3af975SWeston Andros Adamson 	}
5137c3af975SWeston Andros Adamson 
5147c3af975SWeston Andros Adamson 	/* lock each request in the page group */
515d4581383SWeston Andros Adamson 	subreq = head;
516d4581383SWeston Andros Adamson 	do {
517d4581383SWeston Andros Adamson 		/*
518d4581383SWeston Andros Adamson 		 * Subrequests are always contiguous, non overlapping
519309a1d65SWeston Andros Adamson 		 * and in order - but may be repeated (mirrored writes).
520d4581383SWeston Andros Adamson 		 */
521309a1d65SWeston Andros Adamson 		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
522d4581383SWeston Andros Adamson 			/* keep track of how many bytes this group covers */
523d4581383SWeston Andros Adamson 			total_bytes += subreq->wb_bytes;
524309a1d65SWeston Andros Adamson 		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
525309a1d65SWeston Andros Adamson 			    ((subreq->wb_offset + subreq->wb_bytes) >
526309a1d65SWeston Andros Adamson 			     (head->wb_offset + total_bytes)))) {
527309a1d65SWeston Andros Adamson 			nfs_page_group_unlock(head);
528309a1d65SWeston Andros Adamson 			spin_unlock(&inode->i_lock);
529309a1d65SWeston Andros Adamson 			return ERR_PTR(-EIO);
530309a1d65SWeston Andros Adamson 		}
531d4581383SWeston Andros Adamson 
532d4581383SWeston Andros Adamson 		if (!nfs_lock_request(subreq)) {
533d4581383SWeston Andros Adamson 			/* releases page group bit lock and
534d4581383SWeston Andros Adamson 			 * inode spin lock and all references */
535d4581383SWeston Andros Adamson 			ret = nfs_unroll_locks_and_wait(inode, head,
536*6d17d653STrond Myklebust 				subreq);
537d4581383SWeston Andros Adamson 
538d4581383SWeston Andros Adamson 			if (ret == 0)
539d4581383SWeston Andros Adamson 				goto try_again;
540d4581383SWeston Andros Adamson 
541d4581383SWeston Andros Adamson 			return ERR_PTR(ret);
542d4581383SWeston Andros Adamson 		}
543d4581383SWeston Andros Adamson 
544d4581383SWeston Andros Adamson 		subreq = subreq->wb_this_page;
545d4581383SWeston Andros Adamson 	} while (subreq != head);
546d4581383SWeston Andros Adamson 
547d4581383SWeston Andros Adamson 	/* Now that all requests are locked, make sure they aren't on any list.
548d4581383SWeston Andros Adamson 	 * Commit list removal accounting is done after locks are dropped */
549d4581383SWeston Andros Adamson 	subreq = head;
550d4581383SWeston Andros Adamson 	do {
551411a99adSWeston Andros Adamson 		nfs_clear_request_commit(subreq);
552d4581383SWeston Andros Adamson 		subreq = subreq->wb_this_page;
553d4581383SWeston Andros Adamson 	} while (subreq != head);
554d4581383SWeston Andros Adamson 
555d4581383SWeston Andros Adamson 	/* unlink subrequests from head, destroy them later */
556d4581383SWeston Andros Adamson 	if (head->wb_this_page != head) {
557d4581383SWeston Andros Adamson 		/* destroy list will be terminated by head */
558d4581383SWeston Andros Adamson 		destroy_list = head->wb_this_page;
559d4581383SWeston Andros Adamson 		head->wb_this_page = head;
560d4581383SWeston Andros Adamson 
561d4581383SWeston Andros Adamson 		/* change head request to cover whole range that
562d4581383SWeston Andros Adamson 		 * the former page group covered */
563d4581383SWeston Andros Adamson 		head->wb_bytes = total_bytes;
564d4581383SWeston Andros Adamson 	}
565d4581383SWeston Andros Adamson 
566d4581383SWeston Andros Adamson 	/*
567d4581383SWeston Andros Adamson 	 * prepare head request to be added to new pgio descriptor
568d4581383SWeston Andros Adamson 	 */
569d4581383SWeston Andros Adamson 	nfs_page_group_clear_bits(head);
570d4581383SWeston Andros Adamson 
571d4581383SWeston Andros Adamson 	/*
572d4581383SWeston Andros Adamson 	 * some part of the group was still on the inode list - otherwise
573d4581383SWeston Andros Adamson 	 * the group wouldn't be involved in async write.
574d4581383SWeston Andros Adamson 	 * grab a reference for the head request, iff it needs one.
575d4581383SWeston Andros Adamson 	 */
576d4581383SWeston Andros Adamson 	if (!test_and_set_bit(PG_INODE_REF, &head->wb_flags))
577d4581383SWeston Andros Adamson 		kref_get(&head->wb_kref);
578d4581383SWeston Andros Adamson 
579d4581383SWeston Andros Adamson 	nfs_page_group_unlock(head);
580d4581383SWeston Andros Adamson 
581411a99adSWeston Andros Adamson 	/* drop lock to clean uprequests on destroy list */
582d4581383SWeston Andros Adamson 	spin_unlock(&inode->i_lock);
583d4581383SWeston Andros Adamson 
584d4581383SWeston Andros Adamson 	nfs_destroy_unlinked_subrequests(destroy_list, head);
585d4581383SWeston Andros Adamson 
586d4581383SWeston Andros Adamson 	/* still holds ref on head from nfs_page_find_head_request_locked
587d4581383SWeston Andros Adamson 	 * and still has lock on head from lock loop */
588d4581383SWeston Andros Adamson 	return head;
589612c9384STrond Myklebust }
590074cc1deSTrond Myklebust 
5910bcbf039SPeng Tao static void nfs_write_error_remove_page(struct nfs_page *req)
5920bcbf039SPeng Tao {
5930bcbf039SPeng Tao 	nfs_unlock_request(req);
5940bcbf039SPeng Tao 	nfs_end_page_writeback(req);
5950bcbf039SPeng Tao 	generic_error_remove_page(page_file_mapping(req->wb_page),
5960bcbf039SPeng Tao 				  req->wb_page);
5971f84ccdfSFred Isaman 	nfs_release_request(req);
5980bcbf039SPeng Tao }
5990bcbf039SPeng Tao 
600a6598813STrond Myklebust static bool
601a6598813STrond Myklebust nfs_error_is_fatal_on_server(int err)
602a6598813STrond Myklebust {
603a6598813STrond Myklebust 	switch (err) {
604a6598813STrond Myklebust 	case 0:
605a6598813STrond Myklebust 	case -ERESTARTSYS:
606a6598813STrond Myklebust 	case -EINTR:
607a6598813STrond Myklebust 		return false;
608a6598813STrond Myklebust 	}
609a6598813STrond Myklebust 	return nfs_error_is_fatal(err);
610e261f51fSTrond Myklebust }
611074cc1deSTrond Myklebust 
612074cc1deSTrond Myklebust /*
613074cc1deSTrond Myklebust  * Find an associated nfs write request, and prepare to flush it out
614074cc1deSTrond Myklebust  * May return an error if the user signalled nfs_wait_on_request().
615074cc1deSTrond Myklebust  */
616074cc1deSTrond Myklebust static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
617*6d17d653STrond Myklebust 				struct page *page)
618074cc1deSTrond Myklebust {
619074cc1deSTrond Myklebust 	struct nfs_page *req;
620074cc1deSTrond Myklebust 	int ret = 0;
621074cc1deSTrond Myklebust 
622*6d17d653STrond Myklebust 	req = nfs_lock_and_join_requests(page);
623074cc1deSTrond Myklebust 	if (!req)
624074cc1deSTrond Myklebust 		goto out;
625074cc1deSTrond Myklebust 	ret = PTR_ERR(req);
626074cc1deSTrond Myklebust 	if (IS_ERR(req))
627074cc1deSTrond Myklebust 		goto out;
628074cc1deSTrond Myklebust 
629deed85e7STrond Myklebust 	nfs_set_page_writeback(page);
630deed85e7STrond Myklebust 	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
631074cc1deSTrond Myklebust 
632deed85e7STrond Myklebust 	ret = 0;
633a6598813STrond Myklebust 	/* If there is a fatal error that covers this write, just exit */
634a6598813STrond Myklebust 	if (nfs_error_is_fatal_on_server(req->wb_context->error))
635a6598813STrond Myklebust 		goto out_launder;
636a6598813STrond Myklebust 
637f8512ad0SFred Isaman 	if (!nfs_pageio_add_request(pgio, req)) {
638074cc1deSTrond Myklebust 		ret = pgio->pg_error;
6390bcbf039SPeng Tao 		/*
640c373fff7STrond Myklebust 		 * Remove the problematic req upon fatal errors on the server
6410bcbf039SPeng Tao 		 */
6420bcbf039SPeng Tao 		if (nfs_error_is_fatal(ret)) {
6430bcbf039SPeng Tao 			nfs_context_set_write_error(req->wb_context, ret);
644c373fff7STrond Myklebust 			if (nfs_error_is_fatal_on_server(ret))
645a6598813STrond Myklebust 				goto out_launder;
646d6c843b9SPeng Tao 		}
6470bcbf039SPeng Tao 		nfs_redirty_request(req);
6480bcbf039SPeng Tao 		ret = -EAGAIN;
64940f90271STrond Myklebust 	} else
65040f90271STrond Myklebust 		nfs_add_stats(page_file_mapping(page)->host,
65140f90271STrond Myklebust 				NFSIOS_WRITEPAGES, 1);
652074cc1deSTrond Myklebust out:
653074cc1deSTrond Myklebust 	return ret;
654a6598813STrond Myklebust out_launder:
655a6598813STrond Myklebust 	nfs_write_error_remove_page(req);
656a6598813STrond Myklebust 	return ret;
657e261f51fSTrond Myklebust }
658e261f51fSTrond Myklebust 
659d6c843b9SPeng Tao static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
660c373fff7STrond Myklebust 			    struct nfs_pageio_descriptor *pgio)
661f758c885STrond Myklebust {
662cfb506e1STrond Myklebust 	int ret;
663f758c885STrond Myklebust 
6648cd79788SHuang Ying 	nfs_pageio_cond_complete(pgio, page_index(page));
665*6d17d653STrond Myklebust 	ret = nfs_page_async_flush(pgio, page);
666cfb506e1STrond Myklebust 	if (ret == -EAGAIN) {
667cfb506e1STrond Myklebust 		redirty_page_for_writepage(wbc, page);
668cfb506e1STrond Myklebust 		ret = 0;
669cfb506e1STrond Myklebust 	}
670cfb506e1STrond Myklebust 	return ret;
671f758c885STrond Myklebust }
672f758c885STrond Myklebust 
673e261f51fSTrond Myklebust /*
6741da177e4SLinus Torvalds  * Write an mmapped page to the server.
6751da177e4SLinus Torvalds  */
676d6c843b9SPeng Tao static int nfs_writepage_locked(struct page *page,
677c373fff7STrond Myklebust 				struct writeback_control *wbc)
6781da177e4SLinus Torvalds {
679f758c885STrond Myklebust 	struct nfs_pageio_descriptor pgio;
68040f90271STrond Myklebust 	struct inode *inode = page_file_mapping(page)->host;
681e261f51fSTrond Myklebust 	int err;
6821da177e4SLinus Torvalds 
68340f90271STrond Myklebust 	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
684811ed92eSTrond Myklebust 	nfs_pageio_init_write(&pgio, inode, 0,
685a20c93e3SChristoph Hellwig 				false, &nfs_async_write_completion_ops);
686c373fff7STrond Myklebust 	err = nfs_do_writepage(page, wbc, &pgio);
687f758c885STrond Myklebust 	nfs_pageio_complete(&pgio);
688f758c885STrond Myklebust 	if (err < 0)
6894d770ccfSTrond Myklebust 		return err;
690f758c885STrond Myklebust 	if (pgio.pg_error < 0)
691f758c885STrond Myklebust 		return pgio.pg_error;
692f758c885STrond Myklebust 	return 0;
6934d770ccfSTrond Myklebust }
6944d770ccfSTrond Myklebust 
6954d770ccfSTrond Myklebust int nfs_writepage(struct page *page, struct writeback_control *wbc)
6964d770ccfSTrond Myklebust {
697f758c885STrond Myklebust 	int ret;
6984d770ccfSTrond Myklebust 
699c373fff7STrond Myklebust 	ret = nfs_writepage_locked(page, wbc);
7001da177e4SLinus Torvalds 	unlock_page(page);
701f758c885STrond Myklebust 	return ret;
702f758c885STrond Myklebust }
703f758c885STrond Myklebust 
704f758c885STrond Myklebust static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
705f758c885STrond Myklebust {
706f758c885STrond Myklebust 	int ret;
707f758c885STrond Myklebust 
708c373fff7STrond Myklebust 	ret = nfs_do_writepage(page, wbc, data);
709f758c885STrond Myklebust 	unlock_page(page);
710f758c885STrond Myklebust 	return ret;
7111da177e4SLinus Torvalds }
7121da177e4SLinus Torvalds 
713919e3bd9STrond Myklebust static void nfs_io_completion_commit(void *inode)
714919e3bd9STrond Myklebust {
715919e3bd9STrond Myklebust 	nfs_commit_inode(inode, 0);
716919e3bd9STrond Myklebust }
717919e3bd9STrond Myklebust 
7181da177e4SLinus Torvalds int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
7191da177e4SLinus Torvalds {
7201da177e4SLinus Torvalds 	struct inode *inode = mapping->host;
721c63c7b05STrond Myklebust 	struct nfs_pageio_descriptor pgio;
722919e3bd9STrond Myklebust 	struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS);
7231da177e4SLinus Torvalds 	int err;
7241da177e4SLinus Torvalds 
72591d5b470SChuck Lever 	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
72691d5b470SChuck Lever 
727919e3bd9STrond Myklebust 	if (ioc)
728919e3bd9STrond Myklebust 		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);
729919e3bd9STrond Myklebust 
730a20c93e3SChristoph Hellwig 	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
731a20c93e3SChristoph Hellwig 				&nfs_async_write_completion_ops);
732919e3bd9STrond Myklebust 	pgio.pg_io_completion = ioc;
733f758c885STrond Myklebust 	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
734c63c7b05STrond Myklebust 	nfs_pageio_complete(&pgio);
735919e3bd9STrond Myklebust 	nfs_io_completion_put(ioc);
73672cb77f4STrond Myklebust 
737f758c885STrond Myklebust 	if (err < 0)
73872cb77f4STrond Myklebust 		goto out_err;
73972cb77f4STrond Myklebust 	err = pgio.pg_error;
74072cb77f4STrond Myklebust 	if (err < 0)
74172cb77f4STrond Myklebust 		goto out_err;
742c63c7b05STrond Myklebust 	return 0;
74372cb77f4STrond Myklebust out_err:
74472cb77f4STrond Myklebust 	return err;
7451da177e4SLinus Torvalds }
7461da177e4SLinus Torvalds 
7471da177e4SLinus Torvalds /*
7481da177e4SLinus Torvalds  * Insert a write request into an inode
7491da177e4SLinus Torvalds  */
750d6d6dc7cSFred Isaman static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
7511da177e4SLinus Torvalds {
7521da177e4SLinus Torvalds 	struct nfs_inode *nfsi = NFS_I(inode);
753e7d39069STrond Myklebust 
7542bfc6e56SWeston Andros Adamson 	WARN_ON_ONCE(req->wb_this_page != req);
7552bfc6e56SWeston Andros Adamson 
756e7d39069STrond Myklebust 	/* Lock the request! */
7577ad84aa9STrond Myklebust 	nfs_lock_request(req);
758e7d39069STrond Myklebust 
759e7d39069STrond Myklebust 	spin_lock(&inode->i_lock);
760cb1410c7SWeston Andros Adamson 	if (!nfsi->nrequests &&
761cb1410c7SWeston Andros Adamson 	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
762a9a4a87aSTrond Myklebust 		inode->i_version++;
76329418aa4SMel Gorman 	/*
76429418aa4SMel Gorman 	 * Swap-space should not get truncated. Hence no need to plug the race
76529418aa4SMel Gorman 	 * with invalidate/truncate.
76629418aa4SMel Gorman 	 */
76729418aa4SMel Gorman 	if (likely(!PageSwapCache(req->wb_page))) {
7682df485a7STrond Myklebust 		set_bit(PG_MAPPED, &req->wb_flags);
769deb7d638STrond Myklebust 		SetPagePrivate(req->wb_page);
770277459d2STrond Myklebust 		set_page_private(req->wb_page, (unsigned long)req);
77129418aa4SMel Gorman 	}
772cb1410c7SWeston Andros Adamson 	nfsi->nrequests++;
77317089a29SWeston Andros Adamson 	/* this a head request for a page group - mark it as having an
774cb1410c7SWeston Andros Adamson 	 * extra reference so sub groups can follow suit.
775cb1410c7SWeston Andros Adamson 	 * This flag also informs pgio layer when to bump nrequests when
776cb1410c7SWeston Andros Adamson 	 * adding subrequests. */
77717089a29SWeston Andros Adamson 	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
778c03b4024STrond Myklebust 	kref_get(&req->wb_kref);
779e7d39069STrond Myklebust 	spin_unlock(&inode->i_lock);
7801da177e4SLinus Torvalds }
7811da177e4SLinus Torvalds 
7821da177e4SLinus Torvalds /*
78389a09141SPeter Zijlstra  * Remove a write request from an inode
7841da177e4SLinus Torvalds  */
7851da177e4SLinus Torvalds static void nfs_inode_remove_request(struct nfs_page *req)
7861da177e4SLinus Torvalds {
7872b0143b5SDavid Howells 	struct inode *inode = d_inode(req->wb_context->dentry);
7881da177e4SLinus Torvalds 	struct nfs_inode *nfsi = NFS_I(inode);
78920633f04SWeston Andros Adamson 	struct nfs_page *head;
79020633f04SWeston Andros Adamson 
79120633f04SWeston Andros Adamson 	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
79220633f04SWeston Andros Adamson 		head = req->wb_head;
7931da177e4SLinus Torvalds 
794587142f8STrond Myklebust 		spin_lock(&inode->i_lock);
79567911c8fSAnna Schumaker 		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
79620633f04SWeston Andros Adamson 			set_page_private(head->wb_page, 0);
79720633f04SWeston Andros Adamson 			ClearPagePrivate(head->wb_page);
79820633f04SWeston Andros Adamson 			clear_bit(PG_MAPPED, &head->wb_flags);
79929418aa4SMel Gorman 		}
800cb1410c7SWeston Andros Adamson 		nfsi->nrequests--;
801cb1410c7SWeston Andros Adamson 		spin_unlock(&inode->i_lock);
802cb1410c7SWeston Andros Adamson 	} else {
803cb1410c7SWeston Andros Adamson 		spin_lock(&inode->i_lock);
804cb1410c7SWeston Andros Adamson 		nfsi->nrequests--;
805587142f8STrond Myklebust 		spin_unlock(&inode->i_lock);
80620633f04SWeston Andros Adamson 	}
80717089a29SWeston Andros Adamson 
80817089a29SWeston Andros Adamson 	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
8091da177e4SLinus Torvalds 		nfs_release_request(req);
8101da177e4SLinus Torvalds }
8111da177e4SLinus Torvalds 
81261822ab5STrond Myklebust static void
8136d884e8fSFred nfs_mark_request_dirty(struct nfs_page *req)
81461822ab5STrond Myklebust {
81567911c8fSAnna Schumaker 	if (req->wb_page)
81661822ab5STrond Myklebust 		__set_page_dirty_nobuffers(req->wb_page);
81761822ab5STrond Myklebust }
81861822ab5STrond Myklebust 
8193a3908c8STrond Myklebust /*
8203a3908c8STrond Myklebust  * nfs_page_search_commits_for_head_request_locked
8213a3908c8STrond Myklebust  *
8223a3908c8STrond Myklebust  * Search through commit lists on @inode for the head request for @page.
8233a3908c8STrond Myklebust  * Must be called while holding the inode (which is cinfo) lock.
8243a3908c8STrond Myklebust  *
8253a3908c8STrond Myklebust  * Returns the head request if found, or NULL if not found.
8263a3908c8STrond Myklebust  */
8273a3908c8STrond Myklebust static struct nfs_page *
8283a3908c8STrond Myklebust nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
8293a3908c8STrond Myklebust 						struct page *page)
8303a3908c8STrond Myklebust {
8313a3908c8STrond Myklebust 	struct nfs_page *freq, *t;
8323a3908c8STrond Myklebust 	struct nfs_commit_info cinfo;
8333a3908c8STrond Myklebust 	struct inode *inode = &nfsi->vfs_inode;
8343a3908c8STrond Myklebust 
8353a3908c8STrond Myklebust 	nfs_init_cinfo_from_inode(&cinfo, inode);
8363a3908c8STrond Myklebust 
8373a3908c8STrond Myklebust 	/* search through pnfs commit lists */
8383a3908c8STrond Myklebust 	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
8393a3908c8STrond Myklebust 	if (freq)
8403a3908c8STrond Myklebust 		return freq->wb_head;
8413a3908c8STrond Myklebust 
8423a3908c8STrond Myklebust 	/* Linearly search the commit list for the correct request */
8433a3908c8STrond Myklebust 	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
8443a3908c8STrond Myklebust 		if (freq->wb_page == page)
8453a3908c8STrond Myklebust 			return freq->wb_head;
8463a3908c8STrond Myklebust 	}
8473a3908c8STrond Myklebust 
8483a3908c8STrond Myklebust 	return NULL;
8493a3908c8STrond Myklebust }
8503a3908c8STrond Myklebust 
8518dd37758STrond Myklebust /**
85286d80f97STrond Myklebust  * nfs_request_add_commit_list_locked - add request to a commit list
85386d80f97STrond Myklebust  * @req: pointer to a struct nfs_page
85486d80f97STrond Myklebust  * @dst: commit list head
85586d80f97STrond Myklebust  * @cinfo: holds list lock and accounting info
85686d80f97STrond Myklebust  *
85786d80f97STrond Myklebust  * This sets the PG_CLEAN bit, updates the cinfo count of
85886d80f97STrond Myklebust  * number of outstanding requests requiring a commit as well as
85986d80f97STrond Myklebust  * the MM page stats.
86086d80f97STrond Myklebust  *
861fe238e60SDave Wysochanski  * The caller must hold cinfo->inode->i_lock, and the nfs_page lock.
86286d80f97STrond Myklebust  */
86386d80f97STrond Myklebust void
86486d80f97STrond Myklebust nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
86586d80f97STrond Myklebust 			    struct nfs_commit_info *cinfo)
86686d80f97STrond Myklebust {
86786d80f97STrond Myklebust 	set_bit(PG_CLEAN, &req->wb_flags);
86886d80f97STrond Myklebust 	nfs_list_add_request(req, dst);
86986d80f97STrond Myklebust 	cinfo->mds->ncommit++;
87086d80f97STrond Myklebust }
87186d80f97STrond Myklebust EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);
87286d80f97STrond Myklebust 
87386d80f97STrond Myklebust /**
8748dd37758STrond Myklebust  * nfs_request_add_commit_list - add request to a commit list
8758dd37758STrond Myklebust  * @req: pointer to a struct nfs_page
876ea2cf228SFred Isaman  * @dst: commit list head
877ea2cf228SFred Isaman  * @cinfo: holds list lock and accounting info
8788dd37758STrond Myklebust  *
879ea2cf228SFred Isaman  * This sets the PG_CLEAN bit, updates the cinfo count of
8808dd37758STrond Myklebust  * number of outstanding requests requiring a commit as well as
8818dd37758STrond Myklebust  * the MM page stats.
8828dd37758STrond Myklebust  *
883ea2cf228SFred Isaman  * The caller must _not_ hold the cinfo->lock, but must be
8848dd37758STrond Myklebust  * holding the nfs_page lock.
8858dd37758STrond Myklebust  */
8868dd37758STrond Myklebust void
8876272dcc6SAnna Schumaker nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
8888dd37758STrond Myklebust {
889fe238e60SDave Wysochanski 	spin_lock(&cinfo->inode->i_lock);
8906272dcc6SAnna Schumaker 	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
891fe238e60SDave Wysochanski 	spin_unlock(&cinfo->inode->i_lock);
89267911c8fSAnna Schumaker 	if (req->wb_page)
89386d80f97STrond Myklebust 		nfs_mark_page_unstable(req->wb_page, cinfo);
8948dd37758STrond Myklebust }
8958dd37758STrond Myklebust EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
8968dd37758STrond Myklebust 
8978dd37758STrond Myklebust /**
8988dd37758STrond Myklebust  * nfs_request_remove_commit_list - Remove request from a commit list
8998dd37758STrond Myklebust  * @req: pointer to a nfs_page
900ea2cf228SFred Isaman  * @cinfo: holds list lock and accounting info
9018dd37758STrond Myklebust  *
902ea2cf228SFred Isaman  * This clears the PG_CLEAN bit, and updates the cinfo's count of
9038dd37758STrond Myklebust  * number of outstanding requests requiring a commit
9048dd37758STrond Myklebust  * It does not update the MM page stats.
9058dd37758STrond Myklebust  *
906ea2cf228SFred Isaman  * The caller _must_ hold the cinfo->lock and the nfs_page lock.
9078dd37758STrond Myklebust  */
9088dd37758STrond Myklebust void
909ea2cf228SFred Isaman nfs_request_remove_commit_list(struct nfs_page *req,
910ea2cf228SFred Isaman 			       struct nfs_commit_info *cinfo)
9118dd37758STrond Myklebust {
9128dd37758STrond Myklebust 	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
9138dd37758STrond Myklebust 		return;
9148dd37758STrond Myklebust 	nfs_list_remove_request(req);
915ea2cf228SFred Isaman 	cinfo->mds->ncommit--;
9168dd37758STrond Myklebust }
9178dd37758STrond Myklebust EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
9188dd37758STrond Myklebust 
919ea2cf228SFred Isaman static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
920ea2cf228SFred Isaman 				      struct inode *inode)
921ea2cf228SFred Isaman {
922fe238e60SDave Wysochanski 	cinfo->inode = inode;
923ea2cf228SFred Isaman 	cinfo->mds = &NFS_I(inode)->commit_info;
924ea2cf228SFred Isaman 	cinfo->ds = pnfs_get_ds_info(inode);
925b359f9d0SFred Isaman 	cinfo->dreq = NULL;
926f453a54aSFred Isaman 	cinfo->completion_ops = &nfs_commit_completion_ops;
927ea2cf228SFred Isaman }
928ea2cf228SFred Isaman 
929ea2cf228SFred Isaman void nfs_init_cinfo(struct nfs_commit_info *cinfo,
930ea2cf228SFred Isaman 		    struct inode *inode,
931ea2cf228SFred Isaman 		    struct nfs_direct_req *dreq)
932ea2cf228SFred Isaman {
9331763da12SFred Isaman 	if (dreq)
9341763da12SFred Isaman 		nfs_init_cinfo_from_dreq(cinfo, dreq);
9351763da12SFred Isaman 	else
936ea2cf228SFred Isaman 		nfs_init_cinfo_from_inode(cinfo, inode);
937ea2cf228SFred Isaman }
938ea2cf228SFred Isaman EXPORT_SYMBOL_GPL(nfs_init_cinfo);
9398dd37758STrond Myklebust 
9401da177e4SLinus Torvalds /*
9411da177e4SLinus Torvalds  * Add a request to the inode's commit list.
9421da177e4SLinus Torvalds  */
9431763da12SFred Isaman void
944ea2cf228SFred Isaman nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
945b57ff130SWeston Andros Adamson 			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
9461da177e4SLinus Torvalds {
947b57ff130SWeston Andros Adamson 	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
9488dd37758STrond Myklebust 		return;
9496272dcc6SAnna Schumaker 	nfs_request_add_commit_list(req, cinfo);
9501da177e4SLinus Torvalds }
9518e821cadSTrond Myklebust 
952d6d6dc7cSFred Isaman static void
953d6d6dc7cSFred Isaman nfs_clear_page_commit(struct page *page)
954e468bae9STrond Myklebust {
95511fb9989SMel Gorman 	dec_node_page_state(page, NR_UNSTABLE_NFS);
95693f78d88STejun Heo 	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
95793f78d88STejun Heo 		    WB_RECLAIMABLE);
958e468bae9STrond Myklebust }
959d6d6dc7cSFred Isaman 
960411a99adSWeston Andros Adamson /* Called holding inode (/cinfo) lock */
9618dd37758STrond Myklebust static void
962d6d6dc7cSFred Isaman nfs_clear_request_commit(struct nfs_page *req)
963d6d6dc7cSFred Isaman {
9648dd37758STrond Myklebust 	if (test_bit(PG_CLEAN, &req->wb_flags)) {
9652b0143b5SDavid Howells 		struct inode *inode = d_inode(req->wb_context->dentry);
966ea2cf228SFred Isaman 		struct nfs_commit_info cinfo;
967d6d6dc7cSFred Isaman 
968ea2cf228SFred Isaman 		nfs_init_cinfo_from_inode(&cinfo, inode);
969ea2cf228SFred Isaman 		if (!pnfs_clear_request_commit(req, &cinfo)) {
970ea2cf228SFred Isaman 			nfs_request_remove_commit_list(req, &cinfo);
971d6d6dc7cSFred Isaman 		}
9728dd37758STrond Myklebust 		nfs_clear_page_commit(req->wb_page);
9738dd37758STrond Myklebust 	}
974e468bae9STrond Myklebust }
975e468bae9STrond Myklebust 
976d45f60c6SWeston Andros Adamson int nfs_write_need_commit(struct nfs_pgio_header *hdr)
9778e821cadSTrond Myklebust {
978c65e6254SWeston Andros Adamson 	if (hdr->verf.committed == NFS_DATA_SYNC)
979d45f60c6SWeston Andros Adamson 		return hdr->lseg == NULL;
980c65e6254SWeston Andros Adamson 	return hdr->verf.committed != NFS_FILE_SYNC;
9818e821cadSTrond Myklebust }
9828e821cadSTrond Myklebust 
983919e3bd9STrond Myklebust static void nfs_async_write_init(struct nfs_pgio_header *hdr)
984919e3bd9STrond Myklebust {
985919e3bd9STrond Myklebust 	nfs_io_completion_get(hdr->io_completion);
986919e3bd9STrond Myklebust }
987919e3bd9STrond Myklebust 
988061ae2edSFred Isaman static void nfs_write_completion(struct nfs_pgio_header *hdr)
9896c75dc0dSFred Isaman {
990ea2cf228SFred Isaman 	struct nfs_commit_info cinfo;
9916c75dc0dSFred Isaman 	unsigned long bytes = 0;
9926c75dc0dSFred Isaman 
9936c75dc0dSFred Isaman 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
9946c75dc0dSFred Isaman 		goto out;
995ea2cf228SFred Isaman 	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
9966c75dc0dSFred Isaman 	while (!list_empty(&hdr->pages)) {
9976c75dc0dSFred Isaman 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
9986c75dc0dSFred Isaman 
9996c75dc0dSFred Isaman 		bytes += req->wb_bytes;
10006c75dc0dSFred Isaman 		nfs_list_remove_request(req);
10016c75dc0dSFred Isaman 		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
10026c75dc0dSFred Isaman 		    (hdr->good_bytes < bytes)) {
1003d1182b33STrond Myklebust 			nfs_set_pageerror(req->wb_page);
10046c75dc0dSFred Isaman 			nfs_context_set_write_error(req->wb_context, hdr->error);
10056c75dc0dSFred Isaman 			goto remove_req;
10066c75dc0dSFred Isaman 		}
1007c65e6254SWeston Andros Adamson 		if (nfs_write_need_commit(hdr)) {
1008f79d06f5SAnna Schumaker 			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1009b57ff130SWeston Andros Adamson 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1010a7d42ddbSWeston Andros Adamson 				hdr->pgio_mirror_idx);
10116c75dc0dSFred Isaman 			goto next;
10126c75dc0dSFred Isaman 		}
10136c75dc0dSFred Isaman remove_req:
10146c75dc0dSFred Isaman 		nfs_inode_remove_request(req);
10156c75dc0dSFred Isaman next:
10161d1afcbcSTrond Myklebust 		nfs_unlock_request(req);
101720633f04SWeston Andros Adamson 		nfs_end_page_writeback(req);
10183aff4ebbSTrond Myklebust 		nfs_release_request(req);
10196c75dc0dSFred Isaman 	}
10206c75dc0dSFred Isaman out:
1021919e3bd9STrond Myklebust 	nfs_io_completion_put(hdr->io_completion);
10226c75dc0dSFred Isaman 	hdr->release(hdr);
10236c75dc0dSFred Isaman }
10246c75dc0dSFred Isaman 
1025ce59515cSAnna Schumaker unsigned long
1026ea2cf228SFred Isaman nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1027fb8a1f11STrond Myklebust {
1028ea2cf228SFred Isaman 	return cinfo->mds->ncommit;
1029fb8a1f11STrond Myklebust }
1030fb8a1f11STrond Myklebust 
1031fe238e60SDave Wysochanski /* cinfo->inode->i_lock held by caller */
10321763da12SFred Isaman int
1033ea2cf228SFred Isaman nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
1034ea2cf228SFred Isaman 		     struct nfs_commit_info *cinfo, int max)
1035d6d6dc7cSFred Isaman {
1036d6d6dc7cSFred Isaman 	struct nfs_page *req, *tmp;
1037d6d6dc7cSFred Isaman 	int ret = 0;
1038d6d6dc7cSFred Isaman 
1039d6d6dc7cSFred Isaman 	list_for_each_entry_safe(req, tmp, src, wb_list) {
10408dd37758STrond Myklebust 		if (!nfs_lock_request(req))
10418dd37758STrond Myklebust 			continue;
10427ad84aa9STrond Myklebust 		kref_get(&req->wb_kref);
1043fe238e60SDave Wysochanski 		if (cond_resched_lock(&cinfo->inode->i_lock))
10443b3be88dSTrond Myklebust 			list_safe_reset_next(req, tmp, wb_list);
1045ea2cf228SFred Isaman 		nfs_request_remove_commit_list(req, cinfo);
10468dd37758STrond Myklebust 		nfs_list_add_request(req, dst);
1047d6d6dc7cSFred Isaman 		ret++;
10481763da12SFred Isaman 		if ((ret == max) && !cinfo->dreq)
1049d6d6dc7cSFred Isaman 			break;
1050d6d6dc7cSFred Isaman 	}
1051d6d6dc7cSFred Isaman 	return ret;
1052d6d6dc7cSFred Isaman }
1053d6d6dc7cSFred Isaman 
10541da177e4SLinus Torvalds /*
10551da177e4SLinus Torvalds  * nfs_scan_commit - Scan an inode for commit requests
10561da177e4SLinus Torvalds  * @inode: NFS inode to scan
1057ea2cf228SFred Isaman  * @dst: mds destination list
1058ea2cf228SFred Isaman  * @cinfo: mds and ds lists of reqs ready to commit
10591da177e4SLinus Torvalds  *
10601da177e4SLinus Torvalds  * Moves requests from the inode's 'commit' request list.
10611da177e4SLinus Torvalds  * The requests are *not* checked to ensure that they form a contiguous set.
10621da177e4SLinus Torvalds  */
10631763da12SFred Isaman int
1064ea2cf228SFred Isaman nfs_scan_commit(struct inode *inode, struct list_head *dst,
1065ea2cf228SFred Isaman 		struct nfs_commit_info *cinfo)
10661da177e4SLinus Torvalds {
1067d6d6dc7cSFred Isaman 	int ret = 0;
1068fb8a1f11STrond Myklebust 
1069fe238e60SDave Wysochanski 	spin_lock(&cinfo->inode->i_lock);
1070ea2cf228SFred Isaman 	if (cinfo->mds->ncommit > 0) {
10718dd37758STrond Myklebust 		const int max = INT_MAX;
1072d6d6dc7cSFred Isaman 
1073ea2cf228SFred Isaman 		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
1074ea2cf228SFred Isaman 					   cinfo, max);
1075ea2cf228SFred Isaman 		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1076d6d6dc7cSFred Isaman 	}
1077fe238e60SDave Wysochanski 	spin_unlock(&cinfo->inode->i_lock);
1078ff778d02STrond Myklebust 	return ret;
10791da177e4SLinus Torvalds }
1080d6d6dc7cSFred Isaman 
10811da177e4SLinus Torvalds /*
1082e7d39069STrond Myklebust  * Search for an existing write request, and attempt to update
1083e7d39069STrond Myklebust  * it to reflect a new dirty region on a given page.
10841da177e4SLinus Torvalds  *
1085e7d39069STrond Myklebust  * If the attempt fails, then the existing request is flushed out
1086e7d39069STrond Myklebust  * to disk.
10871da177e4SLinus Torvalds  */
1088e7d39069STrond Myklebust static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
1089e7d39069STrond Myklebust 		struct page *page,
1090e7d39069STrond Myklebust 		unsigned int offset,
1091e7d39069STrond Myklebust 		unsigned int bytes)
10921da177e4SLinus Torvalds {
1093e7d39069STrond Myklebust 	struct nfs_page *req;
1094e7d39069STrond Myklebust 	unsigned int rqend;
1095e7d39069STrond Myklebust 	unsigned int end;
10961da177e4SLinus Torvalds 	int error;
1097277459d2STrond Myklebust 
1098e7d39069STrond Myklebust 	if (!PagePrivate(page))
1099e7d39069STrond Myklebust 		return NULL;
1100e7d39069STrond Myklebust 
1101e7d39069STrond Myklebust 	end = offset + bytes;
1102e7d39069STrond Myklebust 	spin_lock(&inode->i_lock);
1103e7d39069STrond Myklebust 
1104e7d39069STrond Myklebust 	for (;;) {
110584d3a9a9SWeston Andros Adamson 		req = nfs_page_find_head_request_locked(NFS_I(inode), page);
1106e7d39069STrond Myklebust 		if (req == NULL)
1107e7d39069STrond Myklebust 			goto out_unlock;
1108e7d39069STrond Myklebust 
11092bfc6e56SWeston Andros Adamson 		/* should be handled by nfs_flush_incompatible */
11102bfc6e56SWeston Andros Adamson 		WARN_ON_ONCE(req->wb_head != req);
11112bfc6e56SWeston Andros Adamson 		WARN_ON_ONCE(req->wb_this_page != req);
11122bfc6e56SWeston Andros Adamson 
1113e7d39069STrond Myklebust 		rqend = req->wb_offset + req->wb_bytes;
1114e7d39069STrond Myklebust 		/*
1115e7d39069STrond Myklebust 		 * Tell the caller to flush out the request if
1116e7d39069STrond Myklebust 		 * the offsets are non-contiguous.
1117e7d39069STrond Myklebust 		 * Note: nfs_flush_incompatible() will already
1118e7d39069STrond Myklebust 		 * have flushed out requests having wrong owners.
1119e7d39069STrond Myklebust 		 */
1120e468bae9STrond Myklebust 		if (offset > rqend
1121e7d39069STrond Myklebust 		    || end < req->wb_offset)
1122e7d39069STrond Myklebust 			goto out_flushme;
1123e7d39069STrond Myklebust 
11247ad84aa9STrond Myklebust 		if (nfs_lock_request(req))
1125e7d39069STrond Myklebust 			break;
1126e7d39069STrond Myklebust 
1127e7d39069STrond Myklebust 		/* The request is locked, so wait and then retry */
1128587142f8STrond Myklebust 		spin_unlock(&inode->i_lock);
11291da177e4SLinus Torvalds 		error = nfs_wait_on_request(req);
11301da177e4SLinus Torvalds 		nfs_release_request(req);
1131e7d39069STrond Myklebust 		if (error != 0)
1132e7d39069STrond Myklebust 			goto out_err;
1133e7d39069STrond Myklebust 		spin_lock(&inode->i_lock);
11341da177e4SLinus Torvalds 	}
11351da177e4SLinus Torvalds 
11361da177e4SLinus Torvalds 	/* Okay, the request matches. Update the region */
11371da177e4SLinus Torvalds 	if (offset < req->wb_offset) {
11381da177e4SLinus Torvalds 		req->wb_offset = offset;
11391da177e4SLinus Torvalds 		req->wb_pgbase = offset;
11401da177e4SLinus Torvalds 	}
11411da177e4SLinus Torvalds 	if (end > rqend)
11421da177e4SLinus Torvalds 		req->wb_bytes = end - req->wb_offset;
1143e7d39069STrond Myklebust 	else
1144e7d39069STrond Myklebust 		req->wb_bytes = rqend - req->wb_offset;
1145e7d39069STrond Myklebust out_unlock:
1146ca138f36SFred Isaman 	if (req)
11478dd37758STrond Myklebust 		nfs_clear_request_commit(req);
1148411a99adSWeston Andros Adamson 	spin_unlock(&inode->i_lock);
1149e7d39069STrond Myklebust 	return req;
1150e7d39069STrond Myklebust out_flushme:
1151e7d39069STrond Myklebust 	spin_unlock(&inode->i_lock);
1152e7d39069STrond Myklebust 	nfs_release_request(req);
1153e7d39069STrond Myklebust 	error = nfs_wb_page(inode, page);
1154e7d39069STrond Myklebust out_err:
1155e7d39069STrond Myklebust 	return ERR_PTR(error);
1156e7d39069STrond Myklebust }
11571da177e4SLinus Torvalds 
1158e7d39069STrond Myklebust /*
1159e7d39069STrond Myklebust  * Try to update an existing write request, or create one if there is none.
1160e7d39069STrond Myklebust  *
1161e7d39069STrond Myklebust  * Note: Should always be called with the Page Lock held to prevent races
1162e7d39069STrond Myklebust  * if we have to add a new request. Also assumes that the caller has
1163e7d39069STrond Myklebust  * already called nfs_flush_incompatible() if necessary.
1164e7d39069STrond Myklebust  */
1165e7d39069STrond Myklebust static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
1166e7d39069STrond Myklebust 		struct page *page, unsigned int offset, unsigned int bytes)
1167e7d39069STrond Myklebust {
1168d56b4ddfSMel Gorman 	struct inode *inode = page_file_mapping(page)->host;
1169e7d39069STrond Myklebust 	struct nfs_page	*req;
1170e7d39069STrond Myklebust 
1171e7d39069STrond Myklebust 	req = nfs_try_to_update_request(inode, page, offset, bytes);
1172e7d39069STrond Myklebust 	if (req != NULL)
1173e7d39069STrond Myklebust 		goto out;
11742bfc6e56SWeston Andros Adamson 	req = nfs_create_request(ctx, page, NULL, offset, bytes);
1175e7d39069STrond Myklebust 	if (IS_ERR(req))
1176e7d39069STrond Myklebust 		goto out;
1177d6d6dc7cSFred Isaman 	nfs_inode_add_request(inode, req);
1178efc91ed0STrond Myklebust out:
117961e930a9STrond Myklebust 	return req;
11801da177e4SLinus Torvalds }
11811da177e4SLinus Torvalds 
1182e7d39069STrond Myklebust static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
1183e7d39069STrond Myklebust 		unsigned int offset, unsigned int count)
1184e7d39069STrond Myklebust {
1185e7d39069STrond Myklebust 	struct nfs_page	*req;
1186e7d39069STrond Myklebust 
1187e7d39069STrond Myklebust 	req = nfs_setup_write_request(ctx, page, offset, count);
1188e7d39069STrond Myklebust 	if (IS_ERR(req))
1189e7d39069STrond Myklebust 		return PTR_ERR(req);
1190e7d39069STrond Myklebust 	/* Update file length */
1191e7d39069STrond Myklebust 	nfs_grow_file(page, offset, count);
1192d72ddcbaSWeston Andros Adamson 	nfs_mark_uptodate(req);
1193a6305ddbSTrond Myklebust 	nfs_mark_request_dirty(req);
11941d1afcbcSTrond Myklebust 	nfs_unlock_and_release_request(req);
1195e7d39069STrond Myklebust 	return 0;
1196e7d39069STrond Myklebust }
1197e7d39069STrond Myklebust 
11981da177e4SLinus Torvalds int nfs_flush_incompatible(struct file *file, struct page *page)
11991da177e4SLinus Torvalds {
1200cd3758e3STrond Myklebust 	struct nfs_open_context *ctx = nfs_file_open_context(file);
12012a369153STrond Myklebust 	struct nfs_lock_context *l_ctx;
1202bd61e0a9SJeff Layton 	struct file_lock_context *flctx = file_inode(file)->i_flctx;
12031da177e4SLinus Torvalds 	struct nfs_page	*req;
12041a54533eSTrond Myklebust 	int do_flush, status;
12051da177e4SLinus Torvalds 	/*
12061da177e4SLinus Torvalds 	 * Look for a request corresponding to this page. If there
12071da177e4SLinus Torvalds 	 * is one, and it belongs to another file, we flush it out
12081da177e4SLinus Torvalds 	 * before we try to copy anything into the page. Do this
12091da177e4SLinus Torvalds 	 * due to the lack of an ACCESS-type call in NFSv2.
12101da177e4SLinus Torvalds 	 * Also do the same if we find a request from an existing
12111da177e4SLinus Torvalds 	 * dropped page.
12121da177e4SLinus Torvalds 	 */
12131a54533eSTrond Myklebust 	do {
121484d3a9a9SWeston Andros Adamson 		req = nfs_page_find_head_request(page);
12151a54533eSTrond Myklebust 		if (req == NULL)
12161a54533eSTrond Myklebust 			return 0;
12172a369153STrond Myklebust 		l_ctx = req->wb_lock_context;
1218138a2935STrond Myklebust 		do_flush = req->wb_page != page ||
1219138a2935STrond Myklebust 			!nfs_match_open_context(req->wb_context, ctx);
12202bfc6e56SWeston Andros Adamson 		/* for now, flush if more than 1 request in page_group */
12212bfc6e56SWeston Andros Adamson 		do_flush |= req->wb_this_page != req;
1222bd61e0a9SJeff Layton 		if (l_ctx && flctx &&
1223bd61e0a9SJeff Layton 		    !(list_empty_careful(&flctx->flc_posix) &&
1224bd61e0a9SJeff Layton 		      list_empty_careful(&flctx->flc_flock))) {
1225d51fdb87SNeilBrown 			do_flush |= l_ctx->lockowner != current->files;
12265263e31eSJeff Layton 		}
12271da177e4SLinus Torvalds 		nfs_release_request(req);
12281a54533eSTrond Myklebust 		if (!do_flush)
12291a54533eSTrond Myklebust 			return 0;
1230d56b4ddfSMel Gorman 		status = nfs_wb_page(page_file_mapping(page)->host, page);
12311a54533eSTrond Myklebust 	} while (status == 0);
12321a54533eSTrond Myklebust 	return status;
12331da177e4SLinus Torvalds }
12341da177e4SLinus Torvalds 
12351da177e4SLinus Torvalds /*
1236dc24826bSAndy Adamson  * Avoid buffered writes when a open context credential's key would
1237dc24826bSAndy Adamson  * expire soon.
1238dc24826bSAndy Adamson  *
1239dc24826bSAndy Adamson  * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
1240dc24826bSAndy Adamson  *
1241dc24826bSAndy Adamson  * Return 0 and set a credential flag which triggers the inode to flush
1242dc24826bSAndy Adamson  * and performs  NFS_FILE_SYNC writes if the key will expired within
1243dc24826bSAndy Adamson  * RPC_KEY_EXPIRE_TIMEO.
1244dc24826bSAndy Adamson  */
1245dc24826bSAndy Adamson int
1246dc24826bSAndy Adamson nfs_key_timeout_notify(struct file *filp, struct inode *inode)
1247dc24826bSAndy Adamson {
1248dc24826bSAndy Adamson 	struct nfs_open_context *ctx = nfs_file_open_context(filp);
1249dc24826bSAndy Adamson 	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1250dc24826bSAndy Adamson 
1251dc24826bSAndy Adamson 	return rpcauth_key_timeout_notify(auth, ctx->cred);
1252dc24826bSAndy Adamson }
1253dc24826bSAndy Adamson 
1254dc24826bSAndy Adamson /*
1255dc24826bSAndy Adamson  * Test if the open context credential key is marked to expire soon.
1256dc24826bSAndy Adamson  */
1257ce52914eSScott Mayhew bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1258dc24826bSAndy Adamson {
1259ce52914eSScott Mayhew 	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1260ce52914eSScott Mayhew 
1261ce52914eSScott Mayhew 	return rpcauth_cred_key_to_expire(auth, ctx->cred);
1262dc24826bSAndy Adamson }
1263dc24826bSAndy Adamson 
1264dc24826bSAndy Adamson /*
12655d47a356STrond Myklebust  * If the page cache is marked as unsafe or invalid, then we can't rely on
12665d47a356STrond Myklebust  * the PageUptodate() flag. In this case, we will need to turn off
12675d47a356STrond Myklebust  * write optimisations that depend on the page contents being correct.
12685d47a356STrond Myklebust  */
12698d197a56STrond Myklebust static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
12705d47a356STrond Myklebust {
1271d529ef83SJeff Layton 	struct nfs_inode *nfsi = NFS_I(inode);
1272d529ef83SJeff Layton 
12738d197a56STrond Myklebust 	if (nfs_have_delegated_attributes(inode))
12748d197a56STrond Myklebust 		goto out;
127518dd78c4SScott Mayhew 	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1276d529ef83SJeff Layton 		return false;
12774db72b40SJeff Layton 	smp_rmb();
1278d529ef83SJeff Layton 	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
12798d197a56STrond Myklebust 		return false;
12808d197a56STrond Myklebust out:
128118dd78c4SScott Mayhew 	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
128218dd78c4SScott Mayhew 		return false;
12838d197a56STrond Myklebust 	return PageUptodate(page) != 0;
12845d47a356STrond Myklebust }
12855d47a356STrond Myklebust 
12865263e31eSJeff Layton static bool
12875263e31eSJeff Layton is_whole_file_wrlock(struct file_lock *fl)
12885263e31eSJeff Layton {
12895263e31eSJeff Layton 	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
12905263e31eSJeff Layton 			fl->fl_type == F_WRLCK;
12915263e31eSJeff Layton }
12925263e31eSJeff Layton 
1293c7559663SScott Mayhew /* If we know the page is up to date, and we're not using byte range locks (or
1294c7559663SScott Mayhew  * if we have the whole file locked for writing), it may be more efficient to
1295c7559663SScott Mayhew  * extend the write to cover the entire page in order to avoid fragmentation
1296c7559663SScott Mayhew  * inefficiencies.
1297c7559663SScott Mayhew  *
1298263b4509SScott Mayhew  * If the file is opened for synchronous writes then we can just skip the rest
1299263b4509SScott Mayhew  * of the checks.
1300c7559663SScott Mayhew  */
1301c7559663SScott Mayhew static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
1302c7559663SScott Mayhew {
13035263e31eSJeff Layton 	int ret;
13045263e31eSJeff Layton 	struct file_lock_context *flctx = inode->i_flctx;
13055263e31eSJeff Layton 	struct file_lock *fl;
13065263e31eSJeff Layton 
1307c7559663SScott Mayhew 	if (file->f_flags & O_DSYNC)
1308c7559663SScott Mayhew 		return 0;
1309263b4509SScott Mayhew 	if (!nfs_write_pageuptodate(page, inode))
1310263b4509SScott Mayhew 		return 0;
1311c7559663SScott Mayhew 	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
1312c7559663SScott Mayhew 		return 1;
1313bd61e0a9SJeff Layton 	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
1314bd61e0a9SJeff Layton 		       list_empty_careful(&flctx->flc_posix)))
13158fa4592aSTrond Myklebust 		return 1;
13165263e31eSJeff Layton 
13175263e31eSJeff Layton 	/* Check to see if there are whole file write locks */
13185263e31eSJeff Layton 	ret = 0;
13196109c850SJeff Layton 	spin_lock(&flctx->flc_lock);
1320bd61e0a9SJeff Layton 	if (!list_empty(&flctx->flc_posix)) {
1321bd61e0a9SJeff Layton 		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
1322bd61e0a9SJeff Layton 					fl_list);
1323bd61e0a9SJeff Layton 		if (is_whole_file_wrlock(fl))
13245263e31eSJeff Layton 			ret = 1;
1325bd61e0a9SJeff Layton 	} else if (!list_empty(&flctx->flc_flock)) {
13265263e31eSJeff Layton 		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
13275263e31eSJeff Layton 					fl_list);
13285263e31eSJeff Layton 		if (fl->fl_type == F_WRLCK)
13295263e31eSJeff Layton 			ret = 1;
13305263e31eSJeff Layton 	}
13316109c850SJeff Layton 	spin_unlock(&flctx->flc_lock);
13325263e31eSJeff Layton 	return ret;
1333c7559663SScott Mayhew }
1334c7559663SScott Mayhew 
13355d47a356STrond Myklebust /*
13361da177e4SLinus Torvalds  * Update and possibly write a cached page of an NFS file.
13371da177e4SLinus Torvalds  *
13381da177e4SLinus Torvalds  * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
13391da177e4SLinus Torvalds  * things with a page scheduled for an RPC call (e.g. invalidate it).
13401da177e4SLinus Torvalds  */
13411da177e4SLinus Torvalds int nfs_updatepage(struct file *file, struct page *page,
13421da177e4SLinus Torvalds 		unsigned int offset, unsigned int count)
13431da177e4SLinus Torvalds {
1344cd3758e3STrond Myklebust 	struct nfs_open_context *ctx = nfs_file_open_context(file);
1345d56b4ddfSMel Gorman 	struct inode	*inode = page_file_mapping(page)->host;
13461da177e4SLinus Torvalds 	int		status = 0;
13471da177e4SLinus Torvalds 
134891d5b470SChuck Lever 	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
134991d5b470SChuck Lever 
13506de1472fSAl Viro 	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
13516de1472fSAl Viro 		file, count, (long long)(page_file_offset(page) + offset));
13521da177e4SLinus Torvalds 
1353149a4fddSBenjamin Coddington 	if (!count)
1354149a4fddSBenjamin Coddington 		goto out;
1355149a4fddSBenjamin Coddington 
1356c7559663SScott Mayhew 	if (nfs_can_extend_write(file, page, inode)) {
135749a70f27STrond Myklebust 		count = max(count + offset, nfs_page_length(page));
13581da177e4SLinus Torvalds 		offset = 0;
13591da177e4SLinus Torvalds 	}
13601da177e4SLinus Torvalds 
1361e21195a7STrond Myklebust 	status = nfs_writepage_setup(ctx, page, offset, count);
136203fa9e84STrond Myklebust 	if (status < 0)
136303fa9e84STrond Myklebust 		nfs_set_pageerror(page);
136459b7c05fSTrond Myklebust 	else
136559b7c05fSTrond Myklebust 		__set_page_dirty_nobuffers(page);
1366149a4fddSBenjamin Coddington out:
136748186c7dSChuck Lever 	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
13681da177e4SLinus Torvalds 			status, (long long)i_size_read(inode));
13691da177e4SLinus Torvalds 	return status;
13701da177e4SLinus Torvalds }
13711da177e4SLinus Torvalds 
13723ff7576dSTrond Myklebust static int flush_task_priority(int how)
13731da177e4SLinus Torvalds {
13741da177e4SLinus Torvalds 	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
13751da177e4SLinus Torvalds 		case FLUSH_HIGHPRI:
13761da177e4SLinus Torvalds 			return RPC_PRIORITY_HIGH;
13771da177e4SLinus Torvalds 		case FLUSH_LOWPRI:
13781da177e4SLinus Torvalds 			return RPC_PRIORITY_LOW;
13791da177e4SLinus Torvalds 	}
13801da177e4SLinus Torvalds 	return RPC_PRIORITY_NORMAL;
13811da177e4SLinus Torvalds }
13821da177e4SLinus Torvalds 
1383d45f60c6SWeston Andros Adamson static void nfs_initiate_write(struct nfs_pgio_header *hdr,
1384d45f60c6SWeston Andros Adamson 			       struct rpc_message *msg,
1385abde71f4STom Haynes 			       const struct nfs_rpc_ops *rpc_ops,
13861ed26f33SAnna Schumaker 			       struct rpc_task_setup *task_setup_data, int how)
13871da177e4SLinus Torvalds {
13883ff7576dSTrond Myklebust 	int priority = flush_task_priority(how);
13891da177e4SLinus Torvalds 
13901ed26f33SAnna Schumaker 	task_setup_data->priority = priority;
1391abde71f4STom Haynes 	rpc_ops->write_setup(hdr, msg);
1392d138d5d1SAndy Adamson 
1393abde71f4STom Haynes 	nfs4_state_protect_write(NFS_SERVER(hdr->inode)->nfs_client,
1394d45f60c6SWeston Andros Adamson 				 &task_setup_data->rpc_client, msg, hdr);
1395275acaafSTrond Myklebust }
1396275acaafSTrond Myklebust 
13976d884e8fSFred /* If a nfs_flush_* function fails, it should remove reqs from @head and
13986d884e8fSFred  * call this on each, which will prepare them to be retried on next
13996d884e8fSFred  * writeback using standard nfs.
14006d884e8fSFred  */
14016d884e8fSFred static void nfs_redirty_request(struct nfs_page *req)
14026d884e8fSFred {
14036d884e8fSFred 	nfs_mark_request_dirty(req);
1404c7070113STrond Myklebust 	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
14051d1afcbcSTrond Myklebust 	nfs_unlock_request(req);
140620633f04SWeston Andros Adamson 	nfs_end_page_writeback(req);
14073aff4ebbSTrond Myklebust 	nfs_release_request(req);
14086d884e8fSFred }
14096d884e8fSFred 
1410061ae2edSFred Isaman static void nfs_async_write_error(struct list_head *head)
14116c75dc0dSFred Isaman {
14126c75dc0dSFred Isaman 	struct nfs_page	*req;
14136c75dc0dSFred Isaman 
14146c75dc0dSFred Isaman 	while (!list_empty(head)) {
14156c75dc0dSFred Isaman 		req = nfs_list_entry(head->next);
14166c75dc0dSFred Isaman 		nfs_list_remove_request(req);
14176c75dc0dSFred Isaman 		nfs_redirty_request(req);
14186c75dc0dSFred Isaman 	}
14196c75dc0dSFred Isaman }
14206c75dc0dSFred Isaman 
1421dc602dd7STrond Myklebust static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
1422dc602dd7STrond Myklebust {
1423dc602dd7STrond Myklebust 	nfs_async_write_error(&hdr->pages);
1424dc602dd7STrond Myklebust }
1425dc602dd7STrond Myklebust 
1426061ae2edSFred Isaman static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1427919e3bd9STrond Myklebust 	.init_hdr = nfs_async_write_init,
1428061ae2edSFred Isaman 	.error_cleanup = nfs_async_write_error,
1429061ae2edSFred Isaman 	.completion = nfs_write_completion,
1430dc602dd7STrond Myklebust 	.reschedule_io = nfs_async_write_reschedule_io,
1431061ae2edSFred Isaman };
1432061ae2edSFred Isaman 
143357208fa7SBryan Schumaker void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1434a20c93e3SChristoph Hellwig 			       struct inode *inode, int ioflags, bool force_mds,
1435061ae2edSFred Isaman 			       const struct nfs_pgio_completion_ops *compl_ops)
14361751c363STrond Myklebust {
1437a20c93e3SChristoph Hellwig 	struct nfs_server *server = NFS_SERVER(inode);
143841d8d5b7SAnna Schumaker 	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1439a20c93e3SChristoph Hellwig 
1440a20c93e3SChristoph Hellwig #ifdef CONFIG_NFS_V4_1
1441a20c93e3SChristoph Hellwig 	if (server->pnfs_curr_ld && !force_mds)
1442a20c93e3SChristoph Hellwig 		pg_ops = server->pnfs_curr_ld->pg_write_ops;
1443a20c93e3SChristoph Hellwig #endif
14444a0de55cSAnna Schumaker 	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1445fbe77c30SBenjamin Coddington 			server->wsize, ioflags, GFP_NOIO);
14461751c363STrond Myklebust }
1447ddda8e0aSBryan Schumaker EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
14481751c363STrond Myklebust 
1449dce81290STrond Myklebust void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1450dce81290STrond Myklebust {
1451a7d42ddbSWeston Andros Adamson 	struct nfs_pgio_mirror *mirror;
1452a7d42ddbSWeston Andros Adamson 
14536f29b9bbSKinglong Mee 	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
14546f29b9bbSKinglong Mee 		pgio->pg_ops->pg_cleanup(pgio);
14556f29b9bbSKinglong Mee 
145641d8d5b7SAnna Schumaker 	pgio->pg_ops = &nfs_pgio_rw_ops;
1457a7d42ddbSWeston Andros Adamson 
1458a7d42ddbSWeston Andros Adamson 	nfs_pageio_stop_mirroring(pgio);
1459a7d42ddbSWeston Andros Adamson 
1460a7d42ddbSWeston Andros Adamson 	mirror = &pgio->pg_mirrors[0];
1461a7d42ddbSWeston Andros Adamson 	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1462dce81290STrond Myklebust }
14631f945357STrond Myklebust EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1464dce81290STrond Myklebust 
14651da177e4SLinus Torvalds 
14660b7c0153SFred Isaman void nfs_commit_prepare(struct rpc_task *task, void *calldata)
14670b7c0153SFred Isaman {
14680b7c0153SFred Isaman 	struct nfs_commit_data *data = calldata;
14690b7c0153SFred Isaman 
14700b7c0153SFred Isaman 	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
14710b7c0153SFred Isaman }
14720b7c0153SFred Isaman 
14731f2edbe3STrond Myklebust /*
14741f2edbe3STrond Myklebust  * Special version of should_remove_suid() that ignores capabilities.
14751f2edbe3STrond Myklebust  */
14761f2edbe3STrond Myklebust static int nfs_should_remove_suid(const struct inode *inode)
14771f2edbe3STrond Myklebust {
14781f2edbe3STrond Myklebust 	umode_t mode = inode->i_mode;
14791f2edbe3STrond Myklebust 	int kill = 0;
1480788e7a89STrond Myklebust 
14811f2edbe3STrond Myklebust 	/* suid always must be killed */
14821f2edbe3STrond Myklebust 	if (unlikely(mode & S_ISUID))
14831f2edbe3STrond Myklebust 		kill = ATTR_KILL_SUID;
14841f2edbe3STrond Myklebust 
14851f2edbe3STrond Myklebust 	/*
14861f2edbe3STrond Myklebust 	 * sgid without any exec bits is just a mandatory locking mark; leave
14871f2edbe3STrond Myklebust 	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
14881f2edbe3STrond Myklebust 	 */
14891f2edbe3STrond Myklebust 	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
14901f2edbe3STrond Myklebust 		kill |= ATTR_KILL_SGID;
14911f2edbe3STrond Myklebust 
14921f2edbe3STrond Myklebust 	if (unlikely(kill && S_ISREG(mode)))
14931f2edbe3STrond Myklebust 		return kill;
14941f2edbe3STrond Myklebust 
14951f2edbe3STrond Myklebust 	return 0;
14961f2edbe3STrond Myklebust }
1497788e7a89STrond Myklebust 
1498a08a8cd3STrond Myklebust static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
1499a08a8cd3STrond Myklebust 		struct nfs_fattr *fattr)
1500a08a8cd3STrond Myklebust {
1501a08a8cd3STrond Myklebust 	struct nfs_pgio_args *argp = &hdr->args;
1502a08a8cd3STrond Myklebust 	struct nfs_pgio_res *resp = &hdr->res;
15032b83d3deSTrond Myklebust 	u64 size = argp->offset + resp->count;
1504a08a8cd3STrond Myklebust 
1505a08a8cd3STrond Myklebust 	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
15062b83d3deSTrond Myklebust 		fattr->size = size;
15072b83d3deSTrond Myklebust 	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
15082b83d3deSTrond Myklebust 		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1509a08a8cd3STrond Myklebust 		return;
15102b83d3deSTrond Myklebust 	}
15112b83d3deSTrond Myklebust 	if (size != fattr->size)
1512a08a8cd3STrond Myklebust 		return;
1513a08a8cd3STrond Myklebust 	/* Set attribute barrier */
1514a08a8cd3STrond Myklebust 	nfs_fattr_set_barrier(fattr);
15152b83d3deSTrond Myklebust 	/* ...and update size */
15162b83d3deSTrond Myklebust 	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1517a08a8cd3STrond Myklebust }
1518a08a8cd3STrond Myklebust 
1519a08a8cd3STrond Myklebust void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
1520a08a8cd3STrond Myklebust {
15212b83d3deSTrond Myklebust 	struct nfs_fattr *fattr = &hdr->fattr;
1522a08a8cd3STrond Myklebust 	struct inode *inode = hdr->inode;
1523a08a8cd3STrond Myklebust 
1524a08a8cd3STrond Myklebust 	spin_lock(&inode->i_lock);
1525a08a8cd3STrond Myklebust 	nfs_writeback_check_extend(hdr, fattr);
1526a08a8cd3STrond Myklebust 	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
1527a08a8cd3STrond Myklebust 	spin_unlock(&inode->i_lock);
1528a08a8cd3STrond Myklebust }
1529a08a8cd3STrond Myklebust EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);
1530a08a8cd3STrond Myklebust 
15311da177e4SLinus Torvalds /*
15321da177e4SLinus Torvalds  * This function is called when the WRITE call is complete.
15331da177e4SLinus Torvalds  */
1534d45f60c6SWeston Andros Adamson static int nfs_writeback_done(struct rpc_task *task,
1535d45f60c6SWeston Andros Adamson 			      struct nfs_pgio_header *hdr,
15360eecb214SAnna Schumaker 			      struct inode *inode)
15371da177e4SLinus Torvalds {
1538788e7a89STrond Myklebust 	int status;
15391da177e4SLinus Torvalds 
1540f551e44fSChuck Lever 	/*
1541f551e44fSChuck Lever 	 * ->write_done will attempt to use post-op attributes to detect
1542f551e44fSChuck Lever 	 * conflicting writes by other clients.  A strict interpretation
1543f551e44fSChuck Lever 	 * of close-to-open would allow us to continue caching even if
1544f551e44fSChuck Lever 	 * another writer had changed the file, but some applications
1545f551e44fSChuck Lever 	 * depend on tighter cache coherency when writing.
1546f551e44fSChuck Lever 	 */
1547d45f60c6SWeston Andros Adamson 	status = NFS_PROTO(inode)->write_done(task, hdr);
1548788e7a89STrond Myklebust 	if (status != 0)
15490eecb214SAnna Schumaker 		return status;
1550d45f60c6SWeston Andros Adamson 	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
155191d5b470SChuck Lever 
1552d45f60c6SWeston Andros Adamson 	if (hdr->res.verf->committed < hdr->args.stable &&
1553d45f60c6SWeston Andros Adamson 	    task->tk_status >= 0) {
15541da177e4SLinus Torvalds 		/* We tried a write call, but the server did not
15551da177e4SLinus Torvalds 		 * commit data to stable storage even though we
15561da177e4SLinus Torvalds 		 * requested it.
15571da177e4SLinus Torvalds 		 * Note: There is a known bug in Tru64 < 5.0 in which
15581da177e4SLinus Torvalds 		 *	 the server reports NFS_DATA_SYNC, but performs
15591da177e4SLinus Torvalds 		 *	 NFS_FILE_SYNC. We therefore implement this checking
15601da177e4SLinus Torvalds 		 *	 as a dprintk() in order to avoid filling syslog.
15611da177e4SLinus Torvalds 		 */
15621da177e4SLinus Torvalds 		static unsigned long    complain;
15631da177e4SLinus Torvalds 
1564a69aef14SFred Isaman 		/* Note this will print the MDS for a DS write */
15651da177e4SLinus Torvalds 		if (time_before(complain, jiffies)) {
15661da177e4SLinus Torvalds 			dprintk("NFS:       faulty NFS server %s:"
15671da177e4SLinus Torvalds 				" (committed = %d) != (stable = %d)\n",
1568cd841605SFred Isaman 				NFS_SERVER(inode)->nfs_client->cl_hostname,
1569d45f60c6SWeston Andros Adamson 				hdr->res.verf->committed, hdr->args.stable);
15701da177e4SLinus Torvalds 			complain = jiffies + 300 * HZ;
15711da177e4SLinus Torvalds 		}
15721da177e4SLinus Torvalds 	}
15731f2edbe3STrond Myklebust 
15741f2edbe3STrond Myklebust 	/* Deal with the suid/sgid bit corner case */
15751f2edbe3STrond Myklebust 	if (nfs_should_remove_suid(inode))
15761f2edbe3STrond Myklebust 		nfs_mark_for_revalidate(inode);
15770eecb214SAnna Schumaker 	return 0;
15780eecb214SAnna Schumaker }
15790eecb214SAnna Schumaker 
15800eecb214SAnna Schumaker /*
15810eecb214SAnna Schumaker  * This function is called when the WRITE call is complete.
15820eecb214SAnna Schumaker  */
1583d45f60c6SWeston Andros Adamson static void nfs_writeback_result(struct rpc_task *task,
1584d45f60c6SWeston Andros Adamson 				 struct nfs_pgio_header *hdr)
15850eecb214SAnna Schumaker {
1586d45f60c6SWeston Andros Adamson 	struct nfs_pgio_args	*argp = &hdr->args;
1587d45f60c6SWeston Andros Adamson 	struct nfs_pgio_res	*resp = &hdr->res;
15881f2edbe3STrond Myklebust 
15891f2edbe3STrond Myklebust 	if (resp->count < argp->count) {
15901da177e4SLinus Torvalds 		static unsigned long    complain;
15911da177e4SLinus Torvalds 
15926c75dc0dSFred Isaman 		/* This a short write! */
1593d45f60c6SWeston Andros Adamson 		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
159491d5b470SChuck Lever 
15951da177e4SLinus Torvalds 		/* Has the server at least made some progress? */
15966c75dc0dSFred Isaman 		if (resp->count == 0) {
15976c75dc0dSFred Isaman 			if (time_before(complain, jiffies)) {
15986c75dc0dSFred Isaman 				printk(KERN_WARNING
15996c75dc0dSFred Isaman 				       "NFS: Server wrote zero bytes, expected %u.\n",
16006c75dc0dSFred Isaman 				       argp->count);
16016c75dc0dSFred Isaman 				complain = jiffies + 300 * HZ;
16026c75dc0dSFred Isaman 			}
1603d45f60c6SWeston Andros Adamson 			nfs_set_pgio_error(hdr, -EIO, argp->offset);
16046c75dc0dSFred Isaman 			task->tk_status = -EIO;
16056c75dc0dSFred Isaman 			return;
16066c75dc0dSFred Isaman 		}
1607f8417b48SKinglong Mee 
1608f8417b48SKinglong Mee 		/* For non rpc-based layout drivers, retry-through-MDS */
1609f8417b48SKinglong Mee 		if (!task->tk_ops) {
1610f8417b48SKinglong Mee 			hdr->pnfs_error = -EAGAIN;
1611f8417b48SKinglong Mee 			return;
1612f8417b48SKinglong Mee 		}
1613f8417b48SKinglong Mee 
16141da177e4SLinus Torvalds 		/* Was this an NFSv2 write or an NFSv3 stable write? */
16151da177e4SLinus Torvalds 		if (resp->verf->committed != NFS_UNSTABLE) {
16161da177e4SLinus Torvalds 			/* Resend from where the server left off */
1617d45f60c6SWeston Andros Adamson 			hdr->mds_offset += resp->count;
16181da177e4SLinus Torvalds 			argp->offset += resp->count;
16191da177e4SLinus Torvalds 			argp->pgbase += resp->count;
16201da177e4SLinus Torvalds 			argp->count -= resp->count;
16211da177e4SLinus Torvalds 		} else {
16221da177e4SLinus Torvalds 			/* Resend as a stable write in order to avoid
16231da177e4SLinus Torvalds 			 * headaches in the case of a server crash.
16241da177e4SLinus Torvalds 			 */
16251da177e4SLinus Torvalds 			argp->stable = NFS_FILE_SYNC;
16261da177e4SLinus Torvalds 		}
1627d00c5d43STrond Myklebust 		rpc_restart_call_prepare(task);
16281da177e4SLinus Torvalds 	}
16291da177e4SLinus Torvalds }
16301da177e4SLinus Torvalds 
1631af7cf057STrond Myklebust static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
163271d0a611STrond Myklebust {
1633af7cf057STrond Myklebust 	return wait_on_atomic_t(&cinfo->rpcs_out,
1634af7cf057STrond Myklebust 			nfs_wait_atomic_killable, TASK_KILLABLE);
1635af7cf057STrond Myklebust }
1636af7cf057STrond Myklebust 
1637af7cf057STrond Myklebust static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
1638af7cf057STrond Myklebust {
1639af7cf057STrond Myklebust 	atomic_inc(&cinfo->rpcs_out);
1640af7cf057STrond Myklebust }
1641af7cf057STrond Myklebust 
1642af7cf057STrond Myklebust static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1643af7cf057STrond Myklebust {
1644af7cf057STrond Myklebust 	if (atomic_dec_and_test(&cinfo->rpcs_out))
1645af7cf057STrond Myklebust 		wake_up_atomic_t(&cinfo->rpcs_out);
164671d0a611STrond Myklebust }
164771d0a611STrond Myklebust 
16480b7c0153SFred Isaman void nfs_commitdata_release(struct nfs_commit_data *data)
16491da177e4SLinus Torvalds {
16500b7c0153SFred Isaman 	put_nfs_open_context(data->context);
16510b7c0153SFred Isaman 	nfs_commit_free(data);
16521da177e4SLinus Torvalds }
1653e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_commitdata_release);
16541da177e4SLinus Torvalds 
16550b7c0153SFred Isaman int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1656c36aae9aSPeng Tao 			const struct nfs_rpc_ops *nfs_ops,
16579ace33cdSFred Isaman 			const struct rpc_call_ops *call_ops,
16589f0ec176SAndy Adamson 			int how, int flags)
16591da177e4SLinus Torvalds {
166007737691STrond Myklebust 	struct rpc_task *task;
16619ace33cdSFred Isaman 	int priority = flush_task_priority(how);
1662bdc7f021STrond Myklebust 	struct rpc_message msg = {
1663bdc7f021STrond Myklebust 		.rpc_argp = &data->args,
1664bdc7f021STrond Myklebust 		.rpc_resp = &data->res,
16659ace33cdSFred Isaman 		.rpc_cred = data->cred,
1666bdc7f021STrond Myklebust 	};
166784115e1cSTrond Myklebust 	struct rpc_task_setup task_setup_data = {
166807737691STrond Myklebust 		.task = &data->task,
16699ace33cdSFred Isaman 		.rpc_client = clnt,
1670bdc7f021STrond Myklebust 		.rpc_message = &msg,
16719ace33cdSFred Isaman 		.callback_ops = call_ops,
167284115e1cSTrond Myklebust 		.callback_data = data,
1673101070caSTrond Myklebust 		.workqueue = nfsiod_workqueue,
16749f0ec176SAndy Adamson 		.flags = RPC_TASK_ASYNC | flags,
16753ff7576dSTrond Myklebust 		.priority = priority,
167684115e1cSTrond Myklebust 	};
1677788e7a89STrond Myklebust 	/* Set up the initial task struct.  */
1678c36aae9aSPeng Tao 	nfs_ops->commit_setup(data, &msg);
16791da177e4SLinus Torvalds 
1680b4839ebeSKinglong Mee 	dprintk("NFS: initiated commit call\n");
1681bdc7f021STrond Myklebust 
16828c21c62cSWeston Andros Adamson 	nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client,
16838c21c62cSWeston Andros Adamson 		NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg);
16848c21c62cSWeston Andros Adamson 
168507737691STrond Myklebust 	task = rpc_run_task(&task_setup_data);
1686dbae4c73STrond Myklebust 	if (IS_ERR(task))
1687dbae4c73STrond Myklebust 		return PTR_ERR(task);
1688d2224e7aSJeff Layton 	if (how & FLUSH_SYNC)
1689d2224e7aSJeff Layton 		rpc_wait_for_completion_task(task);
169007737691STrond Myklebust 	rpc_put_task(task);
1691dbae4c73STrond Myklebust 	return 0;
16921da177e4SLinus Torvalds }
1693e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_initiate_commit);
16941da177e4SLinus Torvalds 
1695378520b8SPeng Tao static loff_t nfs_get_lwb(struct list_head *head)
1696378520b8SPeng Tao {
1697378520b8SPeng Tao 	loff_t lwb = 0;
1698378520b8SPeng Tao 	struct nfs_page *req;
1699378520b8SPeng Tao 
1700378520b8SPeng Tao 	list_for_each_entry(req, head, wb_list)
1701378520b8SPeng Tao 		if (lwb < (req_offset(req) + req->wb_bytes))
1702378520b8SPeng Tao 			lwb = req_offset(req) + req->wb_bytes;
1703378520b8SPeng Tao 
1704378520b8SPeng Tao 	return lwb;
1705378520b8SPeng Tao }
1706378520b8SPeng Tao 
17071da177e4SLinus Torvalds /*
17089ace33cdSFred Isaman  * Set up the argument/result storage required for the RPC call.
17099ace33cdSFred Isaman  */
17100b7c0153SFred Isaman void nfs_init_commit(struct nfs_commit_data *data,
1711988b6dceSFred Isaman 		     struct list_head *head,
1712f453a54aSFred Isaman 		     struct pnfs_layout_segment *lseg,
1713f453a54aSFred Isaman 		     struct nfs_commit_info *cinfo)
17149ace33cdSFred Isaman {
17159ace33cdSFred Isaman 	struct nfs_page *first = nfs_list_entry(head->next);
17162b0143b5SDavid Howells 	struct inode *inode = d_inode(first->wb_context->dentry);
17179ace33cdSFred Isaman 
17189ace33cdSFred Isaman 	/* Set up the RPC argument and reply structs
17199ace33cdSFred Isaman 	 * NB: take care not to mess about with data->commit et al. */
17209ace33cdSFred Isaman 
17219ace33cdSFred Isaman 	list_splice_init(head, &data->pages);
17229ace33cdSFred Isaman 
17239ace33cdSFred Isaman 	data->inode	  = inode;
17249ace33cdSFred Isaman 	data->cred	  = first->wb_context->cred;
1725988b6dceSFred Isaman 	data->lseg	  = lseg; /* reference transferred */
1726378520b8SPeng Tao 	/* only set lwb for pnfs commit */
1727378520b8SPeng Tao 	if (lseg)
1728378520b8SPeng Tao 		data->lwb = nfs_get_lwb(&data->pages);
17299ace33cdSFred Isaman 	data->mds_ops     = &nfs_commit_ops;
1730f453a54aSFred Isaman 	data->completion_ops = cinfo->completion_ops;
1731b359f9d0SFred Isaman 	data->dreq	  = cinfo->dreq;
17329ace33cdSFred Isaman 
17339ace33cdSFred Isaman 	data->args.fh     = NFS_FH(data->inode);
17349ace33cdSFred Isaman 	/* Note: we always request a commit of the entire inode */
17359ace33cdSFred Isaman 	data->args.offset = 0;
17369ace33cdSFred Isaman 	data->args.count  = 0;
17370b7c0153SFred Isaman 	data->context     = get_nfs_open_context(first->wb_context);
17389ace33cdSFred Isaman 	data->res.fattr   = &data->fattr;
17399ace33cdSFred Isaman 	data->res.verf    = &data->verf;
17409ace33cdSFred Isaman 	nfs_fattr_init(&data->fattr);
17419ace33cdSFred Isaman }
1742e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_init_commit);
17439ace33cdSFred Isaman 
1744e0c2b380SFred Isaman void nfs_retry_commit(struct list_head *page_list,
1745ea2cf228SFred Isaman 		      struct pnfs_layout_segment *lseg,
1746b57ff130SWeston Andros Adamson 		      struct nfs_commit_info *cinfo,
1747b57ff130SWeston Andros Adamson 		      u32 ds_commit_idx)
174864bfeb49SFred Isaman {
174964bfeb49SFred Isaman 	struct nfs_page *req;
175064bfeb49SFred Isaman 
175164bfeb49SFred Isaman 	while (!list_empty(page_list)) {
175264bfeb49SFred Isaman 		req = nfs_list_entry(page_list->next);
175364bfeb49SFred Isaman 		nfs_list_remove_request(req);
1754b57ff130SWeston Andros Adamson 		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1755487b9b8aSTom Haynes 		if (!cinfo->dreq)
1756487b9b8aSTom Haynes 			nfs_clear_page_commit(req->wb_page);
17571d1afcbcSTrond Myklebust 		nfs_unlock_and_release_request(req);
175864bfeb49SFred Isaman 	}
175964bfeb49SFred Isaman }
1760e0c2b380SFred Isaman EXPORT_SYMBOL_GPL(nfs_retry_commit);
176164bfeb49SFred Isaman 
1762b20135d0STrond Myklebust static void
1763b20135d0STrond Myklebust nfs_commit_resched_write(struct nfs_commit_info *cinfo,
1764b20135d0STrond Myklebust 		struct nfs_page *req)
1765b20135d0STrond Myklebust {
1766b20135d0STrond Myklebust 	__set_page_dirty_nobuffers(req->wb_page);
1767b20135d0STrond Myklebust }
1768b20135d0STrond Myklebust 
17699ace33cdSFred Isaman /*
17701da177e4SLinus Torvalds  * Commit dirty pages
17711da177e4SLinus Torvalds  */
17721da177e4SLinus Torvalds static int
1773ea2cf228SFred Isaman nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1774ea2cf228SFred Isaman 		struct nfs_commit_info *cinfo)
17751da177e4SLinus Torvalds {
17760b7c0153SFred Isaman 	struct nfs_commit_data	*data;
17771da177e4SLinus Torvalds 
1778ade8febdSWeston Andros Adamson 	/* another commit raced with us */
1779ade8febdSWeston Andros Adamson 	if (list_empty(head))
1780ade8febdSWeston Andros Adamson 		return 0;
1781ade8febdSWeston Andros Adamson 
1782518662e0SNeilBrown 	data = nfs_commitdata_alloc(true);
17831da177e4SLinus Torvalds 
17841da177e4SLinus Torvalds 	/* Set up the argument struct */
1785f453a54aSFred Isaman 	nfs_init_commit(data, head, NULL, cinfo);
1786f453a54aSFred Isaman 	atomic_inc(&cinfo->mds->rpcs_out);
1787c36aae9aSPeng Tao 	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
1788c36aae9aSPeng Tao 				   data->mds_ops, how, 0);
17891da177e4SLinus Torvalds }
17901da177e4SLinus Torvalds 
17911da177e4SLinus Torvalds /*
17921da177e4SLinus Torvalds  * COMMIT call returned
17931da177e4SLinus Torvalds  */
1794788e7a89STrond Myklebust static void nfs_commit_done(struct rpc_task *task, void *calldata)
17951da177e4SLinus Torvalds {
17960b7c0153SFred Isaman 	struct nfs_commit_data	*data = calldata;
17971da177e4SLinus Torvalds 
1798a3f565b1SChuck Lever         dprintk("NFS: %5u nfs_commit_done (status %d)\n",
17991da177e4SLinus Torvalds                                 task->tk_pid, task->tk_status);
18001da177e4SLinus Torvalds 
1801788e7a89STrond Myklebust 	/* Call the NFS version-specific code */
1802c0d0e96bSTrond Myklebust 	NFS_PROTO(data->inode)->commit_done(task, data);
1803c9d8f89dSTrond Myklebust }
1804c9d8f89dSTrond Myklebust 
1805f453a54aSFred Isaman static void nfs_commit_release_pages(struct nfs_commit_data *data)
1806c9d8f89dSTrond Myklebust {
1807c9d8f89dSTrond Myklebust 	struct nfs_page	*req;
1808c9d8f89dSTrond Myklebust 	int status = data->task.tk_status;
1809f453a54aSFred Isaman 	struct nfs_commit_info cinfo;
1810353db796SNeilBrown 	struct nfs_server *nfss;
1811788e7a89STrond Myklebust 
18121da177e4SLinus Torvalds 	while (!list_empty(&data->pages)) {
18131da177e4SLinus Torvalds 		req = nfs_list_entry(data->pages.next);
18141da177e4SLinus Torvalds 		nfs_list_remove_request(req);
181567911c8fSAnna Schumaker 		if (req->wb_page)
1816d6d6dc7cSFred Isaman 			nfs_clear_page_commit(req->wb_page);
18171da177e4SLinus Torvalds 
18181e8968c5SNiels de Vos 		dprintk("NFS:       commit (%s/%llu %d@%lld)",
18193d4ff43dSAl Viro 			req->wb_context->dentry->d_sb->s_id,
18202b0143b5SDavid Howells 			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
18211da177e4SLinus Torvalds 			req->wb_bytes,
18221da177e4SLinus Torvalds 			(long long)req_offset(req));
1823c9d8f89dSTrond Myklebust 		if (status < 0) {
1824c9d8f89dSTrond Myklebust 			nfs_context_set_write_error(req->wb_context, status);
182538a33101SKinglong Mee 			if (req->wb_page)
18261da177e4SLinus Torvalds 				nfs_inode_remove_request(req);
1827ddeaa637SJoe Perches 			dprintk_cont(", error = %d\n", status);
18281da177e4SLinus Torvalds 			goto next;
18291da177e4SLinus Torvalds 		}
18301da177e4SLinus Torvalds 
18311da177e4SLinus Torvalds 		/* Okay, COMMIT succeeded, apparently. Check the verifier
18321da177e4SLinus Torvalds 		 * returned by the server against all stored verfs. */
18338fc3c386STrond Myklebust 		if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) {
18341da177e4SLinus Torvalds 			/* We have a match */
183538a33101SKinglong Mee 			if (req->wb_page)
18361da177e4SLinus Torvalds 				nfs_inode_remove_request(req);
1837ddeaa637SJoe Perches 			dprintk_cont(" OK\n");
18381da177e4SLinus Torvalds 			goto next;
18391da177e4SLinus Torvalds 		}
18401da177e4SLinus Torvalds 		/* We have a mismatch. Write the page again */
1841ddeaa637SJoe Perches 		dprintk_cont(" mismatch\n");
18426d884e8fSFred 		nfs_mark_request_dirty(req);
184305990d1bSTrond Myklebust 		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
18441da177e4SLinus Torvalds 	next:
18451d1afcbcSTrond Myklebust 		nfs_unlock_and_release_request(req);
18461da177e4SLinus Torvalds 	}
1847353db796SNeilBrown 	nfss = NFS_SERVER(data->inode);
1848353db796SNeilBrown 	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
18490db10944SJan Kara 		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1850353db796SNeilBrown 
1851f453a54aSFred Isaman 	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1852af7cf057STrond Myklebust 	nfs_commit_end(cinfo.mds);
18535917ce84SFred Isaman }
18545917ce84SFred Isaman 
18555917ce84SFred Isaman static void nfs_commit_release(void *calldata)
18565917ce84SFred Isaman {
18570b7c0153SFred Isaman 	struct nfs_commit_data *data = calldata;
18585917ce84SFred Isaman 
1859f453a54aSFred Isaman 	data->completion_ops->completion(data);
1860c9d8f89dSTrond Myklebust 	nfs_commitdata_release(calldata);
18611da177e4SLinus Torvalds }
1862788e7a89STrond Myklebust 
1863788e7a89STrond Myklebust static const struct rpc_call_ops nfs_commit_ops = {
18640b7c0153SFred Isaman 	.rpc_call_prepare = nfs_commit_prepare,
1865788e7a89STrond Myklebust 	.rpc_call_done = nfs_commit_done,
1866788e7a89STrond Myklebust 	.rpc_release = nfs_commit_release,
1867788e7a89STrond Myklebust };
18681da177e4SLinus Torvalds 
1869f453a54aSFred Isaman static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1870f453a54aSFred Isaman 	.completion = nfs_commit_release_pages,
1871b20135d0STrond Myklebust 	.resched_write = nfs_commit_resched_write,
1872f453a54aSFred Isaman };
1873f453a54aSFred Isaman 
18741763da12SFred Isaman int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1875ea2cf228SFred Isaman 			    int how, struct nfs_commit_info *cinfo)
187684c53ab5SFred Isaman {
187784c53ab5SFred Isaman 	int status;
187884c53ab5SFred Isaman 
1879ea2cf228SFred Isaman 	status = pnfs_commit_list(inode, head, how, cinfo);
188084c53ab5SFred Isaman 	if (status == PNFS_NOT_ATTEMPTED)
1881ea2cf228SFred Isaman 		status = nfs_commit_list(inode, head, how, cinfo);
188284c53ab5SFred Isaman 	return status;
188384c53ab5SFred Isaman }
188484c53ab5SFred Isaman 
1885b608b283STrond Myklebust int nfs_commit_inode(struct inode *inode, int how)
18861da177e4SLinus Torvalds {
18871da177e4SLinus Torvalds 	LIST_HEAD(head);
1888ea2cf228SFred Isaman 	struct nfs_commit_info cinfo;
188971d0a611STrond Myklebust 	int may_wait = how & FLUSH_SYNC;
1890af7cf057STrond Myklebust 	int error = 0;
1891b8413f98STrond Myklebust 	int res;
18921da177e4SLinus Torvalds 
1893ea2cf228SFred Isaman 	nfs_init_cinfo_from_inode(&cinfo, inode);
1894af7cf057STrond Myklebust 	nfs_commit_begin(cinfo.mds);
1895ea2cf228SFred Isaman 	res = nfs_scan_commit(inode, &head, &cinfo);
1896af7cf057STrond Myklebust 	if (res)
1897ea2cf228SFred Isaman 		error = nfs_generic_commit_list(inode, &head, how, &cinfo);
1898af7cf057STrond Myklebust 	nfs_commit_end(cinfo.mds);
18991da177e4SLinus Torvalds 	if (error < 0)
1900af7cf057STrond Myklebust 		goto out_error;
1901b8413f98STrond Myklebust 	if (!may_wait)
1902b8413f98STrond Myklebust 		goto out_mark_dirty;
1903af7cf057STrond Myklebust 	error = wait_on_commit(cinfo.mds);
1904b8413f98STrond Myklebust 	if (error < 0)
1905b8413f98STrond Myklebust 		return error;
1906c5efa5fcSTrond Myklebust 	return res;
1907af7cf057STrond Myklebust out_error:
1908af7cf057STrond Myklebust 	res = error;
1909c5efa5fcSTrond Myklebust 	/* Note: If we exit without ensuring that the commit is complete,
1910c5efa5fcSTrond Myklebust 	 * we must mark the inode as dirty. Otherwise, future calls to
1911c5efa5fcSTrond Myklebust 	 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
1912c5efa5fcSTrond Myklebust 	 * that the data is on the disk.
1913c5efa5fcSTrond Myklebust 	 */
1914c5efa5fcSTrond Myklebust out_mark_dirty:
1915c5efa5fcSTrond Myklebust 	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
19161da177e4SLinus Torvalds 	return res;
19171da177e4SLinus Torvalds }
1918b20135d0STrond Myklebust EXPORT_SYMBOL_GPL(nfs_commit_inode);
19198fc795f7STrond Myklebust 
1920ae09c31fSAnna Schumaker int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
19218fc795f7STrond Myklebust {
1922420e3646STrond Myklebust 	struct nfs_inode *nfsi = NFS_I(inode);
1923420e3646STrond Myklebust 	int flags = FLUSH_SYNC;
1924420e3646STrond Myklebust 	int ret = 0;
19258fc795f7STrond Myklebust 
19263236c3e1SJeff Layton 	/* no commits means nothing needs to be done */
1927ea2cf228SFred Isaman 	if (!nfsi->commit_info.ncommit)
19283236c3e1SJeff Layton 		return ret;
19293236c3e1SJeff Layton 
1930a00dd6c0SJeff Layton 	if (wbc->sync_mode == WB_SYNC_NONE) {
1931a00dd6c0SJeff Layton 		/* Don't commit yet if this is a non-blocking flush and there
1932a00dd6c0SJeff Layton 		 * are a lot of outstanding writes for this mapping.
1933420e3646STrond Myklebust 		 */
19341a4edf0fSTrond Myklebust 		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1935420e3646STrond Myklebust 			goto out_mark_dirty;
1936420e3646STrond Myklebust 
1937a00dd6c0SJeff Layton 		/* don't wait for the COMMIT response */
1938420e3646STrond Myklebust 		flags = 0;
1939a00dd6c0SJeff Layton 	}
1940a00dd6c0SJeff Layton 
1941420e3646STrond Myklebust 	ret = nfs_commit_inode(inode, flags);
1942420e3646STrond Myklebust 	if (ret >= 0) {
1943420e3646STrond Myklebust 		if (wbc->sync_mode == WB_SYNC_NONE) {
1944420e3646STrond Myklebust 			if (ret < wbc->nr_to_write)
1945420e3646STrond Myklebust 				wbc->nr_to_write -= ret;
1946420e3646STrond Myklebust 			else
1947420e3646STrond Myklebust 				wbc->nr_to_write = 0;
1948420e3646STrond Myklebust 		}
19498fc795f7STrond Myklebust 		return 0;
1950420e3646STrond Myklebust 	}
1951420e3646STrond Myklebust out_mark_dirty:
19528fc795f7STrond Myklebust 	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
19538fc795f7STrond Myklebust 	return ret;
19548fc795f7STrond Myklebust }
195589d77c8fSBryan Schumaker EXPORT_SYMBOL_GPL(nfs_write_inode);
1956863a3c6cSAndy Adamson 
1957acdc53b2STrond Myklebust /*
1958837bb1d7STrond Myklebust  * Wrapper for filemap_write_and_wait_range()
1959837bb1d7STrond Myklebust  *
1960837bb1d7STrond Myklebust  * Needed for pNFS in order to ensure data becomes visible to the
1961837bb1d7STrond Myklebust  * client.
1962837bb1d7STrond Myklebust  */
1963837bb1d7STrond Myklebust int nfs_filemap_write_and_wait_range(struct address_space *mapping,
1964837bb1d7STrond Myklebust 		loff_t lstart, loff_t lend)
1965837bb1d7STrond Myklebust {
1966837bb1d7STrond Myklebust 	int ret;
1967837bb1d7STrond Myklebust 
1968837bb1d7STrond Myklebust 	ret = filemap_write_and_wait_range(mapping, lstart, lend);
1969837bb1d7STrond Myklebust 	if (ret == 0)
1970837bb1d7STrond Myklebust 		ret = pnfs_sync_inode(mapping->host, true);
1971837bb1d7STrond Myklebust 	return ret;
1972837bb1d7STrond Myklebust }
1973837bb1d7STrond Myklebust EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);
1974837bb1d7STrond Myklebust 
1975837bb1d7STrond Myklebust /*
1976acdc53b2STrond Myklebust  * flush the inode to disk.
1977acdc53b2STrond Myklebust  */
1978acdc53b2STrond Myklebust int nfs_wb_all(struct inode *inode)
197934901f70STrond Myklebust {
1980f4ce1299STrond Myklebust 	int ret;
198134901f70STrond Myklebust 
1982f4ce1299STrond Myklebust 	trace_nfs_writeback_inode_enter(inode);
1983f4ce1299STrond Myklebust 
19845bb89b47STrond Myklebust 	ret = filemap_write_and_wait(inode->i_mapping);
19856b196875SChuck Lever 	if (ret)
19866b196875SChuck Lever 		goto out;
19875bb89b47STrond Myklebust 	ret = nfs_commit_inode(inode, FLUSH_SYNC);
19886b196875SChuck Lever 	if (ret < 0)
19896b196875SChuck Lever 		goto out;
19905bb89b47STrond Myklebust 	pnfs_sync_inode(inode, true);
19916b196875SChuck Lever 	ret = 0;
1992f4ce1299STrond Myklebust 
19936b196875SChuck Lever out:
1994f4ce1299STrond Myklebust 	trace_nfs_writeback_inode_exit(inode, ret);
1995f4ce1299STrond Myklebust 	return ret;
19961c75950bSTrond Myklebust }
1997ddda8e0aSBryan Schumaker EXPORT_SYMBOL_GPL(nfs_wb_all);
19981c75950bSTrond Myklebust 
19991b3b4a1aSTrond Myklebust int nfs_wb_page_cancel(struct inode *inode, struct page *page)
20001b3b4a1aSTrond Myklebust {
20011b3b4a1aSTrond Myklebust 	struct nfs_page *req;
20021b3b4a1aSTrond Myklebust 	int ret = 0;
20031b3b4a1aSTrond Myklebust 
2004ba8b06e6STrond Myklebust 	wait_on_page_writeback(page);
20053e217045SWeston Andros Adamson 
20063e217045SWeston Andros Adamson 	/* blocking call to cancel all requests and join to a single (head)
20073e217045SWeston Andros Adamson 	 * request */
2008*6d17d653STrond Myklebust 	req = nfs_lock_and_join_requests(page);
20093e217045SWeston Andros Adamson 
20103e217045SWeston Andros Adamson 	if (IS_ERR(req)) {
20113e217045SWeston Andros Adamson 		ret = PTR_ERR(req);
20123e217045SWeston Andros Adamson 	} else if (req) {
20133e217045SWeston Andros Adamson 		/* all requests from this page have been cancelled by
20143e217045SWeston Andros Adamson 		 * nfs_lock_and_join_requests, so just remove the head
20153e217045SWeston Andros Adamson 		 * request from the inode / page_private pointer and
20163e217045SWeston Andros Adamson 		 * release it */
20171b3b4a1aSTrond Myklebust 		nfs_inode_remove_request(req);
20181d1afcbcSTrond Myklebust 		nfs_unlock_and_release_request(req);
20191b3b4a1aSTrond Myklebust 	}
20203e217045SWeston Andros Adamson 
20211b3b4a1aSTrond Myklebust 	return ret;
20221b3b4a1aSTrond Myklebust }
20231b3b4a1aSTrond Myklebust 
20241c75950bSTrond Myklebust /*
20251c75950bSTrond Myklebust  * Write back all requests on one page - we do this before reading it.
20261c75950bSTrond Myklebust  */
2027c373fff7STrond Myklebust int nfs_wb_page(struct inode *inode, struct page *page)
20281c75950bSTrond Myklebust {
202929418aa4SMel Gorman 	loff_t range_start = page_file_offset(page);
203009cbfeafSKirill A. Shutemov 	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
20317f2f12d9STrond Myklebust 	struct writeback_control wbc = {
20327f2f12d9STrond Myklebust 		.sync_mode = WB_SYNC_ALL,
20337f2f12d9STrond Myklebust 		.nr_to_write = 0,
20347f2f12d9STrond Myklebust 		.range_start = range_start,
20357f2f12d9STrond Myklebust 		.range_end = range_end,
20367f2f12d9STrond Myklebust 	};
20377f2f12d9STrond Myklebust 	int ret;
20387f2f12d9STrond Myklebust 
2039f4ce1299STrond Myklebust 	trace_nfs_writeback_page_enter(inode);
2040f4ce1299STrond Myklebust 
20410522f6adSTrond Myklebust 	for (;;) {
2042ba8b06e6STrond Myklebust 		wait_on_page_writeback(page);
20437f2f12d9STrond Myklebust 		if (clear_page_dirty_for_io(page)) {
2044c373fff7STrond Myklebust 			ret = nfs_writepage_locked(page, &wbc);
20457f2f12d9STrond Myklebust 			if (ret < 0)
20467f2f12d9STrond Myklebust 				goto out_error;
20470522f6adSTrond Myklebust 			continue;
20487f2f12d9STrond Myklebust 		}
2049f4ce1299STrond Myklebust 		ret = 0;
20500522f6adSTrond Myklebust 		if (!PagePrivate(page))
20510522f6adSTrond Myklebust 			break;
20520522f6adSTrond Myklebust 		ret = nfs_commit_inode(inode, FLUSH_SYNC);
20537f2f12d9STrond Myklebust 		if (ret < 0)
20547f2f12d9STrond Myklebust 			goto out_error;
20557f2f12d9STrond Myklebust 	}
20567f2f12d9STrond Myklebust out_error:
2057f4ce1299STrond Myklebust 	trace_nfs_writeback_page_exit(inode, ret);
20587f2f12d9STrond Myklebust 	return ret;
20591c75950bSTrond Myklebust }
20601c75950bSTrond Myklebust 
2061074cc1deSTrond Myklebust #ifdef CONFIG_MIGRATION
2062074cc1deSTrond Myklebust int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2063a6bc32b8SMel Gorman 		struct page *page, enum migrate_mode mode)
2064074cc1deSTrond Myklebust {
20652da95652SJeff Layton 	/*
20662da95652SJeff Layton 	 * If PagePrivate is set, then the page is currently associated with
20672da95652SJeff Layton 	 * an in-progress read or write request. Don't try to migrate it.
20682da95652SJeff Layton 	 *
20692da95652SJeff Layton 	 * FIXME: we could do this in principle, but we'll need a way to ensure
20702da95652SJeff Layton 	 *        that we can safely release the inode reference while holding
20712da95652SJeff Layton 	 *        the page lock.
20722da95652SJeff Layton 	 */
20732da95652SJeff Layton 	if (PagePrivate(page))
20742da95652SJeff Layton 		return -EBUSY;
2075074cc1deSTrond Myklebust 
20768c209ce7SDavid Howells 	if (!nfs_fscache_release_page(page, GFP_KERNEL))
20778c209ce7SDavid Howells 		return -EBUSY;
2078074cc1deSTrond Myklebust 
2079a6bc32b8SMel Gorman 	return migrate_page(mapping, newpage, page, mode);
2080074cc1deSTrond Myklebust }
2081074cc1deSTrond Myklebust #endif
2082074cc1deSTrond Myklebust 
2083f7b422b1SDavid Howells int __init nfs_init_writepagecache(void)
20841da177e4SLinus Torvalds {
20851da177e4SLinus Torvalds 	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
20861e7f3a48SWeston Andros Adamson 					     sizeof(struct nfs_pgio_header),
20871da177e4SLinus Torvalds 					     0, SLAB_HWCACHE_ALIGN,
208820c2df83SPaul Mundt 					     NULL);
20891da177e4SLinus Torvalds 	if (nfs_wdata_cachep == NULL)
20901da177e4SLinus Torvalds 		return -ENOMEM;
20911da177e4SLinus Torvalds 
209293d2341cSMatthew Dobson 	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
20931da177e4SLinus Torvalds 						     nfs_wdata_cachep);
20941da177e4SLinus Torvalds 	if (nfs_wdata_mempool == NULL)
20953dd4765fSJeff Layton 		goto out_destroy_write_cache;
20961da177e4SLinus Torvalds 
20970b7c0153SFred Isaman 	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
20980b7c0153SFred Isaman 					     sizeof(struct nfs_commit_data),
20990b7c0153SFred Isaman 					     0, SLAB_HWCACHE_ALIGN,
21000b7c0153SFred Isaman 					     NULL);
21010b7c0153SFred Isaman 	if (nfs_cdata_cachep == NULL)
21023dd4765fSJeff Layton 		goto out_destroy_write_mempool;
21030b7c0153SFred Isaman 
210493d2341cSMatthew Dobson 	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
21054c100210SYanchuan Nian 						      nfs_cdata_cachep);
21061da177e4SLinus Torvalds 	if (nfs_commit_mempool == NULL)
21073dd4765fSJeff Layton 		goto out_destroy_commit_cache;
21081da177e4SLinus Torvalds 
210989a09141SPeter Zijlstra 	/*
211089a09141SPeter Zijlstra 	 * NFS congestion size, scale with available memory.
211189a09141SPeter Zijlstra 	 *
211289a09141SPeter Zijlstra 	 *  64MB:    8192k
211389a09141SPeter Zijlstra 	 * 128MB:   11585k
211489a09141SPeter Zijlstra 	 * 256MB:   16384k
211589a09141SPeter Zijlstra 	 * 512MB:   23170k
211689a09141SPeter Zijlstra 	 *   1GB:   32768k
211789a09141SPeter Zijlstra 	 *   2GB:   46340k
211889a09141SPeter Zijlstra 	 *   4GB:   65536k
211989a09141SPeter Zijlstra 	 *   8GB:   92681k
212089a09141SPeter Zijlstra 	 *  16GB:  131072k
212189a09141SPeter Zijlstra 	 *
212289a09141SPeter Zijlstra 	 * This allows larger machines to have larger/more transfers.
212389a09141SPeter Zijlstra 	 * Limit the default to 256M
212489a09141SPeter Zijlstra 	 */
212589a09141SPeter Zijlstra 	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
212689a09141SPeter Zijlstra 	if (nfs_congestion_kb > 256*1024)
212789a09141SPeter Zijlstra 		nfs_congestion_kb = 256*1024;
212889a09141SPeter Zijlstra 
21291da177e4SLinus Torvalds 	return 0;
21303dd4765fSJeff Layton 
21313dd4765fSJeff Layton out_destroy_commit_cache:
21323dd4765fSJeff Layton 	kmem_cache_destroy(nfs_cdata_cachep);
21333dd4765fSJeff Layton out_destroy_write_mempool:
21343dd4765fSJeff Layton 	mempool_destroy(nfs_wdata_mempool);
21353dd4765fSJeff Layton out_destroy_write_cache:
21363dd4765fSJeff Layton 	kmem_cache_destroy(nfs_wdata_cachep);
21373dd4765fSJeff Layton 	return -ENOMEM;
21381da177e4SLinus Torvalds }
21391da177e4SLinus Torvalds 
2140266bee88SDavid Brownell void nfs_destroy_writepagecache(void)
21411da177e4SLinus Torvalds {
21421da177e4SLinus Torvalds 	mempool_destroy(nfs_commit_mempool);
21433dd4765fSJeff Layton 	kmem_cache_destroy(nfs_cdata_cachep);
21441da177e4SLinus Torvalds 	mempool_destroy(nfs_wdata_mempool);
21451a1d92c1SAlexey Dobriyan 	kmem_cache_destroy(nfs_wdata_cachep);
21461da177e4SLinus Torvalds }
21471da177e4SLinus Torvalds 
21484a0de55cSAnna Schumaker static const struct nfs_rw_ops nfs_rw_write_ops = {
21494a0de55cSAnna Schumaker 	.rw_alloc_header	= nfs_writehdr_alloc,
21504a0de55cSAnna Schumaker 	.rw_free_header		= nfs_writehdr_free,
21510eecb214SAnna Schumaker 	.rw_done		= nfs_writeback_done,
21520eecb214SAnna Schumaker 	.rw_result		= nfs_writeback_result,
21531ed26f33SAnna Schumaker 	.rw_initiate		= nfs_initiate_write,
21544a0de55cSAnna Schumaker };
2155