1 /* 2 * linux/fs/nfs/write.c 3 * 4 * Write file data over NFS. 5 * 6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/file.h> 14 #include <linux/writeback.h> 15 #include <linux/swap.h> 16 #include <linux/migrate.h> 17 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_mount.h> 21 #include <linux/nfs_page.h> 22 #include <linux/backing-dev.h> 23 #include <linux/export.h> 24 25 #include <asm/uaccess.h> 26 27 #include "delegation.h" 28 #include "internal.h" 29 #include "iostat.h" 30 #include "nfs4_fs.h" 31 #include "fscache.h" 32 #include "pnfs.h" 33 34 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 35 36 #define MIN_POOL_WRITE (32) 37 #define MIN_POOL_COMMIT (4) 38 39 /* 40 * Local function declarations 41 */ 42 static void nfs_redirty_request(struct nfs_page *req); 43 static const struct rpc_call_ops nfs_write_common_ops; 44 static const struct rpc_call_ops nfs_commit_ops; 45 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 46 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 47 48 static struct kmem_cache *nfs_wdata_cachep; 49 static mempool_t *nfs_wdata_mempool; 50 static struct kmem_cache *nfs_cdata_cachep; 51 static mempool_t *nfs_commit_mempool; 52 53 struct nfs_commit_data *nfs_commitdata_alloc(void) 54 { 55 struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 56 57 if (p) { 58 memset(p, 0, sizeof(*p)); 59 INIT_LIST_HEAD(&p->pages); 60 } 61 return p; 62 } 63 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 64 65 void nfs_commit_free(struct nfs_commit_data *p) 66 { 67 mempool_free(p, nfs_commit_mempool); 68 } 69 EXPORT_SYMBOL_GPL(nfs_commit_free); 70 71 struct nfs_write_header *nfs_writehdr_alloc(void) 72 { 73 struct nfs_write_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO); 74 75 if (p) { 76 struct nfs_pgio_header *hdr = &p->header; 77 78 memset(p, 0, sizeof(*p)); 79 INIT_LIST_HEAD(&hdr->pages); 80 INIT_LIST_HEAD(&hdr->rpc_list); 81 spin_lock_init(&hdr->lock); 82 atomic_set(&hdr->refcnt, 0); 83 hdr->verf = &p->verf; 84 } 85 return p; 86 } 87 EXPORT_SYMBOL_GPL(nfs_writehdr_alloc); 88 89 static struct nfs_write_data *nfs_writedata_alloc(struct nfs_pgio_header *hdr, 90 unsigned int pagecount) 91 { 92 struct nfs_write_data *data, *prealloc; 93 94 prealloc = &container_of(hdr, struct nfs_write_header, header)->rpc_data; 95 if (prealloc->header == NULL) 96 data = prealloc; 97 else 98 data = kzalloc(sizeof(*data), GFP_KERNEL); 99 if (!data) 100 goto out; 101 102 if (nfs_pgarray_set(&data->pages, pagecount)) { 103 data->header = hdr; 104 atomic_inc(&hdr->refcnt); 105 } else { 106 if (data != prealloc) 107 kfree(data); 108 data = NULL; 109 } 110 out: 111 return data; 112 } 113 114 void nfs_writehdr_free(struct nfs_pgio_header *hdr) 115 { 116 struct nfs_write_header *whdr = container_of(hdr, struct nfs_write_header, header); 117 mempool_free(whdr, nfs_wdata_mempool); 118 } 119 EXPORT_SYMBOL_GPL(nfs_writehdr_free); 120 121 void nfs_writedata_release(struct nfs_write_data *wdata) 122 { 123 struct nfs_pgio_header *hdr = wdata->header; 124 struct nfs_write_header *write_header = container_of(hdr, struct nfs_write_header, header); 125 126 put_nfs_open_context(wdata->args.context); 127 if (wdata->pages.pagevec != wdata->pages.page_array) 128 kfree(wdata->pages.pagevec); 129 if (wdata != &write_header->rpc_data) 130 kfree(wdata); 131 else 132 wdata->header = NULL; 133 if (atomic_dec_and_test(&hdr->refcnt)) 134 hdr->completion_ops->completion(hdr); 135 } 136 EXPORT_SYMBOL_GPL(nfs_writedata_release); 137 138 static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error) 139 { 140 ctx->error = error; 141 smp_wmb(); 142 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags); 143 } 144 145 static struct nfs_page * 146 nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page) 147 { 148 struct nfs_page *req = NULL; 149 150 if (PagePrivate(page)) 151 req = (struct nfs_page *)page_private(page); 152 else if (unlikely(PageSwapCache(page))) { 153 struct nfs_page *freq, *t; 154 155 /* Linearly search the commit list for the correct req */ 156 list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) { 157 if (freq->wb_page == page) { 158 req = freq; 159 break; 160 } 161 } 162 } 163 164 if (req) 165 kref_get(&req->wb_kref); 166 167 return req; 168 } 169 170 static struct nfs_page *nfs_page_find_request(struct page *page) 171 { 172 struct inode *inode = page_file_mapping(page)->host; 173 struct nfs_page *req = NULL; 174 175 spin_lock(&inode->i_lock); 176 req = nfs_page_find_request_locked(NFS_I(inode), page); 177 spin_unlock(&inode->i_lock); 178 return req; 179 } 180 181 /* Adjust the file length if we're writing beyond the end */ 182 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 183 { 184 struct inode *inode = page_file_mapping(page)->host; 185 loff_t end, i_size; 186 pgoff_t end_index; 187 188 spin_lock(&inode->i_lock); 189 i_size = i_size_read(inode); 190 end_index = (i_size - 1) >> PAGE_CACHE_SHIFT; 191 if (i_size > 0 && page_file_index(page) < end_index) 192 goto out; 193 end = page_file_offset(page) + ((loff_t)offset+count); 194 if (i_size >= end) 195 goto out; 196 i_size_write(inode, end); 197 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 198 out: 199 spin_unlock(&inode->i_lock); 200 } 201 202 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 203 static void nfs_set_pageerror(struct page *page) 204 { 205 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page)); 206 } 207 208 /* We can set the PG_uptodate flag if we see that a write request 209 * covers the full page. 210 */ 211 static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count) 212 { 213 if (PageUptodate(page)) 214 return; 215 if (base != 0) 216 return; 217 if (count != nfs_page_length(page)) 218 return; 219 SetPageUptodate(page); 220 } 221 222 static int wb_priority(struct writeback_control *wbc) 223 { 224 if (wbc->for_reclaim) 225 return FLUSH_HIGHPRI | FLUSH_STABLE; 226 if (wbc->for_kupdate || wbc->for_background) 227 return FLUSH_LOWPRI | FLUSH_COND_STABLE; 228 return FLUSH_COND_STABLE; 229 } 230 231 /* 232 * NFS congestion control 233 */ 234 235 int nfs_congestion_kb; 236 237 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 238 #define NFS_CONGESTION_OFF_THRESH \ 239 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 240 241 static void nfs_set_page_writeback(struct page *page) 242 { 243 struct nfs_server *nfss = NFS_SERVER(page_file_mapping(page)->host); 244 int ret = test_set_page_writeback(page); 245 246 WARN_ON_ONCE(ret != 0); 247 248 if (atomic_long_inc_return(&nfss->writeback) > 249 NFS_CONGESTION_ON_THRESH) { 250 set_bdi_congested(&nfss->backing_dev_info, 251 BLK_RW_ASYNC); 252 } 253 } 254 255 static void nfs_end_page_writeback(struct page *page) 256 { 257 struct inode *inode = page_file_mapping(page)->host; 258 struct nfs_server *nfss = NFS_SERVER(inode); 259 260 end_page_writeback(page); 261 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 262 clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC); 263 } 264 265 static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock) 266 { 267 struct inode *inode = page_file_mapping(page)->host; 268 struct nfs_page *req; 269 int ret; 270 271 spin_lock(&inode->i_lock); 272 for (;;) { 273 req = nfs_page_find_request_locked(NFS_I(inode), page); 274 if (req == NULL) 275 break; 276 if (nfs_lock_request(req)) 277 break; 278 /* Note: If we hold the page lock, as is the case in nfs_writepage, 279 * then the call to nfs_lock_request() will always 280 * succeed provided that someone hasn't already marked the 281 * request as dirty (in which case we don't care). 282 */ 283 spin_unlock(&inode->i_lock); 284 if (!nonblock) 285 ret = nfs_wait_on_request(req); 286 else 287 ret = -EAGAIN; 288 nfs_release_request(req); 289 if (ret != 0) 290 return ERR_PTR(ret); 291 spin_lock(&inode->i_lock); 292 } 293 spin_unlock(&inode->i_lock); 294 return req; 295 } 296 297 /* 298 * Find an associated nfs write request, and prepare to flush it out 299 * May return an error if the user signalled nfs_wait_on_request(). 300 */ 301 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 302 struct page *page, bool nonblock) 303 { 304 struct nfs_page *req; 305 int ret = 0; 306 307 req = nfs_find_and_lock_request(page, nonblock); 308 if (!req) 309 goto out; 310 ret = PTR_ERR(req); 311 if (IS_ERR(req)) 312 goto out; 313 314 nfs_set_page_writeback(page); 315 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 316 317 ret = 0; 318 if (!nfs_pageio_add_request(pgio, req)) { 319 nfs_redirty_request(req); 320 ret = pgio->pg_error; 321 } 322 out: 323 return ret; 324 } 325 326 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio) 327 { 328 struct inode *inode = page_file_mapping(page)->host; 329 int ret; 330 331 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 332 nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1); 333 334 nfs_pageio_cond_complete(pgio, page_file_index(page)); 335 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE); 336 if (ret == -EAGAIN) { 337 redirty_page_for_writepage(wbc, page); 338 ret = 0; 339 } 340 return ret; 341 } 342 343 /* 344 * Write an mmapped page to the server. 345 */ 346 static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc) 347 { 348 struct nfs_pageio_descriptor pgio; 349 int err; 350 351 NFS_PROTO(page_file_mapping(page)->host)->write_pageio_init(&pgio, 352 page->mapping->host, 353 wb_priority(wbc), 354 &nfs_async_write_completion_ops); 355 err = nfs_do_writepage(page, wbc, &pgio); 356 nfs_pageio_complete(&pgio); 357 if (err < 0) 358 return err; 359 if (pgio.pg_error < 0) 360 return pgio.pg_error; 361 return 0; 362 } 363 364 int nfs_writepage(struct page *page, struct writeback_control *wbc) 365 { 366 int ret; 367 368 ret = nfs_writepage_locked(page, wbc); 369 unlock_page(page); 370 return ret; 371 } 372 373 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 374 { 375 int ret; 376 377 ret = nfs_do_writepage(page, wbc, data); 378 unlock_page(page); 379 return ret; 380 } 381 382 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 383 { 384 struct inode *inode = mapping->host; 385 unsigned long *bitlock = &NFS_I(inode)->flags; 386 struct nfs_pageio_descriptor pgio; 387 int err; 388 389 /* Stop dirtying of new pages while we sync */ 390 err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING, 391 nfs_wait_bit_killable, TASK_KILLABLE); 392 if (err) 393 goto out_err; 394 395 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 396 397 NFS_PROTO(inode)->write_pageio_init(&pgio, inode, wb_priority(wbc), &nfs_async_write_completion_ops); 398 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 399 nfs_pageio_complete(&pgio); 400 401 clear_bit_unlock(NFS_INO_FLUSHING, bitlock); 402 smp_mb__after_clear_bit(); 403 wake_up_bit(bitlock, NFS_INO_FLUSHING); 404 405 if (err < 0) 406 goto out_err; 407 err = pgio.pg_error; 408 if (err < 0) 409 goto out_err; 410 return 0; 411 out_err: 412 return err; 413 } 414 415 /* 416 * Insert a write request into an inode 417 */ 418 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 419 { 420 struct nfs_inode *nfsi = NFS_I(inode); 421 422 /* Lock the request! */ 423 nfs_lock_request(req); 424 425 spin_lock(&inode->i_lock); 426 if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 427 inode->i_version++; 428 /* 429 * Swap-space should not get truncated. Hence no need to plug the race 430 * with invalidate/truncate. 431 */ 432 if (likely(!PageSwapCache(req->wb_page))) { 433 set_bit(PG_MAPPED, &req->wb_flags); 434 SetPagePrivate(req->wb_page); 435 set_page_private(req->wb_page, (unsigned long)req); 436 } 437 nfsi->npages++; 438 kref_get(&req->wb_kref); 439 spin_unlock(&inode->i_lock); 440 } 441 442 /* 443 * Remove a write request from an inode 444 */ 445 static void nfs_inode_remove_request(struct nfs_page *req) 446 { 447 struct inode *inode = req->wb_context->dentry->d_inode; 448 struct nfs_inode *nfsi = NFS_I(inode); 449 450 spin_lock(&inode->i_lock); 451 if (likely(!PageSwapCache(req->wb_page))) { 452 set_page_private(req->wb_page, 0); 453 ClearPagePrivate(req->wb_page); 454 clear_bit(PG_MAPPED, &req->wb_flags); 455 } 456 nfsi->npages--; 457 spin_unlock(&inode->i_lock); 458 nfs_release_request(req); 459 } 460 461 static void 462 nfs_mark_request_dirty(struct nfs_page *req) 463 { 464 __set_page_dirty_nobuffers(req->wb_page); 465 } 466 467 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 468 /** 469 * nfs_request_add_commit_list - add request to a commit list 470 * @req: pointer to a struct nfs_page 471 * @dst: commit list head 472 * @cinfo: holds list lock and accounting info 473 * 474 * This sets the PG_CLEAN bit, updates the cinfo count of 475 * number of outstanding requests requiring a commit as well as 476 * the MM page stats. 477 * 478 * The caller must _not_ hold the cinfo->lock, but must be 479 * holding the nfs_page lock. 480 */ 481 void 482 nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst, 483 struct nfs_commit_info *cinfo) 484 { 485 set_bit(PG_CLEAN, &(req)->wb_flags); 486 spin_lock(cinfo->lock); 487 nfs_list_add_request(req, dst); 488 cinfo->mds->ncommit++; 489 spin_unlock(cinfo->lock); 490 if (!cinfo->dreq) { 491 inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 492 inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 493 BDI_RECLAIMABLE); 494 __mark_inode_dirty(req->wb_context->dentry->d_inode, 495 I_DIRTY_DATASYNC); 496 } 497 } 498 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 499 500 /** 501 * nfs_request_remove_commit_list - Remove request from a commit list 502 * @req: pointer to a nfs_page 503 * @cinfo: holds list lock and accounting info 504 * 505 * This clears the PG_CLEAN bit, and updates the cinfo's count of 506 * number of outstanding requests requiring a commit 507 * It does not update the MM page stats. 508 * 509 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 510 */ 511 void 512 nfs_request_remove_commit_list(struct nfs_page *req, 513 struct nfs_commit_info *cinfo) 514 { 515 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 516 return; 517 nfs_list_remove_request(req); 518 cinfo->mds->ncommit--; 519 } 520 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 521 522 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 523 struct inode *inode) 524 { 525 cinfo->lock = &inode->i_lock; 526 cinfo->mds = &NFS_I(inode)->commit_info; 527 cinfo->ds = pnfs_get_ds_info(inode); 528 cinfo->dreq = NULL; 529 cinfo->completion_ops = &nfs_commit_completion_ops; 530 } 531 532 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 533 struct inode *inode, 534 struct nfs_direct_req *dreq) 535 { 536 if (dreq) 537 nfs_init_cinfo_from_dreq(cinfo, dreq); 538 else 539 nfs_init_cinfo_from_inode(cinfo, inode); 540 } 541 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 542 543 /* 544 * Add a request to the inode's commit list. 545 */ 546 void 547 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 548 struct nfs_commit_info *cinfo) 549 { 550 if (pnfs_mark_request_commit(req, lseg, cinfo)) 551 return; 552 nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo); 553 } 554 555 static void 556 nfs_clear_page_commit(struct page *page) 557 { 558 dec_zone_page_state(page, NR_UNSTABLE_NFS); 559 dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE); 560 } 561 562 static void 563 nfs_clear_request_commit(struct nfs_page *req) 564 { 565 if (test_bit(PG_CLEAN, &req->wb_flags)) { 566 struct inode *inode = req->wb_context->dentry->d_inode; 567 struct nfs_commit_info cinfo; 568 569 nfs_init_cinfo_from_inode(&cinfo, inode); 570 if (!pnfs_clear_request_commit(req, &cinfo)) { 571 spin_lock(cinfo.lock); 572 nfs_request_remove_commit_list(req, &cinfo); 573 spin_unlock(cinfo.lock); 574 } 575 nfs_clear_page_commit(req->wb_page); 576 } 577 } 578 579 static inline 580 int nfs_write_need_commit(struct nfs_write_data *data) 581 { 582 if (data->verf.committed == NFS_DATA_SYNC) 583 return data->header->lseg == NULL; 584 return data->verf.committed != NFS_FILE_SYNC; 585 } 586 587 #else 588 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 589 struct inode *inode) 590 { 591 } 592 593 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 594 struct inode *inode, 595 struct nfs_direct_req *dreq) 596 { 597 } 598 599 void 600 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 601 struct nfs_commit_info *cinfo) 602 { 603 } 604 605 static void 606 nfs_clear_request_commit(struct nfs_page *req) 607 { 608 } 609 610 static inline 611 int nfs_write_need_commit(struct nfs_write_data *data) 612 { 613 return 0; 614 } 615 616 #endif 617 618 static void nfs_write_completion(struct nfs_pgio_header *hdr) 619 { 620 struct nfs_commit_info cinfo; 621 unsigned long bytes = 0; 622 623 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 624 goto out; 625 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 626 while (!list_empty(&hdr->pages)) { 627 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 628 629 bytes += req->wb_bytes; 630 nfs_list_remove_request(req); 631 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 632 (hdr->good_bytes < bytes)) { 633 nfs_set_pageerror(req->wb_page); 634 nfs_context_set_write_error(req->wb_context, hdr->error); 635 goto remove_req; 636 } 637 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) { 638 nfs_mark_request_dirty(req); 639 goto next; 640 } 641 if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) { 642 memcpy(&req->wb_verf, &hdr->verf->verifier, sizeof(req->wb_verf)); 643 nfs_mark_request_commit(req, hdr->lseg, &cinfo); 644 goto next; 645 } 646 remove_req: 647 nfs_inode_remove_request(req); 648 next: 649 nfs_unlock_request(req); 650 nfs_end_page_writeback(req->wb_page); 651 nfs_release_request(req); 652 } 653 out: 654 hdr->release(hdr); 655 } 656 657 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 658 static unsigned long 659 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 660 { 661 return cinfo->mds->ncommit; 662 } 663 664 /* cinfo->lock held by caller */ 665 int 666 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 667 struct nfs_commit_info *cinfo, int max) 668 { 669 struct nfs_page *req, *tmp; 670 int ret = 0; 671 672 list_for_each_entry_safe(req, tmp, src, wb_list) { 673 if (!nfs_lock_request(req)) 674 continue; 675 kref_get(&req->wb_kref); 676 if (cond_resched_lock(cinfo->lock)) 677 list_safe_reset_next(req, tmp, wb_list); 678 nfs_request_remove_commit_list(req, cinfo); 679 nfs_list_add_request(req, dst); 680 ret++; 681 if ((ret == max) && !cinfo->dreq) 682 break; 683 } 684 return ret; 685 } 686 687 /* 688 * nfs_scan_commit - Scan an inode for commit requests 689 * @inode: NFS inode to scan 690 * @dst: mds destination list 691 * @cinfo: mds and ds lists of reqs ready to commit 692 * 693 * Moves requests from the inode's 'commit' request list. 694 * The requests are *not* checked to ensure that they form a contiguous set. 695 */ 696 int 697 nfs_scan_commit(struct inode *inode, struct list_head *dst, 698 struct nfs_commit_info *cinfo) 699 { 700 int ret = 0; 701 702 spin_lock(cinfo->lock); 703 if (cinfo->mds->ncommit > 0) { 704 const int max = INT_MAX; 705 706 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 707 cinfo, max); 708 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 709 } 710 spin_unlock(cinfo->lock); 711 return ret; 712 } 713 714 #else 715 static unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 716 { 717 return 0; 718 } 719 720 int nfs_scan_commit(struct inode *inode, struct list_head *dst, 721 struct nfs_commit_info *cinfo) 722 { 723 return 0; 724 } 725 #endif 726 727 /* 728 * Search for an existing write request, and attempt to update 729 * it to reflect a new dirty region on a given page. 730 * 731 * If the attempt fails, then the existing request is flushed out 732 * to disk. 733 */ 734 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 735 struct page *page, 736 unsigned int offset, 737 unsigned int bytes) 738 { 739 struct nfs_page *req; 740 unsigned int rqend; 741 unsigned int end; 742 int error; 743 744 if (!PagePrivate(page)) 745 return NULL; 746 747 end = offset + bytes; 748 spin_lock(&inode->i_lock); 749 750 for (;;) { 751 req = nfs_page_find_request_locked(NFS_I(inode), page); 752 if (req == NULL) 753 goto out_unlock; 754 755 rqend = req->wb_offset + req->wb_bytes; 756 /* 757 * Tell the caller to flush out the request if 758 * the offsets are non-contiguous. 759 * Note: nfs_flush_incompatible() will already 760 * have flushed out requests having wrong owners. 761 */ 762 if (offset > rqend 763 || end < req->wb_offset) 764 goto out_flushme; 765 766 if (nfs_lock_request(req)) 767 break; 768 769 /* The request is locked, so wait and then retry */ 770 spin_unlock(&inode->i_lock); 771 error = nfs_wait_on_request(req); 772 nfs_release_request(req); 773 if (error != 0) 774 goto out_err; 775 spin_lock(&inode->i_lock); 776 } 777 778 /* Okay, the request matches. Update the region */ 779 if (offset < req->wb_offset) { 780 req->wb_offset = offset; 781 req->wb_pgbase = offset; 782 } 783 if (end > rqend) 784 req->wb_bytes = end - req->wb_offset; 785 else 786 req->wb_bytes = rqend - req->wb_offset; 787 out_unlock: 788 spin_unlock(&inode->i_lock); 789 if (req) 790 nfs_clear_request_commit(req); 791 return req; 792 out_flushme: 793 spin_unlock(&inode->i_lock); 794 nfs_release_request(req); 795 error = nfs_wb_page(inode, page); 796 out_err: 797 return ERR_PTR(error); 798 } 799 800 /* 801 * Try to update an existing write request, or create one if there is none. 802 * 803 * Note: Should always be called with the Page Lock held to prevent races 804 * if we have to add a new request. Also assumes that the caller has 805 * already called nfs_flush_incompatible() if necessary. 806 */ 807 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 808 struct page *page, unsigned int offset, unsigned int bytes) 809 { 810 struct inode *inode = page_file_mapping(page)->host; 811 struct nfs_page *req; 812 813 req = nfs_try_to_update_request(inode, page, offset, bytes); 814 if (req != NULL) 815 goto out; 816 req = nfs_create_request(ctx, inode, page, offset, bytes); 817 if (IS_ERR(req)) 818 goto out; 819 nfs_inode_add_request(inode, req); 820 out: 821 return req; 822 } 823 824 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 825 unsigned int offset, unsigned int count) 826 { 827 struct nfs_page *req; 828 829 req = nfs_setup_write_request(ctx, page, offset, count); 830 if (IS_ERR(req)) 831 return PTR_ERR(req); 832 /* Update file length */ 833 nfs_grow_file(page, offset, count); 834 nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes); 835 nfs_mark_request_dirty(req); 836 nfs_unlock_and_release_request(req); 837 return 0; 838 } 839 840 int nfs_flush_incompatible(struct file *file, struct page *page) 841 { 842 struct nfs_open_context *ctx = nfs_file_open_context(file); 843 struct nfs_lock_context *l_ctx; 844 struct nfs_page *req; 845 int do_flush, status; 846 /* 847 * Look for a request corresponding to this page. If there 848 * is one, and it belongs to another file, we flush it out 849 * before we try to copy anything into the page. Do this 850 * due to the lack of an ACCESS-type call in NFSv2. 851 * Also do the same if we find a request from an existing 852 * dropped page. 853 */ 854 do { 855 req = nfs_page_find_request(page); 856 if (req == NULL) 857 return 0; 858 l_ctx = req->wb_lock_context; 859 do_flush = req->wb_page != page || req->wb_context != ctx; 860 if (l_ctx) { 861 do_flush |= l_ctx->lockowner.l_owner != current->files 862 || l_ctx->lockowner.l_pid != current->tgid; 863 } 864 nfs_release_request(req); 865 if (!do_flush) 866 return 0; 867 status = nfs_wb_page(page_file_mapping(page)->host, page); 868 } while (status == 0); 869 return status; 870 } 871 872 /* 873 * If the page cache is marked as unsafe or invalid, then we can't rely on 874 * the PageUptodate() flag. In this case, we will need to turn off 875 * write optimisations that depend on the page contents being correct. 876 */ 877 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode) 878 { 879 if (nfs_have_delegated_attributes(inode)) 880 goto out; 881 if (NFS_I(inode)->cache_validity & (NFS_INO_INVALID_DATA|NFS_INO_REVAL_PAGECACHE)) 882 return false; 883 out: 884 return PageUptodate(page) != 0; 885 } 886 887 /* 888 * Update and possibly write a cached page of an NFS file. 889 * 890 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 891 * things with a page scheduled for an RPC call (e.g. invalidate it). 892 */ 893 int nfs_updatepage(struct file *file, struct page *page, 894 unsigned int offset, unsigned int count) 895 { 896 struct nfs_open_context *ctx = nfs_file_open_context(file); 897 struct inode *inode = page_file_mapping(page)->host; 898 int status = 0; 899 900 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 901 902 dprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n", 903 file->f_path.dentry->d_parent->d_name.name, 904 file->f_path.dentry->d_name.name, count, 905 (long long)(page_file_offset(page) + offset)); 906 907 /* If we're not using byte range locks, and we know the page 908 * is up to date, it may be more efficient to extend the write 909 * to cover the entire page in order to avoid fragmentation 910 * inefficiencies. 911 */ 912 if (nfs_write_pageuptodate(page, inode) && 913 inode->i_flock == NULL && 914 !(file->f_flags & O_DSYNC)) { 915 count = max(count + offset, nfs_page_length(page)); 916 offset = 0; 917 } 918 919 status = nfs_writepage_setup(ctx, page, offset, count); 920 if (status < 0) 921 nfs_set_pageerror(page); 922 else 923 __set_page_dirty_nobuffers(page); 924 925 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 926 status, (long long)i_size_read(inode)); 927 return status; 928 } 929 930 static int flush_task_priority(int how) 931 { 932 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 933 case FLUSH_HIGHPRI: 934 return RPC_PRIORITY_HIGH; 935 case FLUSH_LOWPRI: 936 return RPC_PRIORITY_LOW; 937 } 938 return RPC_PRIORITY_NORMAL; 939 } 940 941 int nfs_initiate_write(struct rpc_clnt *clnt, 942 struct nfs_write_data *data, 943 const struct rpc_call_ops *call_ops, 944 int how, int flags) 945 { 946 struct inode *inode = data->header->inode; 947 int priority = flush_task_priority(how); 948 struct rpc_task *task; 949 struct rpc_message msg = { 950 .rpc_argp = &data->args, 951 .rpc_resp = &data->res, 952 .rpc_cred = data->header->cred, 953 }; 954 struct rpc_task_setup task_setup_data = { 955 .rpc_client = clnt, 956 .task = &data->task, 957 .rpc_message = &msg, 958 .callback_ops = call_ops, 959 .callback_data = data, 960 .workqueue = nfsiod_workqueue, 961 .flags = RPC_TASK_ASYNC | flags, 962 .priority = priority, 963 }; 964 int ret = 0; 965 966 /* Set up the initial task struct. */ 967 NFS_PROTO(inode)->write_setup(data, &msg); 968 969 dprintk("NFS: %5u initiated write call " 970 "(req %s/%lld, %u bytes @ offset %llu)\n", 971 data->task.tk_pid, 972 inode->i_sb->s_id, 973 (long long)NFS_FILEID(inode), 974 data->args.count, 975 (unsigned long long)data->args.offset); 976 977 task = rpc_run_task(&task_setup_data); 978 if (IS_ERR(task)) { 979 ret = PTR_ERR(task); 980 goto out; 981 } 982 if (how & FLUSH_SYNC) { 983 ret = rpc_wait_for_completion_task(task); 984 if (ret == 0) 985 ret = task->tk_status; 986 } 987 rpc_put_task(task); 988 out: 989 return ret; 990 } 991 EXPORT_SYMBOL_GPL(nfs_initiate_write); 992 993 /* 994 * Set up the argument/result storage required for the RPC call. 995 */ 996 static void nfs_write_rpcsetup(struct nfs_write_data *data, 997 unsigned int count, unsigned int offset, 998 int how, struct nfs_commit_info *cinfo) 999 { 1000 struct nfs_page *req = data->header->req; 1001 1002 /* Set up the RPC argument and reply structs 1003 * NB: take care not to mess about with data->commit et al. */ 1004 1005 data->args.fh = NFS_FH(data->header->inode); 1006 data->args.offset = req_offset(req) + offset; 1007 /* pnfs_set_layoutcommit needs this */ 1008 data->mds_offset = data->args.offset; 1009 data->args.pgbase = req->wb_pgbase + offset; 1010 data->args.pages = data->pages.pagevec; 1011 data->args.count = count; 1012 data->args.context = get_nfs_open_context(req->wb_context); 1013 data->args.lock_context = req->wb_lock_context; 1014 data->args.stable = NFS_UNSTABLE; 1015 switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) { 1016 case 0: 1017 break; 1018 case FLUSH_COND_STABLE: 1019 if (nfs_reqs_to_commit(cinfo)) 1020 break; 1021 default: 1022 data->args.stable = NFS_FILE_SYNC; 1023 } 1024 1025 data->res.fattr = &data->fattr; 1026 data->res.count = count; 1027 data->res.verf = &data->verf; 1028 nfs_fattr_init(&data->fattr); 1029 } 1030 1031 static int nfs_do_write(struct nfs_write_data *data, 1032 const struct rpc_call_ops *call_ops, 1033 int how) 1034 { 1035 struct inode *inode = data->header->inode; 1036 1037 return nfs_initiate_write(NFS_CLIENT(inode), data, call_ops, how, 0); 1038 } 1039 1040 static int nfs_do_multiple_writes(struct list_head *head, 1041 const struct rpc_call_ops *call_ops, 1042 int how) 1043 { 1044 struct nfs_write_data *data; 1045 int ret = 0; 1046 1047 while (!list_empty(head)) { 1048 int ret2; 1049 1050 data = list_first_entry(head, struct nfs_write_data, list); 1051 list_del_init(&data->list); 1052 1053 ret2 = nfs_do_write(data, call_ops, how); 1054 if (ret == 0) 1055 ret = ret2; 1056 } 1057 return ret; 1058 } 1059 1060 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1061 * call this on each, which will prepare them to be retried on next 1062 * writeback using standard nfs. 1063 */ 1064 static void nfs_redirty_request(struct nfs_page *req) 1065 { 1066 nfs_mark_request_dirty(req); 1067 nfs_unlock_request(req); 1068 nfs_end_page_writeback(req->wb_page); 1069 nfs_release_request(req); 1070 } 1071 1072 static void nfs_async_write_error(struct list_head *head) 1073 { 1074 struct nfs_page *req; 1075 1076 while (!list_empty(head)) { 1077 req = nfs_list_entry(head->next); 1078 nfs_list_remove_request(req); 1079 nfs_redirty_request(req); 1080 } 1081 } 1082 1083 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1084 .error_cleanup = nfs_async_write_error, 1085 .completion = nfs_write_completion, 1086 }; 1087 1088 static void nfs_flush_error(struct nfs_pageio_descriptor *desc, 1089 struct nfs_pgio_header *hdr) 1090 { 1091 set_bit(NFS_IOHDR_REDO, &hdr->flags); 1092 while (!list_empty(&hdr->rpc_list)) { 1093 struct nfs_write_data *data = list_first_entry(&hdr->rpc_list, 1094 struct nfs_write_data, list); 1095 list_del(&data->list); 1096 nfs_writedata_release(data); 1097 } 1098 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1099 } 1100 1101 /* 1102 * Generate multiple small requests to write out a single 1103 * contiguous dirty area on one page. 1104 */ 1105 static int nfs_flush_multi(struct nfs_pageio_descriptor *desc, 1106 struct nfs_pgio_header *hdr) 1107 { 1108 struct nfs_page *req = hdr->req; 1109 struct page *page = req->wb_page; 1110 struct nfs_write_data *data; 1111 size_t wsize = desc->pg_bsize, nbytes; 1112 unsigned int offset; 1113 int requests = 0; 1114 struct nfs_commit_info cinfo; 1115 1116 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1117 1118 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1119 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo) || 1120 desc->pg_count > wsize)) 1121 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1122 1123 1124 offset = 0; 1125 nbytes = desc->pg_count; 1126 do { 1127 size_t len = min(nbytes, wsize); 1128 1129 data = nfs_writedata_alloc(hdr, 1); 1130 if (!data) { 1131 nfs_flush_error(desc, hdr); 1132 return -ENOMEM; 1133 } 1134 data->pages.pagevec[0] = page; 1135 nfs_write_rpcsetup(data, len, offset, desc->pg_ioflags, &cinfo); 1136 list_add(&data->list, &hdr->rpc_list); 1137 requests++; 1138 nbytes -= len; 1139 offset += len; 1140 } while (nbytes != 0); 1141 nfs_list_remove_request(req); 1142 nfs_list_add_request(req, &hdr->pages); 1143 desc->pg_rpc_callops = &nfs_write_common_ops; 1144 return 0; 1145 } 1146 1147 /* 1148 * Create an RPC task for the given write request and kick it. 1149 * The page must have been locked by the caller. 1150 * 1151 * It may happen that the page we're passed is not marked dirty. 1152 * This is the case if nfs_updatepage detects a conflicting request 1153 * that has been written but not committed. 1154 */ 1155 static int nfs_flush_one(struct nfs_pageio_descriptor *desc, 1156 struct nfs_pgio_header *hdr) 1157 { 1158 struct nfs_page *req; 1159 struct page **pages; 1160 struct nfs_write_data *data; 1161 struct list_head *head = &desc->pg_list; 1162 struct nfs_commit_info cinfo; 1163 1164 data = nfs_writedata_alloc(hdr, nfs_page_array_len(desc->pg_base, 1165 desc->pg_count)); 1166 if (!data) { 1167 nfs_flush_error(desc, hdr); 1168 return -ENOMEM; 1169 } 1170 1171 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1172 pages = data->pages.pagevec; 1173 while (!list_empty(head)) { 1174 req = nfs_list_entry(head->next); 1175 nfs_list_remove_request(req); 1176 nfs_list_add_request(req, &hdr->pages); 1177 *pages++ = req->wb_page; 1178 } 1179 1180 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1181 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) 1182 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1183 1184 /* Set up the argument struct */ 1185 nfs_write_rpcsetup(data, desc->pg_count, 0, desc->pg_ioflags, &cinfo); 1186 list_add(&data->list, &hdr->rpc_list); 1187 desc->pg_rpc_callops = &nfs_write_common_ops; 1188 return 0; 1189 } 1190 1191 int nfs_generic_flush(struct nfs_pageio_descriptor *desc, 1192 struct nfs_pgio_header *hdr) 1193 { 1194 if (desc->pg_bsize < PAGE_CACHE_SIZE) 1195 return nfs_flush_multi(desc, hdr); 1196 return nfs_flush_one(desc, hdr); 1197 } 1198 EXPORT_SYMBOL_GPL(nfs_generic_flush); 1199 1200 static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 1201 { 1202 struct nfs_write_header *whdr; 1203 struct nfs_pgio_header *hdr; 1204 int ret; 1205 1206 whdr = nfs_writehdr_alloc(); 1207 if (!whdr) { 1208 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1209 return -ENOMEM; 1210 } 1211 hdr = &whdr->header; 1212 nfs_pgheader_init(desc, hdr, nfs_writehdr_free); 1213 atomic_inc(&hdr->refcnt); 1214 ret = nfs_generic_flush(desc, hdr); 1215 if (ret == 0) 1216 ret = nfs_do_multiple_writes(&hdr->rpc_list, 1217 desc->pg_rpc_callops, 1218 desc->pg_ioflags); 1219 if (atomic_dec_and_test(&hdr->refcnt)) 1220 hdr->completion_ops->completion(hdr); 1221 return ret; 1222 } 1223 1224 static const struct nfs_pageio_ops nfs_pageio_write_ops = { 1225 .pg_test = nfs_generic_pg_test, 1226 .pg_doio = nfs_generic_pg_writepages, 1227 }; 1228 1229 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1230 struct inode *inode, int ioflags, 1231 const struct nfs_pgio_completion_ops *compl_ops) 1232 { 1233 nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops, compl_ops, 1234 NFS_SERVER(inode)->wsize, ioflags); 1235 } 1236 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1237 1238 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1239 { 1240 pgio->pg_ops = &nfs_pageio_write_ops; 1241 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1242 } 1243 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1244 1245 1246 void nfs_write_prepare(struct rpc_task *task, void *calldata) 1247 { 1248 struct nfs_write_data *data = calldata; 1249 NFS_PROTO(data->header->inode)->write_rpc_prepare(task, data); 1250 } 1251 1252 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1253 { 1254 struct nfs_commit_data *data = calldata; 1255 1256 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1257 } 1258 1259 /* 1260 * Handle a write reply that flushes a whole page. 1261 * 1262 * FIXME: There is an inherent race with invalidate_inode_pages and 1263 * writebacks since the page->count is kept > 1 for as long 1264 * as the page has a write request pending. 1265 */ 1266 static void nfs_writeback_done_common(struct rpc_task *task, void *calldata) 1267 { 1268 struct nfs_write_data *data = calldata; 1269 1270 nfs_writeback_done(task, data); 1271 } 1272 1273 static void nfs_writeback_release_common(void *calldata) 1274 { 1275 struct nfs_write_data *data = calldata; 1276 struct nfs_pgio_header *hdr = data->header; 1277 int status = data->task.tk_status; 1278 1279 if ((status >= 0) && nfs_write_need_commit(data)) { 1280 spin_lock(&hdr->lock); 1281 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) 1282 ; /* Do nothing */ 1283 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) 1284 memcpy(hdr->verf, &data->verf, sizeof(*hdr->verf)); 1285 else if (memcmp(hdr->verf, &data->verf, sizeof(*hdr->verf))) 1286 set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags); 1287 spin_unlock(&hdr->lock); 1288 } 1289 nfs_writedata_release(data); 1290 } 1291 1292 static const struct rpc_call_ops nfs_write_common_ops = { 1293 .rpc_call_prepare = nfs_write_prepare, 1294 .rpc_call_done = nfs_writeback_done_common, 1295 .rpc_release = nfs_writeback_release_common, 1296 }; 1297 1298 1299 /* 1300 * This function is called when the WRITE call is complete. 1301 */ 1302 void nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data) 1303 { 1304 struct nfs_writeargs *argp = &data->args; 1305 struct nfs_writeres *resp = &data->res; 1306 struct inode *inode = data->header->inode; 1307 int status; 1308 1309 dprintk("NFS: %5u nfs_writeback_done (status %d)\n", 1310 task->tk_pid, task->tk_status); 1311 1312 /* 1313 * ->write_done will attempt to use post-op attributes to detect 1314 * conflicting writes by other clients. A strict interpretation 1315 * of close-to-open would allow us to continue caching even if 1316 * another writer had changed the file, but some applications 1317 * depend on tighter cache coherency when writing. 1318 */ 1319 status = NFS_PROTO(inode)->write_done(task, data); 1320 if (status != 0) 1321 return; 1322 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, resp->count); 1323 1324 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1325 if (resp->verf->committed < argp->stable && task->tk_status >= 0) { 1326 /* We tried a write call, but the server did not 1327 * commit data to stable storage even though we 1328 * requested it. 1329 * Note: There is a known bug in Tru64 < 5.0 in which 1330 * the server reports NFS_DATA_SYNC, but performs 1331 * NFS_FILE_SYNC. We therefore implement this checking 1332 * as a dprintk() in order to avoid filling syslog. 1333 */ 1334 static unsigned long complain; 1335 1336 /* Note this will print the MDS for a DS write */ 1337 if (time_before(complain, jiffies)) { 1338 dprintk("NFS: faulty NFS server %s:" 1339 " (committed = %d) != (stable = %d)\n", 1340 NFS_SERVER(inode)->nfs_client->cl_hostname, 1341 resp->verf->committed, argp->stable); 1342 complain = jiffies + 300 * HZ; 1343 } 1344 } 1345 #endif 1346 if (task->tk_status < 0) 1347 nfs_set_pgio_error(data->header, task->tk_status, argp->offset); 1348 else if (resp->count < argp->count) { 1349 static unsigned long complain; 1350 1351 /* This a short write! */ 1352 nfs_inc_stats(inode, NFSIOS_SHORTWRITE); 1353 1354 /* Has the server at least made some progress? */ 1355 if (resp->count == 0) { 1356 if (time_before(complain, jiffies)) { 1357 printk(KERN_WARNING 1358 "NFS: Server wrote zero bytes, expected %u.\n", 1359 argp->count); 1360 complain = jiffies + 300 * HZ; 1361 } 1362 nfs_set_pgio_error(data->header, -EIO, argp->offset); 1363 task->tk_status = -EIO; 1364 return; 1365 } 1366 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1367 if (resp->verf->committed != NFS_UNSTABLE) { 1368 /* Resend from where the server left off */ 1369 data->mds_offset += resp->count; 1370 argp->offset += resp->count; 1371 argp->pgbase += resp->count; 1372 argp->count -= resp->count; 1373 } else { 1374 /* Resend as a stable write in order to avoid 1375 * headaches in the case of a server crash. 1376 */ 1377 argp->stable = NFS_FILE_SYNC; 1378 } 1379 rpc_restart_call_prepare(task); 1380 } 1381 } 1382 1383 1384 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1385 static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait) 1386 { 1387 int ret; 1388 1389 if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags)) 1390 return 1; 1391 if (!may_wait) 1392 return 0; 1393 ret = out_of_line_wait_on_bit_lock(&nfsi->flags, 1394 NFS_INO_COMMIT, 1395 nfs_wait_bit_killable, 1396 TASK_KILLABLE); 1397 return (ret < 0) ? ret : 1; 1398 } 1399 1400 static void nfs_commit_clear_lock(struct nfs_inode *nfsi) 1401 { 1402 clear_bit(NFS_INO_COMMIT, &nfsi->flags); 1403 smp_mb__after_clear_bit(); 1404 wake_up_bit(&nfsi->flags, NFS_INO_COMMIT); 1405 } 1406 1407 void nfs_commitdata_release(struct nfs_commit_data *data) 1408 { 1409 put_nfs_open_context(data->context); 1410 nfs_commit_free(data); 1411 } 1412 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1413 1414 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1415 const struct rpc_call_ops *call_ops, 1416 int how, int flags) 1417 { 1418 struct rpc_task *task; 1419 int priority = flush_task_priority(how); 1420 struct rpc_message msg = { 1421 .rpc_argp = &data->args, 1422 .rpc_resp = &data->res, 1423 .rpc_cred = data->cred, 1424 }; 1425 struct rpc_task_setup task_setup_data = { 1426 .task = &data->task, 1427 .rpc_client = clnt, 1428 .rpc_message = &msg, 1429 .callback_ops = call_ops, 1430 .callback_data = data, 1431 .workqueue = nfsiod_workqueue, 1432 .flags = RPC_TASK_ASYNC | flags, 1433 .priority = priority, 1434 }; 1435 /* Set up the initial task struct. */ 1436 NFS_PROTO(data->inode)->commit_setup(data, &msg); 1437 1438 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid); 1439 1440 task = rpc_run_task(&task_setup_data); 1441 if (IS_ERR(task)) 1442 return PTR_ERR(task); 1443 if (how & FLUSH_SYNC) 1444 rpc_wait_for_completion_task(task); 1445 rpc_put_task(task); 1446 return 0; 1447 } 1448 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1449 1450 /* 1451 * Set up the argument/result storage required for the RPC call. 1452 */ 1453 void nfs_init_commit(struct nfs_commit_data *data, 1454 struct list_head *head, 1455 struct pnfs_layout_segment *lseg, 1456 struct nfs_commit_info *cinfo) 1457 { 1458 struct nfs_page *first = nfs_list_entry(head->next); 1459 struct inode *inode = first->wb_context->dentry->d_inode; 1460 1461 /* Set up the RPC argument and reply structs 1462 * NB: take care not to mess about with data->commit et al. */ 1463 1464 list_splice_init(head, &data->pages); 1465 1466 data->inode = inode; 1467 data->cred = first->wb_context->cred; 1468 data->lseg = lseg; /* reference transferred */ 1469 data->mds_ops = &nfs_commit_ops; 1470 data->completion_ops = cinfo->completion_ops; 1471 data->dreq = cinfo->dreq; 1472 1473 data->args.fh = NFS_FH(data->inode); 1474 /* Note: we always request a commit of the entire inode */ 1475 data->args.offset = 0; 1476 data->args.count = 0; 1477 data->context = get_nfs_open_context(first->wb_context); 1478 data->res.fattr = &data->fattr; 1479 data->res.verf = &data->verf; 1480 nfs_fattr_init(&data->fattr); 1481 } 1482 EXPORT_SYMBOL_GPL(nfs_init_commit); 1483 1484 void nfs_retry_commit(struct list_head *page_list, 1485 struct pnfs_layout_segment *lseg, 1486 struct nfs_commit_info *cinfo) 1487 { 1488 struct nfs_page *req; 1489 1490 while (!list_empty(page_list)) { 1491 req = nfs_list_entry(page_list->next); 1492 nfs_list_remove_request(req); 1493 nfs_mark_request_commit(req, lseg, cinfo); 1494 if (!cinfo->dreq) { 1495 dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 1496 dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 1497 BDI_RECLAIMABLE); 1498 } 1499 nfs_unlock_and_release_request(req); 1500 } 1501 } 1502 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1503 1504 /* 1505 * Commit dirty pages 1506 */ 1507 static int 1508 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1509 struct nfs_commit_info *cinfo) 1510 { 1511 struct nfs_commit_data *data; 1512 1513 data = nfs_commitdata_alloc(); 1514 1515 if (!data) 1516 goto out_bad; 1517 1518 /* Set up the argument struct */ 1519 nfs_init_commit(data, head, NULL, cinfo); 1520 atomic_inc(&cinfo->mds->rpcs_out); 1521 return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops, 1522 how, 0); 1523 out_bad: 1524 nfs_retry_commit(head, NULL, cinfo); 1525 cinfo->completion_ops->error_cleanup(NFS_I(inode)); 1526 return -ENOMEM; 1527 } 1528 1529 /* 1530 * COMMIT call returned 1531 */ 1532 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1533 { 1534 struct nfs_commit_data *data = calldata; 1535 1536 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1537 task->tk_pid, task->tk_status); 1538 1539 /* Call the NFS version-specific code */ 1540 NFS_PROTO(data->inode)->commit_done(task, data); 1541 } 1542 1543 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1544 { 1545 struct nfs_page *req; 1546 int status = data->task.tk_status; 1547 struct nfs_commit_info cinfo; 1548 1549 while (!list_empty(&data->pages)) { 1550 req = nfs_list_entry(data->pages.next); 1551 nfs_list_remove_request(req); 1552 nfs_clear_page_commit(req->wb_page); 1553 1554 dprintk("NFS: commit (%s/%lld %d@%lld)", 1555 req->wb_context->dentry->d_sb->s_id, 1556 (long long)NFS_FILEID(req->wb_context->dentry->d_inode), 1557 req->wb_bytes, 1558 (long long)req_offset(req)); 1559 if (status < 0) { 1560 nfs_context_set_write_error(req->wb_context, status); 1561 nfs_inode_remove_request(req); 1562 dprintk(", error = %d\n", status); 1563 goto next; 1564 } 1565 1566 /* Okay, COMMIT succeeded, apparently. Check the verifier 1567 * returned by the server against all stored verfs. */ 1568 if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) { 1569 /* We have a match */ 1570 nfs_inode_remove_request(req); 1571 dprintk(" OK\n"); 1572 goto next; 1573 } 1574 /* We have a mismatch. Write the page again */ 1575 dprintk(" mismatch\n"); 1576 nfs_mark_request_dirty(req); 1577 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags); 1578 next: 1579 nfs_unlock_and_release_request(req); 1580 } 1581 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1582 if (atomic_dec_and_test(&cinfo.mds->rpcs_out)) 1583 nfs_commit_clear_lock(NFS_I(data->inode)); 1584 } 1585 1586 static void nfs_commit_release(void *calldata) 1587 { 1588 struct nfs_commit_data *data = calldata; 1589 1590 data->completion_ops->completion(data); 1591 nfs_commitdata_release(calldata); 1592 } 1593 1594 static const struct rpc_call_ops nfs_commit_ops = { 1595 .rpc_call_prepare = nfs_commit_prepare, 1596 .rpc_call_done = nfs_commit_done, 1597 .rpc_release = nfs_commit_release, 1598 }; 1599 1600 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1601 .completion = nfs_commit_release_pages, 1602 .error_cleanup = nfs_commit_clear_lock, 1603 }; 1604 1605 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1606 int how, struct nfs_commit_info *cinfo) 1607 { 1608 int status; 1609 1610 status = pnfs_commit_list(inode, head, how, cinfo); 1611 if (status == PNFS_NOT_ATTEMPTED) 1612 status = nfs_commit_list(inode, head, how, cinfo); 1613 return status; 1614 } 1615 1616 int nfs_commit_inode(struct inode *inode, int how) 1617 { 1618 LIST_HEAD(head); 1619 struct nfs_commit_info cinfo; 1620 int may_wait = how & FLUSH_SYNC; 1621 int res; 1622 1623 res = nfs_commit_set_lock(NFS_I(inode), may_wait); 1624 if (res <= 0) 1625 goto out_mark_dirty; 1626 nfs_init_cinfo_from_inode(&cinfo, inode); 1627 res = nfs_scan_commit(inode, &head, &cinfo); 1628 if (res) { 1629 int error; 1630 1631 error = nfs_generic_commit_list(inode, &head, how, &cinfo); 1632 if (error < 0) 1633 return error; 1634 if (!may_wait) 1635 goto out_mark_dirty; 1636 error = wait_on_bit(&NFS_I(inode)->flags, 1637 NFS_INO_COMMIT, 1638 nfs_wait_bit_killable, 1639 TASK_KILLABLE); 1640 if (error < 0) 1641 return error; 1642 } else 1643 nfs_commit_clear_lock(NFS_I(inode)); 1644 return res; 1645 /* Note: If we exit without ensuring that the commit is complete, 1646 * we must mark the inode as dirty. Otherwise, future calls to 1647 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure 1648 * that the data is on the disk. 1649 */ 1650 out_mark_dirty: 1651 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1652 return res; 1653 } 1654 1655 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1656 { 1657 struct nfs_inode *nfsi = NFS_I(inode); 1658 int flags = FLUSH_SYNC; 1659 int ret = 0; 1660 1661 /* no commits means nothing needs to be done */ 1662 if (!nfsi->commit_info.ncommit) 1663 return ret; 1664 1665 if (wbc->sync_mode == WB_SYNC_NONE) { 1666 /* Don't commit yet if this is a non-blocking flush and there 1667 * are a lot of outstanding writes for this mapping. 1668 */ 1669 if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1)) 1670 goto out_mark_dirty; 1671 1672 /* don't wait for the COMMIT response */ 1673 flags = 0; 1674 } 1675 1676 ret = nfs_commit_inode(inode, flags); 1677 if (ret >= 0) { 1678 if (wbc->sync_mode == WB_SYNC_NONE) { 1679 if (ret < wbc->nr_to_write) 1680 wbc->nr_to_write -= ret; 1681 else 1682 wbc->nr_to_write = 0; 1683 } 1684 return 0; 1685 } 1686 out_mark_dirty: 1687 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1688 return ret; 1689 } 1690 #else 1691 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1692 { 1693 return 0; 1694 } 1695 #endif 1696 1697 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1698 { 1699 return nfs_commit_unstable_pages(inode, wbc); 1700 } 1701 EXPORT_SYMBOL_GPL(nfs_write_inode); 1702 1703 /* 1704 * flush the inode to disk. 1705 */ 1706 int nfs_wb_all(struct inode *inode) 1707 { 1708 struct writeback_control wbc = { 1709 .sync_mode = WB_SYNC_ALL, 1710 .nr_to_write = LONG_MAX, 1711 .range_start = 0, 1712 .range_end = LLONG_MAX, 1713 }; 1714 1715 return sync_inode(inode, &wbc); 1716 } 1717 EXPORT_SYMBOL_GPL(nfs_wb_all); 1718 1719 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 1720 { 1721 struct nfs_page *req; 1722 int ret = 0; 1723 1724 for (;;) { 1725 wait_on_page_writeback(page); 1726 req = nfs_page_find_request(page); 1727 if (req == NULL) 1728 break; 1729 if (nfs_lock_request(req)) { 1730 nfs_clear_request_commit(req); 1731 nfs_inode_remove_request(req); 1732 /* 1733 * In case nfs_inode_remove_request has marked the 1734 * page as being dirty 1735 */ 1736 cancel_dirty_page(page, PAGE_CACHE_SIZE); 1737 nfs_unlock_and_release_request(req); 1738 break; 1739 } 1740 ret = nfs_wait_on_request(req); 1741 nfs_release_request(req); 1742 if (ret < 0) 1743 break; 1744 } 1745 return ret; 1746 } 1747 1748 /* 1749 * Write back all requests on one page - we do this before reading it. 1750 */ 1751 int nfs_wb_page(struct inode *inode, struct page *page) 1752 { 1753 loff_t range_start = page_file_offset(page); 1754 loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1); 1755 struct writeback_control wbc = { 1756 .sync_mode = WB_SYNC_ALL, 1757 .nr_to_write = 0, 1758 .range_start = range_start, 1759 .range_end = range_end, 1760 }; 1761 int ret; 1762 1763 for (;;) { 1764 wait_on_page_writeback(page); 1765 if (clear_page_dirty_for_io(page)) { 1766 ret = nfs_writepage_locked(page, &wbc); 1767 if (ret < 0) 1768 goto out_error; 1769 continue; 1770 } 1771 if (!PagePrivate(page)) 1772 break; 1773 ret = nfs_commit_inode(inode, FLUSH_SYNC); 1774 if (ret < 0) 1775 goto out_error; 1776 } 1777 return 0; 1778 out_error: 1779 return ret; 1780 } 1781 1782 #ifdef CONFIG_MIGRATION 1783 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 1784 struct page *page, enum migrate_mode mode) 1785 { 1786 /* 1787 * If PagePrivate is set, then the page is currently associated with 1788 * an in-progress read or write request. Don't try to migrate it. 1789 * 1790 * FIXME: we could do this in principle, but we'll need a way to ensure 1791 * that we can safely release the inode reference while holding 1792 * the page lock. 1793 */ 1794 if (PagePrivate(page)) 1795 return -EBUSY; 1796 1797 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 1798 return -EBUSY; 1799 1800 return migrate_page(mapping, newpage, page, mode); 1801 } 1802 #endif 1803 1804 int __init nfs_init_writepagecache(void) 1805 { 1806 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 1807 sizeof(struct nfs_write_header), 1808 0, SLAB_HWCACHE_ALIGN, 1809 NULL); 1810 if (nfs_wdata_cachep == NULL) 1811 return -ENOMEM; 1812 1813 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 1814 nfs_wdata_cachep); 1815 if (nfs_wdata_mempool == NULL) 1816 goto out_destroy_write_cache; 1817 1818 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 1819 sizeof(struct nfs_commit_data), 1820 0, SLAB_HWCACHE_ALIGN, 1821 NULL); 1822 if (nfs_cdata_cachep == NULL) 1823 goto out_destroy_write_mempool; 1824 1825 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 1826 nfs_cdata_cachep); 1827 if (nfs_commit_mempool == NULL) 1828 goto out_destroy_commit_cache; 1829 1830 /* 1831 * NFS congestion size, scale with available memory. 1832 * 1833 * 64MB: 8192k 1834 * 128MB: 11585k 1835 * 256MB: 16384k 1836 * 512MB: 23170k 1837 * 1GB: 32768k 1838 * 2GB: 46340k 1839 * 4GB: 65536k 1840 * 8GB: 92681k 1841 * 16GB: 131072k 1842 * 1843 * This allows larger machines to have larger/more transfers. 1844 * Limit the default to 256M 1845 */ 1846 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); 1847 if (nfs_congestion_kb > 256*1024) 1848 nfs_congestion_kb = 256*1024; 1849 1850 return 0; 1851 1852 out_destroy_commit_cache: 1853 kmem_cache_destroy(nfs_cdata_cachep); 1854 out_destroy_write_mempool: 1855 mempool_destroy(nfs_wdata_mempool); 1856 out_destroy_write_cache: 1857 kmem_cache_destroy(nfs_wdata_cachep); 1858 return -ENOMEM; 1859 } 1860 1861 void nfs_destroy_writepagecache(void) 1862 { 1863 mempool_destroy(nfs_commit_mempool); 1864 kmem_cache_destroy(nfs_cdata_cachep); 1865 mempool_destroy(nfs_wdata_mempool); 1866 kmem_cache_destroy(nfs_wdata_cachep); 1867 } 1868 1869