1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode->i_state & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 436 if (err != -EBUSY) { 437 dst->seqid = lo->plh_stateid.seqid; 438 *dst_range = range; 439 ret = true; 440 } 441 } 442 out: 443 spin_unlock(&inode->i_lock); 444 pnfs_free_lseg_list(&head); 445 return ret; 446 } 447 448 /* 449 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 450 * 451 * In order to continue using the pnfs_layout_hdr, a full recovery 452 * is required. 453 * Note that caller must hold inode->i_lock. 454 */ 455 int 456 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 457 struct list_head *lseg_list) 458 { 459 struct pnfs_layout_range range = { 460 .iomode = IOMODE_ANY, 461 .offset = 0, 462 .length = NFS4_MAX_UINT64, 463 }; 464 struct pnfs_layout_segment *lseg, *next; 465 466 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 467 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 468 pnfs_clear_lseg_state(lseg, lseg_list); 469 pnfs_clear_layoutreturn_info(lo); 470 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 471 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 472 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 473 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 474 pnfs_clear_layoutreturn_waitbit(lo); 475 return !list_empty(&lo->plh_segs); 476 } 477 478 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 479 struct list_head *lseg_list, 480 enum pnfs_iomode iomode, u32 seq) 481 { 482 struct pnfs_layout_range range = { 483 .iomode = iomode, 484 .length = NFS4_MAX_UINT64, 485 }; 486 487 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); 488 } 489 490 static int 491 pnfs_iomode_to_fail_bit(u32 iomode) 492 { 493 return iomode == IOMODE_RW ? 494 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 495 } 496 497 static void 498 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 499 { 500 lo->plh_retry_timestamp = jiffies; 501 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 502 refcount_inc(&lo->plh_refcount); 503 } 504 505 static void 506 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 507 { 508 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 509 refcount_dec(&lo->plh_refcount); 510 } 511 512 static void 513 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 514 { 515 struct inode *inode = lo->plh_inode; 516 struct pnfs_layout_range range = { 517 .iomode = iomode, 518 .offset = 0, 519 .length = NFS4_MAX_UINT64, 520 }; 521 LIST_HEAD(head); 522 523 spin_lock(&inode->i_lock); 524 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 525 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 526 spin_unlock(&inode->i_lock); 527 pnfs_free_lseg_list(&head); 528 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 529 iomode == IOMODE_RW ? "RW" : "READ"); 530 } 531 532 static bool 533 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 534 { 535 unsigned long start, end; 536 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 537 538 if (test_bit(fail_bit, &lo->plh_flags) == 0) 539 return false; 540 end = jiffies; 541 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 542 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 543 /* It is time to retry the failed layoutgets */ 544 pnfs_layout_clear_fail_bit(lo, fail_bit); 545 return false; 546 } 547 return true; 548 } 549 550 static void 551 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 552 const struct pnfs_layout_range *range, 553 const nfs4_stateid *stateid) 554 { 555 INIT_LIST_HEAD(&lseg->pls_list); 556 INIT_LIST_HEAD(&lseg->pls_lc_list); 557 INIT_LIST_HEAD(&lseg->pls_commits); 558 refcount_set(&lseg->pls_refcount, 1); 559 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 560 lseg->pls_layout = lo; 561 lseg->pls_range = *range; 562 lseg->pls_seq = be32_to_cpu(stateid->seqid); 563 } 564 565 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 566 { 567 if (lseg != NULL) { 568 struct inode *inode = lseg->pls_layout->plh_inode; 569 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 570 } 571 } 572 573 static void 574 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 575 struct pnfs_layout_segment *lseg) 576 { 577 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 578 list_del_init(&lseg->pls_list); 579 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 580 refcount_dec(&lo->plh_refcount); 581 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 582 return; 583 if (list_empty(&lo->plh_segs) && 584 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 585 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 586 if (atomic_read(&lo->plh_outstanding) == 0) 587 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 588 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 589 } 590 } 591 592 static bool 593 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 594 struct pnfs_layout_segment *lseg) 595 { 596 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 597 pnfs_layout_is_valid(lo)) { 598 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 599 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 600 return true; 601 } 602 return false; 603 } 604 605 void 606 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 607 { 608 struct pnfs_layout_hdr *lo; 609 struct inode *inode; 610 611 if (!lseg) 612 return; 613 614 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 615 refcount_read(&lseg->pls_refcount), 616 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 617 618 lo = lseg->pls_layout; 619 inode = lo->plh_inode; 620 621 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 622 pnfs_get_layout_hdr(lo); 623 pnfs_layout_remove_lseg(lo, lseg); 624 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 625 lseg = NULL; 626 spin_unlock(&inode->i_lock); 627 pnfs_free_lseg(lseg); 628 pnfs_put_layout_hdr(lo); 629 } 630 } 631 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 632 633 /* 634 * is l2 fully contained in l1? 635 * start1 end1 636 * [----------------------------------) 637 * start2 end2 638 * [----------------) 639 */ 640 static bool 641 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 642 const struct pnfs_layout_range *l2) 643 { 644 u64 start1 = l1->offset; 645 u64 end1 = pnfs_end_offset(start1, l1->length); 646 u64 start2 = l2->offset; 647 u64 end2 = pnfs_end_offset(start2, l2->length); 648 649 return (start1 <= start2) && (end1 >= end2); 650 } 651 652 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 653 struct list_head *tmp_list) 654 { 655 if (!refcount_dec_and_test(&lseg->pls_refcount)) 656 return false; 657 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 658 list_add(&lseg->pls_list, tmp_list); 659 return true; 660 } 661 662 /* Returns 1 if lseg is removed from list, 0 otherwise */ 663 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 664 struct list_head *tmp_list) 665 { 666 int rv = 0; 667 668 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 669 /* Remove the reference keeping the lseg in the 670 * list. It will now be removed when all 671 * outstanding io is finished. 672 */ 673 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 674 refcount_read(&lseg->pls_refcount)); 675 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 676 rv = 1; 677 } 678 return rv; 679 } 680 681 static bool 682 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 683 const struct pnfs_layout_range *recall_range) 684 { 685 return (recall_range->iomode == IOMODE_ANY || 686 lseg_range->iomode == recall_range->iomode) && 687 pnfs_lseg_range_intersecting(lseg_range, recall_range); 688 } 689 690 static bool 691 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 692 const struct pnfs_layout_range *recall_range, 693 u32 seq) 694 { 695 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 696 return false; 697 if (recall_range == NULL) 698 return true; 699 return pnfs_should_free_range(&lseg->pls_range, recall_range); 700 } 701 702 /** 703 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 704 * @lo: layout header containing the lsegs 705 * @tmp_list: list head where doomed lsegs should go 706 * @recall_range: optional recall range argument to match (may be NULL) 707 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 708 * 709 * Walk the list of lsegs in the layout header, and tear down any that should 710 * be destroyed. If "recall_range" is specified then the segment must match 711 * that range. If "seq" is non-zero, then only match segments that were handed 712 * out at or before that sequence. 713 * 714 * Returns number of matching invalid lsegs remaining in list after scanning 715 * it and purging them. 716 */ 717 int 718 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 719 struct list_head *tmp_list, 720 const struct pnfs_layout_range *recall_range, 721 u32 seq) 722 { 723 struct pnfs_layout_segment *lseg, *next; 724 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 725 int remaining = 0; 726 727 dprintk("%s:Begin lo %p\n", __func__, lo); 728 729 if (list_empty(&lo->plh_segs)) 730 return 0; 731 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 732 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 733 dprintk("%s: freeing lseg %p iomode %d seq %u " 734 "offset %llu length %llu\n", __func__, 735 lseg, lseg->pls_range.iomode, lseg->pls_seq, 736 lseg->pls_range.offset, lseg->pls_range.length); 737 if (mark_lseg_invalid(lseg, tmp_list)) 738 continue; 739 remaining++; 740 pnfs_lseg_cancel_io(server, lseg); 741 } 742 dprintk("%s:Return %i\n", __func__, remaining); 743 return remaining; 744 } 745 746 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 747 { 748 struct pnfs_layout_segment *lseg; 749 750 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 751 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 752 } 753 754 static void 755 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 756 struct list_head *free_me, 757 const struct pnfs_layout_range *range, 758 u32 seq) 759 { 760 struct pnfs_layout_segment *lseg, *next; 761 762 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 763 if (pnfs_match_lseg_recall(lseg, range, seq)) 764 list_move_tail(&lseg->pls_list, free_me); 765 } 766 } 767 768 /* note free_me must contain lsegs from a single layout_hdr */ 769 void 770 pnfs_free_lseg_list(struct list_head *free_me) 771 { 772 struct pnfs_layout_segment *lseg, *tmp; 773 774 if (list_empty(free_me)) 775 return; 776 777 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 778 list_del(&lseg->pls_list); 779 pnfs_free_lseg(lseg); 780 } 781 } 782 783 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 784 { 785 struct pnfs_layout_hdr *lo; 786 LIST_HEAD(tmp_list); 787 788 spin_lock(&nfsi->vfs_inode.i_lock); 789 lo = nfsi->layout; 790 if (lo) { 791 pnfs_get_layout_hdr(lo); 792 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 793 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 794 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 795 spin_unlock(&nfsi->vfs_inode.i_lock); 796 pnfs_free_lseg_list(&tmp_list); 797 nfs_commit_inode(&nfsi->vfs_inode, 0); 798 pnfs_put_layout_hdr(lo); 799 } else 800 spin_unlock(&nfsi->vfs_inode.i_lock); 801 return lo; 802 } 803 804 void pnfs_destroy_layout(struct nfs_inode *nfsi) 805 { 806 __pnfs_destroy_layout(nfsi); 807 } 808 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 809 810 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 811 { 812 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 813 struct inode *inode = &nfsi->vfs_inode; 814 815 if (lo) { 816 spin_lock(&inode->i_lock); 817 wait_var_event_spinlock(lo, nfsi->layout != lo, 818 &inode->i_lock); 819 spin_unlock(&inode->i_lock); 820 } 821 } 822 823 static bool 824 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 825 struct list_head *layout_list) 826 { 827 struct pnfs_layout_hdr *lo; 828 bool ret = false; 829 830 spin_lock(&inode->i_lock); 831 lo = NFS_I(inode)->layout; 832 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 833 pnfs_get_layout_hdr(lo); 834 list_add(&lo->plh_bulk_destroy, layout_list); 835 ret = true; 836 } 837 spin_unlock(&inode->i_lock); 838 return ret; 839 } 840 841 /* Caller must hold rcu_read_lock and clp->cl_lock */ 842 static int 843 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 844 struct nfs_server *server, 845 struct list_head *layout_list) 846 __must_hold(&clp->cl_lock) 847 __must_hold(RCU) 848 { 849 struct pnfs_layout_hdr *lo, *next; 850 struct inode *inode; 851 852 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 853 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 854 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 855 !list_empty(&lo->plh_bulk_destroy)) 856 continue; 857 /* If the sb is being destroyed, just bail */ 858 if (!nfs_sb_active(server->super)) 859 break; 860 inode = pnfs_grab_inode_layout_hdr(lo); 861 if (inode != NULL) { 862 if (pnfs_layout_add_bulk_destroy_list(inode, 863 layout_list)) 864 continue; 865 rcu_read_unlock(); 866 spin_unlock(&clp->cl_lock); 867 iput(inode); 868 } else { 869 rcu_read_unlock(); 870 spin_unlock(&clp->cl_lock); 871 } 872 nfs_sb_deactive(server->super); 873 spin_lock(&clp->cl_lock); 874 rcu_read_lock(); 875 return -EAGAIN; 876 } 877 return 0; 878 } 879 880 static int 881 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 882 enum pnfs_layout_destroy_mode mode) 883 { 884 struct pnfs_layout_hdr *lo; 885 struct inode *inode; 886 LIST_HEAD(lseg_list); 887 int ret = 0; 888 889 while (!list_empty(layout_list)) { 890 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 891 plh_bulk_destroy); 892 dprintk("%s freeing layout for inode %lu\n", __func__, 893 lo->plh_inode->i_ino); 894 inode = lo->plh_inode; 895 896 pnfs_layoutcommit_inode(inode, false); 897 898 spin_lock(&inode->i_lock); 899 list_del_init(&lo->plh_bulk_destroy); 900 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 901 pnfs_mark_layout_stateid_return(lo, &lseg_list, 902 IOMODE_ANY, 0); 903 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 904 if (mode == PNFS_LAYOUT_BULK_RETURN) 905 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 906 ret = -EAGAIN; 907 } 908 spin_unlock(&inode->i_lock); 909 pnfs_free_lseg_list(&lseg_list); 910 /* Free all lsegs that are attached to commit buckets */ 911 nfs_commit_inode(inode, 0); 912 pnfs_put_layout_hdr(lo); 913 nfs_iput_and_deactive(inode); 914 } 915 return ret; 916 } 917 918 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 919 enum pnfs_layout_destroy_mode mode) 920 { 921 struct nfs_server *server; 922 LIST_HEAD(layout_list); 923 924 spin_lock(&clp->cl_lock); 925 rcu_read_lock(); 926 restart: 927 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 928 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 929 continue; 930 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 931 server, 932 &layout_list) != 0) 933 goto restart; 934 } 935 rcu_read_unlock(); 936 spin_unlock(&clp->cl_lock); 937 938 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 939 } 940 941 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 942 struct list_head *list) 943 { 944 struct nfs_server *server; 945 946 spin_lock(&clp->cl_lock); 947 rcu_read_lock(); 948 restart: 949 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 950 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 951 list) != 0) 952 goto restart; 953 } 954 rcu_read_unlock(); 955 spin_unlock(&clp->cl_lock); 956 } 957 958 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 959 struct list_head *list, 960 enum pnfs_layout_destroy_mode mode) 961 { 962 pnfs_layout_build_destroy_list_byclient(clp, list); 963 return pnfs_layout_free_bulk_destroy_list(list, mode); 964 } 965 966 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 967 enum pnfs_layout_destroy_mode mode) 968 { 969 LIST_HEAD(layout_list); 970 971 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 972 } 973 974 /* 975 * Called by the state manager to remove all layouts established under an 976 * expired lease. 977 */ 978 void 979 pnfs_destroy_all_layouts(struct nfs_client *clp) 980 { 981 nfs4_deviceid_mark_client_invalid(clp); 982 nfs4_deviceid_purge_client(clp); 983 984 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 985 } 986 987 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 988 struct list_head *list) 989 { 990 struct nfs_server *server; 991 992 spin_lock(&clp->cl_lock); 993 rcu_read_lock(); 994 restart: 995 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 996 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 997 continue; 998 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 999 list) != 0) 1000 goto restart; 1001 } 1002 rcu_read_unlock(); 1003 spin_unlock(&clp->cl_lock); 1004 } 1005 1006 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1007 { 1008 struct pnfs_layout_hdr *lo; 1009 struct nfs_server *server; 1010 int ret; 1011 1012 list_for_each_entry(lo, list, plh_bulk_destroy) { 1013 server = NFS_SERVER(lo->plh_inode); 1014 ret = pnfs_layout_return_on_reboot(lo); 1015 switch (ret) { 1016 case 0: 1017 continue; 1018 case -NFS4ERR_BAD_STATEID: 1019 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1020 break; 1021 case -NFS4ERR_NO_GRACE: 1022 break; 1023 default: 1024 goto err; 1025 } 1026 break; 1027 } 1028 return 0; 1029 err: 1030 return ret; 1031 } 1032 1033 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1034 { 1035 LIST_HEAD(list); 1036 int ret = 0, ret2; 1037 1038 pnfs_layout_build_recover_list_byclient(clp, &list); 1039 if (!list_empty(&list)) 1040 ret = pnfs_layout_bulk_list_reboot(&list); 1041 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1042 PNFS_LAYOUT_INVALIDATE); 1043 if (!ret) 1044 ret = ret2; 1045 return (ret == 0) ? 0 : -EAGAIN; 1046 } 1047 1048 static void 1049 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1050 { 1051 const struct cred *old; 1052 1053 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1054 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1055 put_cred(old); 1056 } 1057 } 1058 1059 /* update lo->plh_stateid with new if is more recent */ 1060 void 1061 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1062 const struct cred *cred, bool update_barrier) 1063 { 1064 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1065 u32 newseq = be32_to_cpu(new->seqid); 1066 1067 if (!pnfs_layout_is_valid(lo)) { 1068 pnfs_set_layout_cred(lo, cred); 1069 nfs4_stateid_copy(&lo->plh_stateid, new); 1070 lo->plh_barrier = newseq; 1071 pnfs_clear_layoutreturn_info(lo); 1072 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1073 return; 1074 } 1075 1076 if (pnfs_seqid_is_newer(newseq, oldseq)) 1077 nfs4_stateid_copy(&lo->plh_stateid, new); 1078 1079 if (update_barrier) { 1080 pnfs_barrier_update(lo, newseq); 1081 return; 1082 } 1083 /* 1084 * Because of wraparound, we want to keep the barrier 1085 * "close" to the current seqids. We really only want to 1086 * get here from a layoutget call. 1087 */ 1088 if (atomic_read(&lo->plh_outstanding) == 1) 1089 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1090 } 1091 1092 static bool 1093 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1094 const nfs4_stateid *stateid) 1095 { 1096 u32 seqid = be32_to_cpu(stateid->seqid); 1097 1098 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1099 } 1100 1101 /* lget is set to 1 if called from inside send_layoutget call chain */ 1102 static bool 1103 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1104 { 1105 return lo->plh_block_lgets || 1106 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1107 } 1108 1109 static struct nfs_server * 1110 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1111 { 1112 struct nfs_server *server; 1113 1114 if (inode) { 1115 server = NFS_SERVER(inode); 1116 } else { 1117 struct dentry *parent_dir = dget_parent(ctx->dentry); 1118 server = NFS_SERVER(parent_dir->d_inode); 1119 dput(parent_dir); 1120 } 1121 return server; 1122 } 1123 1124 static void nfs4_free_pages(struct page **pages, size_t size) 1125 { 1126 int i; 1127 1128 if (!pages) 1129 return; 1130 1131 for (i = 0; i < size; i++) { 1132 if (!pages[i]) 1133 break; 1134 __free_page(pages[i]); 1135 } 1136 kfree(pages); 1137 } 1138 1139 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1140 { 1141 struct page **pages; 1142 int i; 1143 1144 pages = kmalloc_array(size, sizeof(struct page *), gfp_flags); 1145 if (!pages) { 1146 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1147 return NULL; 1148 } 1149 1150 for (i = 0; i < size; i++) { 1151 pages[i] = alloc_page(gfp_flags); 1152 if (!pages[i]) { 1153 dprintk("%s: failed to allocate page\n", __func__); 1154 nfs4_free_pages(pages, i); 1155 return NULL; 1156 } 1157 } 1158 1159 return pages; 1160 } 1161 1162 static struct nfs4_layoutget * 1163 pnfs_alloc_init_layoutget_args(struct inode *ino, 1164 struct nfs_open_context *ctx, 1165 const nfs4_stateid *stateid, 1166 const struct pnfs_layout_range *range, 1167 gfp_t gfp_flags) 1168 { 1169 struct nfs_server *server = pnfs_find_server(ino, ctx); 1170 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1171 size_t max_pages = max_response_pages(server); 1172 struct nfs4_layoutget *lgp; 1173 1174 dprintk("--> %s\n", __func__); 1175 1176 lgp = kzalloc(sizeof(*lgp), gfp_flags); 1177 if (lgp == NULL) 1178 return NULL; 1179 1180 if (max_reply_sz) { 1181 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1182 if (npages < max_pages) 1183 max_pages = npages; 1184 } 1185 1186 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1187 if (!lgp->args.layout.pages) { 1188 kfree(lgp); 1189 return NULL; 1190 } 1191 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1192 lgp->res.layoutp = &lgp->args.layout; 1193 1194 /* Don't confuse uninitialised result and success */ 1195 lgp->res.status = -NFS4ERR_DELAY; 1196 1197 lgp->args.minlength = PAGE_SIZE; 1198 if (lgp->args.minlength > range->length) 1199 lgp->args.minlength = range->length; 1200 if (ino) { 1201 loff_t i_size = i_size_read(ino); 1202 1203 if (range->iomode == IOMODE_READ) { 1204 if (range->offset >= i_size) 1205 lgp->args.minlength = 0; 1206 else if (i_size - range->offset < lgp->args.minlength) 1207 lgp->args.minlength = i_size - range->offset; 1208 } 1209 } 1210 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 1211 pnfs_copy_range(&lgp->args.range, range); 1212 lgp->args.type = server->pnfs_curr_ld->id; 1213 lgp->args.inode = ino; 1214 lgp->args.ctx = get_nfs_open_context(ctx); 1215 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1216 lgp->gfp_flags = gfp_flags; 1217 lgp->cred = ctx->cred; 1218 return lgp; 1219 } 1220 1221 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1222 { 1223 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1224 1225 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1226 pnfs_put_layout_hdr(lgp->lo); 1227 put_nfs_open_context(lgp->args.ctx); 1228 kfree(lgp); 1229 } 1230 1231 static void pnfs_clear_layoutcommit(struct inode *inode, 1232 struct list_head *head) 1233 { 1234 struct nfs_inode *nfsi = NFS_I(inode); 1235 struct pnfs_layout_segment *lseg, *tmp; 1236 1237 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1238 return; 1239 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1240 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1241 continue; 1242 pnfs_lseg_dec_and_remove_zero(lseg, head); 1243 } 1244 } 1245 1246 static void 1247 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1248 const nfs4_stateid *arg_stateid, 1249 const struct pnfs_layout_range *range, 1250 struct list_head *freeme) 1251 { 1252 if (pnfs_layout_is_valid(lo) && 1253 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1254 pnfs_reset_return_info(lo); 1255 else 1256 pnfs_mark_layout_stateid_invalid(lo, freeme); 1257 pnfs_clear_layoutreturn_waitbit(lo); 1258 } 1259 1260 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1261 const nfs4_stateid *arg_stateid, 1262 const struct pnfs_layout_range *range) 1263 { 1264 struct inode *inode = lo->plh_inode; 1265 LIST_HEAD(freeme); 1266 1267 spin_lock(&inode->i_lock); 1268 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1269 spin_unlock(&inode->i_lock); 1270 pnfs_free_lseg_list(&freeme); 1271 } 1272 1273 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1274 const nfs4_stateid *arg_stateid, 1275 const struct pnfs_layout_range *range, 1276 const nfs4_stateid *stateid) 1277 { 1278 struct inode *inode = lo->plh_inode; 1279 LIST_HEAD(freeme); 1280 1281 spin_lock(&inode->i_lock); 1282 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1283 goto out_unlock; 1284 if (stateid && pnfs_layout_is_valid(lo)) { 1285 u32 seq = be32_to_cpu(arg_stateid->seqid); 1286 1287 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1288 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1289 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1290 pnfs_reset_return_info(lo); 1291 } else 1292 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1293 out_unlock: 1294 pnfs_clear_layoutreturn_waitbit(lo); 1295 spin_unlock(&inode->i_lock); 1296 pnfs_free_lseg_list(&freeme); 1297 1298 } 1299 1300 static bool 1301 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1302 nfs4_stateid *stateid, 1303 const struct cred **cred, 1304 enum pnfs_iomode *iomode) 1305 { 1306 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1307 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1308 return false; 1309 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1310 return false; 1311 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1312 pnfs_get_layout_hdr(lo); 1313 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1314 *cred = get_cred(lo->plh_lc_cred); 1315 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1316 if (lo->plh_return_seq != 0) 1317 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1318 if (iomode != NULL) 1319 *iomode = lo->plh_return_iomode; 1320 pnfs_clear_layoutreturn_info(lo); 1321 } else if (iomode != NULL) 1322 *iomode = IOMODE_ANY; 1323 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1324 return true; 1325 } 1326 1327 static void 1328 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1329 struct pnfs_layout_hdr *lo, 1330 const nfs4_stateid *stateid, 1331 enum pnfs_iomode iomode) 1332 { 1333 struct inode *inode = lo->plh_inode; 1334 1335 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1336 args->inode = inode; 1337 args->range.iomode = iomode; 1338 args->range.offset = 0; 1339 args->range.length = NFS4_MAX_UINT64; 1340 args->layout = lo; 1341 nfs4_stateid_copy(&args->stateid, stateid); 1342 } 1343 1344 static int 1345 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1346 const nfs4_stateid *stateid, 1347 const struct cred **pcred, 1348 enum pnfs_iomode iomode, 1349 unsigned int flags) 1350 { 1351 struct inode *ino = lo->plh_inode; 1352 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1353 struct nfs4_layoutreturn *lrp; 1354 const struct cred *cred = *pcred; 1355 int status = 0; 1356 1357 *pcred = NULL; 1358 lrp = kzalloc(sizeof(*lrp), nfs_io_gfp_mask()); 1359 if (unlikely(lrp == NULL)) { 1360 status = -ENOMEM; 1361 spin_lock(&ino->i_lock); 1362 pnfs_clear_layoutreturn_waitbit(lo); 1363 spin_unlock(&ino->i_lock); 1364 put_cred(cred); 1365 pnfs_put_layout_hdr(lo); 1366 goto out; 1367 } 1368 1369 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1370 lrp->args.ld_private = &lrp->ld_private; 1371 lrp->clp = NFS_SERVER(ino)->nfs_client; 1372 lrp->cred = cred; 1373 if (ld->prepare_layoutreturn) 1374 ld->prepare_layoutreturn(&lrp->args); 1375 1376 status = nfs4_proc_layoutreturn(lrp, flags); 1377 out: 1378 dprintk("<-- %s status: %d\n", __func__, status); 1379 return status; 1380 } 1381 1382 /* Return true if layoutreturn is needed */ 1383 static bool 1384 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1385 { 1386 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1387 return false; 1388 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1389 lo->plh_return_iomode, 1390 lo->plh_return_seq) != EBUSY; 1391 } 1392 1393 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1394 { 1395 struct inode *inode= lo->plh_inode; 1396 1397 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1398 return; 1399 spin_lock(&inode->i_lock); 1400 if (pnfs_layout_need_return(lo)) { 1401 const struct cred *cred; 1402 nfs4_stateid stateid; 1403 enum pnfs_iomode iomode; 1404 bool send; 1405 1406 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1407 spin_unlock(&inode->i_lock); 1408 if (send) { 1409 /* Send an async layoutreturn so we dont deadlock */ 1410 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1411 PNFS_FL_LAYOUTRETURN_ASYNC); 1412 } 1413 } else 1414 spin_unlock(&inode->i_lock); 1415 } 1416 1417 /* 1418 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1419 * when the layout segment list is empty. 1420 * 1421 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1422 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1423 * deviceid is marked invalid. 1424 */ 1425 int 1426 _pnfs_return_layout(struct inode *ino) 1427 { 1428 struct pnfs_layout_hdr *lo = NULL; 1429 struct nfs_inode *nfsi = NFS_I(ino); 1430 struct pnfs_layout_range range = { 1431 .iomode = IOMODE_ANY, 1432 .offset = 0, 1433 .length = NFS4_MAX_UINT64, 1434 }; 1435 LIST_HEAD(tmp_list); 1436 const struct cred *cred; 1437 nfs4_stateid stateid; 1438 int status = 0; 1439 bool send, valid_layout; 1440 1441 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1442 1443 spin_lock(&ino->i_lock); 1444 lo = nfsi->layout; 1445 if (!lo) { 1446 spin_unlock(&ino->i_lock); 1447 dprintk("NFS: %s no layout to return\n", __func__); 1448 goto out; 1449 } 1450 /* Reference matched in nfs4_layoutreturn_release */ 1451 pnfs_get_layout_hdr(lo); 1452 /* Is there an outstanding layoutreturn ? */ 1453 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1454 spin_unlock(&ino->i_lock); 1455 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1456 TASK_UNINTERRUPTIBLE)) 1457 goto out_put_layout_hdr; 1458 spin_lock(&ino->i_lock); 1459 } 1460 valid_layout = pnfs_layout_is_valid(lo); 1461 pnfs_clear_layoutcommit(ino, &tmp_list); 1462 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); 1463 1464 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1465 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1466 1467 /* Don't send a LAYOUTRETURN if list was initially empty */ 1468 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1469 !valid_layout) { 1470 spin_unlock(&ino->i_lock); 1471 dprintk("NFS: %s no layout segments to return\n", __func__); 1472 goto out_wait_layoutreturn; 1473 } 1474 1475 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1476 spin_unlock(&ino->i_lock); 1477 if (send) 1478 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1479 0); 1480 out_wait_layoutreturn: 1481 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1482 out_put_layout_hdr: 1483 pnfs_free_lseg_list(&tmp_list); 1484 pnfs_put_layout_hdr(lo); 1485 out: 1486 dprintk("<-- %s status: %d\n", __func__, status); 1487 return status; 1488 } 1489 1490 int 1491 pnfs_commit_and_return_layout(struct inode *inode) 1492 { 1493 struct pnfs_layout_hdr *lo; 1494 int ret; 1495 1496 spin_lock(&inode->i_lock); 1497 lo = NFS_I(inode)->layout; 1498 if (lo == NULL) { 1499 spin_unlock(&inode->i_lock); 1500 return 0; 1501 } 1502 pnfs_get_layout_hdr(lo); 1503 /* Block new layoutgets and read/write to ds */ 1504 lo->plh_block_lgets++; 1505 spin_unlock(&inode->i_lock); 1506 filemap_fdatawait(inode->i_mapping); 1507 ret = pnfs_layoutcommit_inode(inode, true); 1508 if (ret == 0) 1509 ret = _pnfs_return_layout(inode); 1510 spin_lock(&inode->i_lock); 1511 lo->plh_block_lgets--; 1512 spin_unlock(&inode->i_lock); 1513 pnfs_put_layout_hdr(lo); 1514 return ret; 1515 } 1516 1517 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1518 { 1519 struct inode *inode = lo->plh_inode; 1520 const struct cred *cred; 1521 1522 spin_lock(&inode->i_lock); 1523 if (!pnfs_layout_is_valid(lo)) { 1524 spin_unlock(&inode->i_lock); 1525 return 0; 1526 } 1527 cred = get_cred(lo->plh_lc_cred); 1528 pnfs_get_layout_hdr(lo); 1529 spin_unlock(&inode->i_lock); 1530 1531 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1532 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1533 } 1534 1535 bool pnfs_roc(struct inode *ino, 1536 struct nfs4_layoutreturn_args *args, 1537 struct nfs4_layoutreturn_res *res, 1538 const struct cred *cred) 1539 { 1540 struct nfs_inode *nfsi = NFS_I(ino); 1541 struct nfs_open_context *ctx; 1542 struct nfs4_state *state; 1543 struct pnfs_layout_hdr *lo; 1544 struct pnfs_layout_segment *lseg, *next; 1545 const struct cred *lc_cred; 1546 nfs4_stateid stateid; 1547 enum pnfs_iomode iomode = 0; 1548 bool layoutreturn = false, roc = false; 1549 bool skip_read = false; 1550 1551 if (!nfs_have_layout(ino)) 1552 return false; 1553 retry: 1554 rcu_read_lock(); 1555 spin_lock(&ino->i_lock); 1556 lo = nfsi->layout; 1557 if (!lo || !pnfs_layout_is_valid(lo) || 1558 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1559 lo = NULL; 1560 goto out_noroc; 1561 } 1562 pnfs_get_layout_hdr(lo); 1563 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1564 spin_unlock(&ino->i_lock); 1565 rcu_read_unlock(); 1566 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1567 TASK_UNINTERRUPTIBLE); 1568 pnfs_put_layout_hdr(lo); 1569 goto retry; 1570 } 1571 1572 /* no roc if we hold a delegation */ 1573 if (nfs4_check_delegation(ino, FMODE_READ)) { 1574 if (nfs4_check_delegation(ino, FMODE_WRITE)) 1575 goto out_noroc; 1576 skip_read = true; 1577 } 1578 1579 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1580 state = ctx->state; 1581 if (state == NULL) 1582 continue; 1583 /* Don't return layout if there is open file state */ 1584 if (state->state & FMODE_WRITE) 1585 goto out_noroc; 1586 if (state->state & FMODE_READ) 1587 skip_read = true; 1588 } 1589 1590 1591 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1592 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1593 continue; 1594 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1595 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1596 continue; 1597 /* 1598 * Note: mark lseg for return so pnfs_layout_remove_lseg 1599 * doesn't invalidate the layout for us. 1600 */ 1601 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1602 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1603 continue; 1604 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1605 } 1606 1607 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1608 goto out_noroc; 1609 1610 /* ROC in two conditions: 1611 * 1. there are ROC lsegs 1612 * 2. we don't send layoutreturn 1613 */ 1614 /* lo ref dropped in pnfs_roc_release() */ 1615 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1616 /* If the creds don't match, we can't compound the layoutreturn */ 1617 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1618 goto out_noroc; 1619 1620 roc = layoutreturn; 1621 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1622 res->lrs_present = 0; 1623 layoutreturn = false; 1624 put_cred(lc_cred); 1625 1626 out_noroc: 1627 spin_unlock(&ino->i_lock); 1628 rcu_read_unlock(); 1629 pnfs_layoutcommit_inode(ino, true); 1630 if (roc) { 1631 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1632 if (ld->prepare_layoutreturn) 1633 ld->prepare_layoutreturn(args); 1634 pnfs_put_layout_hdr(lo); 1635 return true; 1636 } 1637 if (layoutreturn) 1638 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1639 pnfs_put_layout_hdr(lo); 1640 return false; 1641 } 1642 1643 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1644 struct nfs4_layoutreturn_res **respp, int *ret) 1645 { 1646 struct nfs4_layoutreturn_args *arg = *argpp; 1647 int retval = -EAGAIN; 1648 1649 if (!arg) 1650 return 0; 1651 /* Handle Layoutreturn errors */ 1652 switch (*ret) { 1653 case 0: 1654 retval = 0; 1655 break; 1656 case -NFS4ERR_NOMATCHING_LAYOUT: 1657 /* Was there an RPC level error? If not, retry */ 1658 if (task->tk_rpc_status == 0) 1659 break; 1660 /* 1661 * Is there a fatal network level error? 1662 * If so release the layout, but flag the error. 1663 */ 1664 if ((task->tk_rpc_status == -ENETDOWN || 1665 task->tk_rpc_status == -ENETUNREACH) && 1666 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1667 *ret = 0; 1668 (*respp)->lrs_present = 0; 1669 retval = -EIO; 1670 break; 1671 } 1672 /* If the call was not sent, let caller handle it */ 1673 if (!RPC_WAS_SENT(task)) 1674 return 0; 1675 /* 1676 * Otherwise, assume the call succeeded and 1677 * that we need to release the layout 1678 */ 1679 *ret = 0; 1680 (*respp)->lrs_present = 0; 1681 retval = 0; 1682 break; 1683 case -NFS4ERR_DELAY: 1684 /* Let the caller handle the retry */ 1685 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1686 return 0; 1687 case -NFS4ERR_OLD_STATEID: 1688 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1689 &arg->range, arg->inode)) 1690 break; 1691 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1692 return -EAGAIN; 1693 } 1694 *argpp = NULL; 1695 *respp = NULL; 1696 return retval; 1697 } 1698 1699 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1700 struct nfs4_layoutreturn_res *res, int ret) 1701 { 1702 struct pnfs_layout_hdr *lo = args->layout; 1703 struct inode *inode = args->inode; 1704 const nfs4_stateid *res_stateid = NULL; 1705 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1706 LIST_HEAD(freeme); 1707 1708 switch (ret) { 1709 case -NFS4ERR_BADSESSION: 1710 case -NFS4ERR_DEADSESSION: 1711 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1712 case -NFS4ERR_NOMATCHING_LAYOUT: 1713 spin_lock(&inode->i_lock); 1714 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1715 &args->range, &freeme); 1716 spin_unlock(&inode->i_lock); 1717 pnfs_free_lseg_list(&freeme); 1718 break; 1719 case 0: 1720 if (res->lrs_present) 1721 res_stateid = &res->stateid; 1722 fallthrough; 1723 default: 1724 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1725 res_stateid); 1726 } 1727 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1728 if (ld_private && ld_private->ops && ld_private->ops->free) 1729 ld_private->ops->free(ld_private); 1730 pnfs_put_layout_hdr(lo); 1731 } 1732 1733 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1734 { 1735 struct nfs_inode *nfsi = NFS_I(ino); 1736 struct pnfs_layout_hdr *lo; 1737 bool sleep = false; 1738 1739 /* we might not have grabbed lo reference. so need to check under 1740 * i_lock */ 1741 spin_lock(&ino->i_lock); 1742 lo = nfsi->layout; 1743 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1744 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1745 sleep = true; 1746 } 1747 spin_unlock(&ino->i_lock); 1748 return sleep; 1749 } 1750 1751 /* 1752 * Compare two layout segments for sorting into layout cache. 1753 * We want to preferentially return RW over RO layouts, so ensure those 1754 * are seen first. 1755 */ 1756 static s64 1757 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1758 const struct pnfs_layout_range *l2) 1759 { 1760 s64 d; 1761 1762 /* high offset > low offset */ 1763 d = l1->offset - l2->offset; 1764 if (d) 1765 return d; 1766 1767 /* short length > long length */ 1768 d = l2->length - l1->length; 1769 if (d) 1770 return d; 1771 1772 /* read > read/write */ 1773 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1774 } 1775 1776 static bool 1777 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1778 const struct pnfs_layout_range *l2) 1779 { 1780 return pnfs_lseg_range_cmp(l1, l2) > 0; 1781 } 1782 1783 static bool 1784 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1785 struct pnfs_layout_segment *old) 1786 { 1787 return false; 1788 } 1789 1790 void 1791 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1792 struct pnfs_layout_segment *lseg, 1793 bool (*is_after)(const struct pnfs_layout_range *, 1794 const struct pnfs_layout_range *), 1795 bool (*do_merge)(struct pnfs_layout_segment *, 1796 struct pnfs_layout_segment *), 1797 struct list_head *free_me) 1798 { 1799 struct pnfs_layout_segment *lp, *tmp; 1800 1801 dprintk("%s:Begin\n", __func__); 1802 1803 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1804 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1805 continue; 1806 if (do_merge(lseg, lp)) { 1807 mark_lseg_invalid(lp, free_me); 1808 continue; 1809 } 1810 if (is_after(&lseg->pls_range, &lp->pls_range)) 1811 continue; 1812 list_add_tail(&lseg->pls_list, &lp->pls_list); 1813 dprintk("%s: inserted lseg %p " 1814 "iomode %d offset %llu length %llu before " 1815 "lp %p iomode %d offset %llu length %llu\n", 1816 __func__, lseg, lseg->pls_range.iomode, 1817 lseg->pls_range.offset, lseg->pls_range.length, 1818 lp, lp->pls_range.iomode, lp->pls_range.offset, 1819 lp->pls_range.length); 1820 goto out; 1821 } 1822 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1823 dprintk("%s: inserted lseg %p " 1824 "iomode %d offset %llu length %llu at tail\n", 1825 __func__, lseg, lseg->pls_range.iomode, 1826 lseg->pls_range.offset, lseg->pls_range.length); 1827 out: 1828 pnfs_get_layout_hdr(lo); 1829 1830 dprintk("%s:Return\n", __func__); 1831 } 1832 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1833 1834 static void 1835 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1836 struct pnfs_layout_segment *lseg, 1837 struct list_head *free_me) 1838 { 1839 struct inode *inode = lo->plh_inode; 1840 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1841 1842 if (ld->add_lseg != NULL) 1843 ld->add_lseg(lo, lseg, free_me); 1844 else 1845 pnfs_generic_layout_insert_lseg(lo, lseg, 1846 pnfs_lseg_range_is_after, 1847 pnfs_lseg_no_merge, 1848 free_me); 1849 } 1850 1851 static struct pnfs_layout_hdr * 1852 alloc_init_layout_hdr(struct inode *ino, 1853 struct nfs_open_context *ctx, 1854 gfp_t gfp_flags) 1855 { 1856 struct pnfs_layout_hdr *lo; 1857 1858 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1859 if (!lo) 1860 return NULL; 1861 refcount_set(&lo->plh_refcount, 1); 1862 INIT_LIST_HEAD(&lo->plh_layouts); 1863 INIT_LIST_HEAD(&lo->plh_segs); 1864 INIT_LIST_HEAD(&lo->plh_return_segs); 1865 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1866 lo->plh_inode = ino; 1867 lo->plh_lc_cred = get_cred(ctx->cred); 1868 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1869 return lo; 1870 } 1871 1872 static struct pnfs_layout_hdr * 1873 pnfs_find_alloc_layout(struct inode *ino, 1874 struct nfs_open_context *ctx, 1875 gfp_t gfp_flags) 1876 __releases(&ino->i_lock) 1877 __acquires(&ino->i_lock) 1878 { 1879 struct nfs_inode *nfsi = NFS_I(ino); 1880 struct pnfs_layout_hdr *new = NULL; 1881 1882 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1883 1884 if (nfsi->layout != NULL) 1885 goto out_existing; 1886 spin_unlock(&ino->i_lock); 1887 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1888 spin_lock(&ino->i_lock); 1889 1890 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1891 nfsi->layout = new; 1892 return new; 1893 } else if (new != NULL) 1894 pnfs_free_layout_hdr(new); 1895 out_existing: 1896 pnfs_get_layout_hdr(nfsi->layout); 1897 return nfsi->layout; 1898 } 1899 1900 /* 1901 * iomode matching rules: 1902 * iomode lseg strict match 1903 * iomode 1904 * ----- ----- ------ ----- 1905 * ANY READ N/A true 1906 * ANY RW N/A true 1907 * RW READ N/A false 1908 * RW RW N/A true 1909 * READ READ N/A true 1910 * READ RW true false 1911 * READ RW false true 1912 */ 1913 static bool 1914 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1915 const struct pnfs_layout_range *range, 1916 bool strict_iomode) 1917 { 1918 struct pnfs_layout_range range1; 1919 1920 if ((range->iomode == IOMODE_RW && 1921 ls_range->iomode != IOMODE_RW) || 1922 (range->iomode != ls_range->iomode && 1923 strict_iomode) || 1924 !pnfs_lseg_range_intersecting(ls_range, range)) 1925 return false; 1926 1927 /* range1 covers only the first byte in the range */ 1928 range1 = *range; 1929 range1.length = 1; 1930 return pnfs_lseg_range_contained(ls_range, &range1); 1931 } 1932 1933 /* 1934 * lookup range in layout 1935 */ 1936 static struct pnfs_layout_segment * 1937 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1938 struct pnfs_layout_range *range, 1939 bool strict_iomode) 1940 { 1941 struct pnfs_layout_segment *lseg, *ret = NULL; 1942 1943 dprintk("%s:Begin\n", __func__); 1944 1945 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1946 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1947 pnfs_lseg_range_match(&lseg->pls_range, range, 1948 strict_iomode)) { 1949 ret = pnfs_get_lseg(lseg); 1950 break; 1951 } 1952 } 1953 1954 dprintk("%s:Return lseg %p ref %d\n", 1955 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 1956 return ret; 1957 } 1958 1959 /* 1960 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1961 * to the MDS or over pNFS 1962 * 1963 * The nfs_inode read_io and write_io fields are cumulative counters reset 1964 * when there are no layout segments. Note that in pnfs_update_layout iomode 1965 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1966 * WRITE request. 1967 * 1968 * A return of true means use MDS I/O. 1969 * 1970 * From rfc 5661: 1971 * If a file's size is smaller than the file size threshold, data accesses 1972 * SHOULD be sent to the metadata server. If an I/O request has a length that 1973 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1974 * server. If both file size and I/O size are provided, the client SHOULD 1975 * reach or exceed both thresholds before sending its read or write 1976 * requests to the data server. 1977 */ 1978 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1979 struct inode *ino, int iomode) 1980 { 1981 struct nfs4_threshold *t = ctx->mdsthreshold; 1982 struct nfs_inode *nfsi = NFS_I(ino); 1983 loff_t fsize = i_size_read(ino); 1984 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1985 1986 if (t == NULL) 1987 return ret; 1988 1989 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1990 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1991 1992 switch (iomode) { 1993 case IOMODE_READ: 1994 if (t->bm & THRESHOLD_RD) { 1995 dprintk("%s fsize %llu\n", __func__, fsize); 1996 size_set = true; 1997 if (fsize < t->rd_sz) 1998 size = true; 1999 } 2000 if (t->bm & THRESHOLD_RD_IO) { 2001 dprintk("%s nfsi->read_io %llu\n", __func__, 2002 nfsi->read_io); 2003 io_set = true; 2004 if (nfsi->read_io < t->rd_io_sz) 2005 io = true; 2006 } 2007 break; 2008 case IOMODE_RW: 2009 if (t->bm & THRESHOLD_WR) { 2010 dprintk("%s fsize %llu\n", __func__, fsize); 2011 size_set = true; 2012 if (fsize < t->wr_sz) 2013 size = true; 2014 } 2015 if (t->bm & THRESHOLD_WR_IO) { 2016 dprintk("%s nfsi->write_io %llu\n", __func__, 2017 nfsi->write_io); 2018 io_set = true; 2019 if (nfsi->write_io < t->wr_io_sz) 2020 io = true; 2021 } 2022 break; 2023 } 2024 if (size_set && io_set) { 2025 if (size && io) 2026 ret = true; 2027 } else if (size || io) 2028 ret = true; 2029 2030 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2031 return ret; 2032 } 2033 2034 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2035 { 2036 /* 2037 * send layoutcommit as it can hold up layoutreturn due to lseg 2038 * reference 2039 */ 2040 pnfs_layoutcommit_inode(lo->plh_inode, false); 2041 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2042 nfs_wait_bit_killable, 2043 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2044 } 2045 2046 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2047 { 2048 atomic_inc(&lo->plh_outstanding); 2049 } 2050 2051 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2052 { 2053 if (atomic_dec_and_test(&lo->plh_outstanding) && 2054 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2055 smp_mb__after_atomic(); 2056 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2057 } 2058 } 2059 2060 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2061 { 2062 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2063 } 2064 2065 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 2066 { 2067 unsigned long *bitlock = &lo->plh_flags; 2068 2069 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 2070 smp_mb__after_atomic(); 2071 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 2072 } 2073 2074 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2075 struct nfs_server *server) 2076 { 2077 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2078 struct nfs_client *clp = server->nfs_client; 2079 2080 /* The lo must be on the clp list if there is any 2081 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2082 */ 2083 spin_lock(&clp->cl_lock); 2084 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2085 spin_unlock(&clp->cl_lock); 2086 } 2087 } 2088 2089 /* 2090 * Layout segment is retreived from the server if not cached. 2091 * The appropriate layout segment is referenced and returned to the caller. 2092 */ 2093 struct pnfs_layout_segment * 2094 pnfs_update_layout(struct inode *ino, 2095 struct nfs_open_context *ctx, 2096 loff_t pos, 2097 u64 count, 2098 enum pnfs_iomode iomode, 2099 bool strict_iomode, 2100 gfp_t gfp_flags) 2101 { 2102 struct pnfs_layout_range arg = { 2103 .iomode = iomode, 2104 .offset = pos, 2105 .length = count, 2106 }; 2107 unsigned pg_offset; 2108 struct nfs_server *server = NFS_SERVER(ino); 2109 struct nfs_client *clp = server->nfs_client; 2110 struct pnfs_layout_hdr *lo = NULL; 2111 struct pnfs_layout_segment *lseg = NULL; 2112 struct nfs4_layoutget *lgp; 2113 nfs4_stateid stateid; 2114 struct nfs4_exception exception = { 2115 .inode = ino, 2116 }; 2117 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2118 bool first; 2119 2120 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2121 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2122 PNFS_UPDATE_LAYOUT_NO_PNFS); 2123 goto out; 2124 } 2125 2126 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2127 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2128 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2129 goto out; 2130 } 2131 2132 lookup_again: 2133 if (!nfs4_valid_open_stateid(ctx->state)) { 2134 trace_pnfs_update_layout(ino, pos, count, 2135 iomode, lo, lseg, 2136 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2137 lseg = ERR_PTR(-EIO); 2138 goto out; 2139 } 2140 2141 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2142 if (IS_ERR(lseg)) 2143 goto out; 2144 first = false; 2145 spin_lock(&ino->i_lock); 2146 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2147 if (lo == NULL) { 2148 spin_unlock(&ino->i_lock); 2149 lseg = ERR_PTR(-ENOMEM); 2150 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2151 PNFS_UPDATE_LAYOUT_NOMEM); 2152 goto out; 2153 } 2154 2155 /* Do we even need to bother with this? */ 2156 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2157 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2158 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2159 dprintk("%s matches recall, use MDS\n", __func__); 2160 goto out_unlock; 2161 } 2162 2163 /* if LAYOUTGET already failed once we don't try again */ 2164 if (pnfs_layout_io_test_failed(lo, iomode)) { 2165 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2166 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2167 goto out_unlock; 2168 } 2169 2170 /* 2171 * If the layout segment list is empty, but there are outstanding 2172 * layoutget calls, then they might be subject to a layoutrecall. 2173 */ 2174 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2175 atomic_read(&lo->plh_outstanding) != 0) { 2176 spin_unlock(&ino->i_lock); 2177 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2178 TASK_KILLABLE)); 2179 if (IS_ERR(lseg)) 2180 goto out_put_layout_hdr; 2181 pnfs_put_layout_hdr(lo); 2182 goto lookup_again; 2183 } 2184 2185 /* 2186 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2187 * for LAYOUTRETURN. 2188 */ 2189 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2190 spin_unlock(&ino->i_lock); 2191 dprintk("%s wait for layoutreturn\n", __func__); 2192 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2193 if (!IS_ERR(lseg)) { 2194 pnfs_put_layout_hdr(lo); 2195 dprintk("%s retrying\n", __func__); 2196 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2197 lseg, 2198 PNFS_UPDATE_LAYOUT_RETRY); 2199 goto lookup_again; 2200 } 2201 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2202 PNFS_UPDATE_LAYOUT_RETURN); 2203 goto out_put_layout_hdr; 2204 } 2205 2206 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2207 if (lseg) { 2208 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2209 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2210 goto out_unlock; 2211 } 2212 2213 /* 2214 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2215 * stateid, or it has been invalidated, then we must use the open 2216 * stateid. 2217 */ 2218 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2219 int status; 2220 2221 /* 2222 * The first layoutget for the file. Need to serialize per 2223 * RFC 5661 Errata 3208. 2224 */ 2225 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2226 &lo->plh_flags)) { 2227 spin_unlock(&ino->i_lock); 2228 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2229 NFS_LAYOUT_FIRST_LAYOUTGET, 2230 TASK_KILLABLE)); 2231 if (IS_ERR(lseg)) 2232 goto out_put_layout_hdr; 2233 pnfs_put_layout_hdr(lo); 2234 dprintk("%s retrying\n", __func__); 2235 goto lookup_again; 2236 } 2237 2238 spin_unlock(&ino->i_lock); 2239 first = true; 2240 status = nfs4_select_rw_stateid(ctx->state, 2241 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2242 NULL, &stateid, NULL); 2243 if (status != 0) { 2244 lseg = ERR_PTR(status); 2245 trace_pnfs_update_layout(ino, pos, count, 2246 iomode, lo, lseg, 2247 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2248 nfs4_schedule_stateid_recovery(server, ctx->state); 2249 pnfs_clear_first_layoutget(lo); 2250 pnfs_put_layout_hdr(lo); 2251 goto lookup_again; 2252 } 2253 spin_lock(&ino->i_lock); 2254 } else { 2255 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2256 } 2257 2258 if (pnfs_layoutgets_blocked(lo)) { 2259 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2260 PNFS_UPDATE_LAYOUT_BLOCKED); 2261 goto out_unlock; 2262 } 2263 nfs_layoutget_begin(lo); 2264 spin_unlock(&ino->i_lock); 2265 2266 _add_to_server_list(lo, server); 2267 2268 pg_offset = arg.offset & ~PAGE_MASK; 2269 if (pg_offset) { 2270 arg.offset -= pg_offset; 2271 arg.length += pg_offset; 2272 } 2273 if (arg.length != NFS4_MAX_UINT64) 2274 arg.length = PAGE_ALIGN(arg.length); 2275 2276 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); 2277 if (!lgp) { 2278 lseg = ERR_PTR(-ENOMEM); 2279 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2280 PNFS_UPDATE_LAYOUT_NOMEM); 2281 nfs_layoutget_end(lo); 2282 goto out_put_layout_hdr; 2283 } 2284 2285 lgp->lo = lo; 2286 pnfs_get_layout_hdr(lo); 2287 2288 lseg = nfs4_proc_layoutget(lgp, &exception); 2289 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2290 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2291 nfs_layoutget_end(lo); 2292 if (IS_ERR(lseg)) { 2293 switch(PTR_ERR(lseg)) { 2294 case -EBUSY: 2295 if (time_after(jiffies, giveup)) 2296 lseg = NULL; 2297 break; 2298 case -ERECALLCONFLICT: 2299 case -EAGAIN: 2300 break; 2301 case -ENODATA: 2302 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2303 pnfs_layout_set_fail_bit( 2304 lo, pnfs_iomode_to_fail_bit(iomode)); 2305 lseg = NULL; 2306 goto out_put_layout_hdr; 2307 default: 2308 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2309 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2310 lseg = NULL; 2311 } 2312 goto out_put_layout_hdr; 2313 } 2314 if (lseg) { 2315 if (!exception.retry) 2316 goto out_put_layout_hdr; 2317 if (first) 2318 pnfs_clear_first_layoutget(lo); 2319 trace_pnfs_update_layout(ino, pos, count, 2320 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2321 pnfs_put_layout_hdr(lo); 2322 goto lookup_again; 2323 } 2324 } else { 2325 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2326 } 2327 2328 out_put_layout_hdr: 2329 if (first) 2330 pnfs_clear_first_layoutget(lo); 2331 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2332 PNFS_UPDATE_LAYOUT_EXIT); 2333 pnfs_put_layout_hdr(lo); 2334 out: 2335 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2336 "(%s, offset: %llu, length: %llu)\n", 2337 __func__, ino->i_sb->s_id, 2338 (unsigned long long)NFS_FILEID(ino), 2339 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2340 iomode==IOMODE_RW ? "read/write" : "read-only", 2341 (unsigned long long)pos, 2342 (unsigned long long)count); 2343 return lseg; 2344 out_unlock: 2345 spin_unlock(&ino->i_lock); 2346 goto out_put_layout_hdr; 2347 } 2348 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2349 2350 static bool 2351 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2352 { 2353 switch (range->iomode) { 2354 case IOMODE_READ: 2355 case IOMODE_RW: 2356 break; 2357 default: 2358 return false; 2359 } 2360 if (range->offset == NFS4_MAX_UINT64) 2361 return false; 2362 if (range->length == 0) 2363 return false; 2364 if (range->length != NFS4_MAX_UINT64 && 2365 range->length > NFS4_MAX_UINT64 - range->offset) 2366 return false; 2367 return true; 2368 } 2369 2370 static struct pnfs_layout_hdr * 2371 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2372 { 2373 struct pnfs_layout_hdr *lo; 2374 2375 spin_lock(&ino->i_lock); 2376 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2377 if (!lo) 2378 goto out_unlock; 2379 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2380 goto out_unlock; 2381 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2382 goto out_unlock; 2383 if (pnfs_layoutgets_blocked(lo)) 2384 goto out_unlock; 2385 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2386 goto out_unlock; 2387 nfs_layoutget_begin(lo); 2388 spin_unlock(&ino->i_lock); 2389 _add_to_server_list(lo, NFS_SERVER(ino)); 2390 return lo; 2391 2392 out_unlock: 2393 spin_unlock(&ino->i_lock); 2394 pnfs_put_layout_hdr(lo); 2395 return NULL; 2396 } 2397 2398 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2399 struct nfs_open_context *ctx) 2400 { 2401 struct inode *ino = data->dentry->d_inode; 2402 struct pnfs_layout_range rng = { 2403 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2404 IOMODE_RW: IOMODE_READ, 2405 .offset = 0, 2406 .length = NFS4_MAX_UINT64, 2407 }; 2408 struct nfs4_layoutget *lgp; 2409 struct pnfs_layout_hdr *lo; 2410 2411 /* Heuristic: don't send layoutget if we have cached data */ 2412 if (rng.iomode == IOMODE_READ && 2413 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2414 return; 2415 2416 lo = _pnfs_grab_empty_layout(ino, ctx); 2417 if (!lo) 2418 return; 2419 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2420 nfs_io_gfp_mask()); 2421 if (!lgp) { 2422 pnfs_clear_first_layoutget(lo); 2423 nfs_layoutget_end(lo); 2424 pnfs_put_layout_hdr(lo); 2425 return; 2426 } 2427 lgp->lo = lo; 2428 data->lgp = lgp; 2429 data->o_arg.lg_args = &lgp->args; 2430 data->o_res.lg_res = &lgp->res; 2431 } 2432 2433 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2434 struct nfs_open_context *ctx) 2435 { 2436 struct inode *ino = data->dentry->d_inode; 2437 struct pnfs_layout_range rng = { 2438 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2439 IOMODE_RW: IOMODE_READ, 2440 .offset = 0, 2441 .length = NFS4_MAX_UINT64, 2442 }; 2443 struct nfs4_layoutget *lgp; 2444 2445 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2446 nfs_io_gfp_mask()); 2447 if (!lgp) 2448 return; 2449 data->lgp = lgp; 2450 data->o_arg.lg_args = &lgp->args; 2451 data->o_res.lg_res = &lgp->res; 2452 } 2453 2454 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2455 struct nfs_open_context *ctx) 2456 { 2457 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2458 2459 if (!(pnfs_enabled_sb(server) && 2460 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2461 return; 2462 /* Could check on max_ops, but currently hardcoded high enough */ 2463 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2464 return; 2465 if (data->lgp) 2466 return; 2467 if (data->state) 2468 _lgopen_prepare_attached(data, ctx); 2469 else 2470 _lgopen_prepare_floating(data, ctx); 2471 } 2472 2473 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2474 struct nfs_open_context *ctx) 2475 { 2476 struct pnfs_layout_hdr *lo; 2477 struct pnfs_layout_segment *lseg; 2478 struct nfs_server *srv = NFS_SERVER(ino); 2479 u32 iomode; 2480 2481 if (!lgp) 2482 return; 2483 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2484 if (lgp->res.status) { 2485 switch (lgp->res.status) { 2486 default: 2487 break; 2488 /* 2489 * Halt lgopen attempts if the server doesn't recognise 2490 * the "current stateid" value, the layout type, or the 2491 * layoutget operation as being valid. 2492 * Also if it complains about too many ops in the compound 2493 * or of the request/reply being too big. 2494 */ 2495 case -NFS4ERR_BAD_STATEID: 2496 case -NFS4ERR_NOTSUPP: 2497 case -NFS4ERR_REP_TOO_BIG: 2498 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2499 case -NFS4ERR_REQ_TOO_BIG: 2500 case -NFS4ERR_TOO_MANY_OPS: 2501 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2502 srv->caps &= ~NFS_CAP_LGOPEN; 2503 } 2504 return; 2505 } 2506 if (!lgp->lo) { 2507 lo = _pnfs_grab_empty_layout(ino, ctx); 2508 if (!lo) 2509 return; 2510 lgp->lo = lo; 2511 } else 2512 lo = lgp->lo; 2513 2514 lseg = pnfs_layout_process(lgp); 2515 if (!IS_ERR(lseg)) { 2516 iomode = lgp->args.range.iomode; 2517 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2518 pnfs_put_lseg(lseg); 2519 } 2520 } 2521 2522 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2523 { 2524 if (lgp != NULL) { 2525 if (lgp->lo) { 2526 pnfs_clear_first_layoutget(lgp->lo); 2527 nfs_layoutget_end(lgp->lo); 2528 } 2529 pnfs_layoutget_free(lgp); 2530 } 2531 } 2532 2533 struct pnfs_layout_segment * 2534 pnfs_layout_process(struct nfs4_layoutget *lgp) 2535 { 2536 struct pnfs_layout_hdr *lo = lgp->lo; 2537 struct nfs4_layoutget_res *res = &lgp->res; 2538 struct pnfs_layout_segment *lseg; 2539 struct inode *ino = lo->plh_inode; 2540 LIST_HEAD(free_me); 2541 2542 if (!pnfs_sanity_check_layout_range(&res->range)) 2543 return ERR_PTR(-EINVAL); 2544 2545 /* Inject layout blob into I/O device driver */ 2546 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2547 if (IS_ERR_OR_NULL(lseg)) { 2548 if (!lseg) 2549 lseg = ERR_PTR(-ENOMEM); 2550 2551 dprintk("%s: Could not allocate layout: error %ld\n", 2552 __func__, PTR_ERR(lseg)); 2553 return lseg; 2554 } 2555 2556 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2557 2558 spin_lock(&ino->i_lock); 2559 if (pnfs_layoutgets_blocked(lo)) { 2560 dprintk("%s forget reply due to state\n", __func__); 2561 goto out_forget; 2562 } 2563 2564 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2565 !pnfs_is_first_layoutget(lo)) 2566 goto out_forget; 2567 2568 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2569 /* existing state ID, make sure the sequence number matches. */ 2570 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2571 if (!pnfs_layout_is_valid(lo)) 2572 lo->plh_barrier = 0; 2573 dprintk("%s forget reply due to sequence\n", __func__); 2574 goto out_forget; 2575 } 2576 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2577 } else if (pnfs_layout_is_valid(lo)) { 2578 /* 2579 * We got an entirely new state ID. Mark all segments for the 2580 * inode invalid, and retry the layoutget 2581 */ 2582 struct pnfs_layout_range range = { 2583 .iomode = IOMODE_ANY, 2584 .length = NFS4_MAX_UINT64, 2585 }; 2586 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); 2587 goto out_forget; 2588 } else { 2589 /* We have a completely new layout */ 2590 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2591 } 2592 2593 pnfs_get_lseg(lseg); 2594 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2595 2596 2597 if (res->return_on_close) 2598 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2599 2600 spin_unlock(&ino->i_lock); 2601 pnfs_free_lseg_list(&free_me); 2602 return lseg; 2603 2604 out_forget: 2605 spin_unlock(&ino->i_lock); 2606 lseg->pls_layout = lo; 2607 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2608 return ERR_PTR(-EAGAIN); 2609 } 2610 2611 /** 2612 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2613 * @lo: pointer to layout header 2614 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2615 * @return_range: describe layout segment ranges to be returned 2616 * @seq: stateid seqid to match 2617 * 2618 * This function is mainly intended for use by layoutrecall. It attempts 2619 * to free the layout segment immediately, or else to mark it for return 2620 * as soon as its reference count drops to zero. 2621 * 2622 * Returns 2623 * - 0: a layoutreturn needs to be scheduled. 2624 * - EBUSY: there are layout segment that are still in use. 2625 * - ENOENT: there are no layout segments that need to be returned. 2626 */ 2627 int 2628 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2629 struct list_head *tmp_list, 2630 const struct pnfs_layout_range *return_range, 2631 u32 seq) 2632 { 2633 struct pnfs_layout_segment *lseg, *next; 2634 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2635 int remaining = 0; 2636 2637 dprintk("%s:Begin lo %p\n", __func__, lo); 2638 2639 assert_spin_locked(&lo->plh_inode->i_lock); 2640 2641 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2642 tmp_list = &lo->plh_return_segs; 2643 2644 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2645 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2646 dprintk("%s: marking lseg %p iomode %d " 2647 "offset %llu length %llu\n", __func__, 2648 lseg, lseg->pls_range.iomode, 2649 lseg->pls_range.offset, 2650 lseg->pls_range.length); 2651 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2652 tmp_list = &lo->plh_return_segs; 2653 if (mark_lseg_invalid(lseg, tmp_list)) 2654 continue; 2655 remaining++; 2656 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2657 pnfs_lseg_cancel_io(server, lseg); 2658 } 2659 2660 if (remaining) { 2661 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2662 return -EBUSY; 2663 } 2664 2665 if (!list_empty(&lo->plh_return_segs)) { 2666 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2667 return 0; 2668 } 2669 2670 return -ENOENT; 2671 } 2672 2673 static void 2674 pnfs_mark_layout_for_return(struct inode *inode, 2675 const struct pnfs_layout_range *range) 2676 { 2677 struct pnfs_layout_hdr *lo; 2678 bool return_now = false; 2679 2680 spin_lock(&inode->i_lock); 2681 lo = NFS_I(inode)->layout; 2682 if (!pnfs_layout_is_valid(lo)) { 2683 spin_unlock(&inode->i_lock); 2684 return; 2685 } 2686 pnfs_set_plh_return_info(lo, range->iomode, 0); 2687 /* 2688 * mark all matching lsegs so that we are sure to have no live 2689 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2690 * for how it works. 2691 */ 2692 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { 2693 const struct cred *cred; 2694 nfs4_stateid stateid; 2695 enum pnfs_iomode iomode; 2696 2697 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2698 spin_unlock(&inode->i_lock); 2699 if (return_now) 2700 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2701 PNFS_FL_LAYOUTRETURN_ASYNC); 2702 } else { 2703 spin_unlock(&inode->i_lock); 2704 nfs_commit_inode(inode, 0); 2705 } 2706 } 2707 2708 void pnfs_error_mark_layout_for_return(struct inode *inode, 2709 struct pnfs_layout_segment *lseg) 2710 { 2711 struct pnfs_layout_range range = { 2712 .iomode = lseg->pls_range.iomode, 2713 .offset = 0, 2714 .length = NFS4_MAX_UINT64, 2715 }; 2716 2717 pnfs_mark_layout_for_return(inode, &range); 2718 } 2719 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2720 2721 static bool 2722 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2723 { 2724 return pnfs_layout_is_valid(lo) && 2725 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2726 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2727 } 2728 2729 static struct pnfs_layout_segment * 2730 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2731 const struct pnfs_layout_range *range, 2732 enum pnfs_iomode iomode) 2733 { 2734 struct pnfs_layout_segment *lseg; 2735 2736 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2737 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2738 continue; 2739 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2740 continue; 2741 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2742 continue; 2743 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2744 return lseg; 2745 } 2746 return NULL; 2747 } 2748 2749 /* Find open file states whose mode matches that of the range */ 2750 static bool 2751 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2752 const struct pnfs_layout_range *range) 2753 { 2754 struct list_head *head; 2755 struct nfs_open_context *ctx; 2756 fmode_t mode = 0; 2757 2758 if (!pnfs_layout_can_be_returned(lo) || 2759 !pnfs_find_first_lseg(lo, range, range->iomode)) 2760 return false; 2761 2762 head = &NFS_I(lo->plh_inode)->open_files; 2763 list_for_each_entry_rcu(ctx, head, list) { 2764 if (ctx->state) 2765 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2766 } 2767 2768 switch (range->iomode) { 2769 default: 2770 break; 2771 case IOMODE_READ: 2772 mode &= ~FMODE_WRITE; 2773 break; 2774 case IOMODE_RW: 2775 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2776 mode &= ~FMODE_READ; 2777 } 2778 return mode == 0; 2779 } 2780 2781 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2782 void *data) 2783 { 2784 const struct pnfs_layout_range *range = data; 2785 const struct cred *cred; 2786 struct pnfs_layout_hdr *lo; 2787 struct inode *inode; 2788 nfs4_stateid stateid; 2789 enum pnfs_iomode iomode; 2790 2791 restart: 2792 rcu_read_lock(); 2793 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2794 inode = lo->plh_inode; 2795 if (!inode || !pnfs_layout_can_be_returned(lo) || 2796 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2797 continue; 2798 spin_lock(&inode->i_lock); 2799 if (!lo->plh_inode || 2800 !pnfs_should_return_unused_layout(lo, range)) { 2801 spin_unlock(&inode->i_lock); 2802 continue; 2803 } 2804 pnfs_get_layout_hdr(lo); 2805 pnfs_set_plh_return_info(lo, range->iomode, 0); 2806 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2807 range, 0) != 0 || 2808 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2809 spin_unlock(&inode->i_lock); 2810 rcu_read_unlock(); 2811 pnfs_put_layout_hdr(lo); 2812 cond_resched(); 2813 goto restart; 2814 } 2815 spin_unlock(&inode->i_lock); 2816 rcu_read_unlock(); 2817 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2818 PNFS_FL_LAYOUTRETURN_ASYNC); 2819 pnfs_put_layout_hdr(lo); 2820 cond_resched(); 2821 goto restart; 2822 } 2823 rcu_read_unlock(); 2824 return 0; 2825 } 2826 2827 void 2828 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2829 enum pnfs_iomode iomode) 2830 { 2831 struct pnfs_layout_range range = { 2832 .iomode = iomode, 2833 .offset = 0, 2834 .length = NFS4_MAX_UINT64, 2835 }; 2836 2837 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2838 &range); 2839 } 2840 2841 /* Check if we have we have a valid layout but if there isn't an intersection 2842 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 2843 */ 2844 void 2845 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 2846 struct nfs_page *req) 2847 { 2848 if (pgio->pg_lseg == NULL || 2849 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 2850 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 2851 return; 2852 pnfs_put_lseg(pgio->pg_lseg); 2853 pgio->pg_lseg = NULL; 2854 } 2855 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2856 2857 void 2858 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2859 { 2860 u64 rd_size; 2861 2862 pnfs_generic_pg_check_layout(pgio, req); 2863 if (pgio->pg_lseg == NULL) { 2864 if (pgio->pg_dreq == NULL) 2865 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2866 else 2867 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 2868 req_offset(req)); 2869 2870 pgio->pg_lseg = 2871 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2872 req_offset(req), rd_size, 2873 IOMODE_READ, false, 2874 nfs_io_gfp_mask()); 2875 if (IS_ERR(pgio->pg_lseg)) { 2876 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2877 pgio->pg_lseg = NULL; 2878 return; 2879 } 2880 } 2881 /* If no lseg, fall back to read through mds */ 2882 if (pgio->pg_lseg == NULL) 2883 nfs_pageio_reset_read_mds(pgio); 2884 2885 } 2886 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2887 2888 void 2889 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2890 struct nfs_page *req, u64 wb_size) 2891 { 2892 pnfs_generic_pg_check_layout(pgio, req); 2893 if (pgio->pg_lseg == NULL) { 2894 pgio->pg_lseg = 2895 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2896 req_offset(req), wb_size, IOMODE_RW, 2897 false, nfs_io_gfp_mask()); 2898 if (IS_ERR(pgio->pg_lseg)) { 2899 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2900 pgio->pg_lseg = NULL; 2901 return; 2902 } 2903 } 2904 /* If no lseg, fall back to write through mds */ 2905 if (pgio->pg_lseg == NULL) 2906 nfs_pageio_reset_write_mds(pgio); 2907 } 2908 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2909 2910 void 2911 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2912 { 2913 if (desc->pg_lseg) { 2914 pnfs_put_lseg(desc->pg_lseg); 2915 desc->pg_lseg = NULL; 2916 } 2917 } 2918 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2919 2920 /* 2921 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2922 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2923 */ 2924 size_t 2925 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2926 struct nfs_page *prev, struct nfs_page *req) 2927 { 2928 unsigned int size; 2929 u64 seg_end, req_start, seg_left; 2930 2931 size = nfs_generic_pg_test(pgio, prev, req); 2932 if (!size) 2933 return 0; 2934 2935 /* 2936 * 'size' contains the number of bytes left in the current page (up 2937 * to the original size asked for in @req->wb_bytes). 2938 * 2939 * Calculate how many bytes are left in the layout segment 2940 * and if there are less bytes than 'size', return that instead. 2941 * 2942 * Please also note that 'end_offset' is actually the offset of the 2943 * first byte that lies outside the pnfs_layout_range. FIXME? 2944 * 2945 */ 2946 if (pgio->pg_lseg) { 2947 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2948 pgio->pg_lseg->pls_range.length); 2949 req_start = req_offset(req); 2950 2951 /* start of request is past the last byte of this segment */ 2952 if (req_start >= seg_end) 2953 return 0; 2954 2955 /* adjust 'size' iff there are fewer bytes left in the 2956 * segment than what nfs_generic_pg_test returned */ 2957 seg_left = seg_end - req_start; 2958 if (seg_left < size) 2959 size = (unsigned int)seg_left; 2960 } 2961 2962 return size; 2963 } 2964 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2965 2966 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2967 { 2968 struct nfs_pageio_descriptor pgio; 2969 2970 /* Resend all requests through the MDS */ 2971 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2972 hdr->completion_ops); 2973 return nfs_pageio_resend(&pgio, hdr); 2974 } 2975 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 2976 2977 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 2978 { 2979 2980 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 2981 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2982 PNFS_LAYOUTRET_ON_ERROR) { 2983 pnfs_return_layout(hdr->inode); 2984 } 2985 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2986 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 2987 } 2988 2989 /* 2990 * Called by non rpc-based layout drivers 2991 */ 2992 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 2993 { 2994 if (likely(!hdr->pnfs_error)) { 2995 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 2996 hdr->mds_offset + hdr->res.count); 2997 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2998 } 2999 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3000 if (unlikely(hdr->pnfs_error)) 3001 pnfs_ld_handle_write_error(hdr); 3002 hdr->mds_ops->rpc_release(hdr); 3003 } 3004 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3005 3006 static void 3007 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3008 struct nfs_pgio_header *hdr) 3009 { 3010 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3011 3012 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3013 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3014 nfs_pageio_reset_write_mds(desc); 3015 mirror->pg_recoalesce = 1; 3016 } 3017 hdr->completion_ops->completion(hdr); 3018 } 3019 3020 static enum pnfs_try_status 3021 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3022 const struct rpc_call_ops *call_ops, 3023 struct pnfs_layout_segment *lseg, 3024 int how) 3025 { 3026 struct inode *inode = hdr->inode; 3027 enum pnfs_try_status trypnfs; 3028 struct nfs_server *nfss = NFS_SERVER(inode); 3029 3030 hdr->mds_ops = call_ops; 3031 3032 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 3033 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3034 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3035 if (trypnfs != PNFS_NOT_ATTEMPTED) 3036 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3037 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3038 return trypnfs; 3039 } 3040 3041 static void 3042 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3043 struct nfs_pgio_header *hdr, int how) 3044 { 3045 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3046 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3047 enum pnfs_try_status trypnfs; 3048 3049 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3050 switch (trypnfs) { 3051 case PNFS_NOT_ATTEMPTED: 3052 pnfs_write_through_mds(desc, hdr); 3053 break; 3054 case PNFS_ATTEMPTED: 3055 break; 3056 case PNFS_TRY_AGAIN: 3057 /* cleanup hdr and prepare to redo pnfs */ 3058 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3059 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3060 list_splice_init(&hdr->pages, &mirror->pg_list); 3061 mirror->pg_recoalesce = 1; 3062 } 3063 hdr->mds_ops->rpc_release(hdr); 3064 } 3065 } 3066 3067 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3068 { 3069 pnfs_put_lseg(hdr->lseg); 3070 nfs_pgio_header_free(hdr); 3071 } 3072 3073 int 3074 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3075 { 3076 struct nfs_pgio_header *hdr; 3077 int ret; 3078 3079 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3080 if (!hdr) { 3081 desc->pg_error = -ENOMEM; 3082 return desc->pg_error; 3083 } 3084 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3085 3086 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3087 ret = nfs_generic_pgio(desc, hdr); 3088 if (!ret) 3089 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3090 3091 return ret; 3092 } 3093 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3094 3095 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3096 { 3097 struct nfs_pageio_descriptor pgio; 3098 3099 /* Resend all requests through the MDS */ 3100 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3101 return nfs_pageio_resend(&pgio, hdr); 3102 } 3103 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3104 3105 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3106 { 3107 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3108 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3109 PNFS_LAYOUTRET_ON_ERROR) { 3110 pnfs_return_layout(hdr->inode); 3111 } 3112 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3113 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3114 } 3115 3116 /* 3117 * Called by non rpc-based layout drivers 3118 */ 3119 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3120 { 3121 if (likely(!hdr->pnfs_error)) 3122 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3123 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3124 if (unlikely(hdr->pnfs_error)) 3125 pnfs_ld_handle_read_error(hdr); 3126 hdr->mds_ops->rpc_release(hdr); 3127 } 3128 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3129 3130 static void 3131 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3132 struct nfs_pgio_header *hdr) 3133 { 3134 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3135 3136 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3137 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3138 nfs_pageio_reset_read_mds(desc); 3139 mirror->pg_recoalesce = 1; 3140 } 3141 hdr->completion_ops->completion(hdr); 3142 } 3143 3144 /* 3145 * Call the appropriate parallel I/O subsystem read function. 3146 */ 3147 static enum pnfs_try_status 3148 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3149 const struct rpc_call_ops *call_ops, 3150 struct pnfs_layout_segment *lseg) 3151 { 3152 struct inode *inode = hdr->inode; 3153 struct nfs_server *nfss = NFS_SERVER(inode); 3154 enum pnfs_try_status trypnfs; 3155 3156 hdr->mds_ops = call_ops; 3157 3158 dprintk("%s: Reading ino:%lu %u@%llu\n", 3159 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3160 3161 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3162 if (trypnfs != PNFS_NOT_ATTEMPTED) 3163 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3164 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3165 return trypnfs; 3166 } 3167 3168 /* Resend all requests through pnfs. */ 3169 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3170 unsigned int mirror_idx) 3171 { 3172 struct nfs_pageio_descriptor pgio; 3173 3174 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3175 /* Prevent deadlocks with layoutreturn! */ 3176 pnfs_put_lseg(hdr->lseg); 3177 hdr->lseg = NULL; 3178 3179 nfs_pageio_init_read(&pgio, hdr->inode, false, 3180 hdr->completion_ops); 3181 pgio.pg_mirror_idx = mirror_idx; 3182 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3183 } 3184 } 3185 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3186 3187 static void 3188 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3189 { 3190 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3191 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3192 enum pnfs_try_status trypnfs; 3193 3194 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3195 switch (trypnfs) { 3196 case PNFS_NOT_ATTEMPTED: 3197 pnfs_read_through_mds(desc, hdr); 3198 break; 3199 case PNFS_ATTEMPTED: 3200 break; 3201 case PNFS_TRY_AGAIN: 3202 /* cleanup hdr and prepare to redo pnfs */ 3203 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3204 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3205 list_splice_init(&hdr->pages, &mirror->pg_list); 3206 mirror->pg_recoalesce = 1; 3207 } 3208 hdr->mds_ops->rpc_release(hdr); 3209 } 3210 } 3211 3212 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3213 { 3214 pnfs_put_lseg(hdr->lseg); 3215 nfs_pgio_header_free(hdr); 3216 } 3217 3218 int 3219 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3220 { 3221 struct nfs_pgio_header *hdr; 3222 int ret; 3223 3224 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3225 if (!hdr) { 3226 desc->pg_error = -ENOMEM; 3227 return desc->pg_error; 3228 } 3229 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3230 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3231 ret = nfs_generic_pgio(desc, hdr); 3232 if (!ret) 3233 pnfs_do_read(desc, hdr); 3234 return ret; 3235 } 3236 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3237 3238 static void pnfs_clear_layoutcommitting(struct inode *inode) 3239 { 3240 unsigned long *bitlock = &NFS_I(inode)->flags; 3241 3242 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 3243 smp_mb__after_atomic(); 3244 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 3245 } 3246 3247 /* 3248 * There can be multiple RW segments. 3249 */ 3250 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3251 { 3252 struct pnfs_layout_segment *lseg; 3253 3254 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3255 if (lseg->pls_range.iomode == IOMODE_RW && 3256 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3257 list_add(&lseg->pls_lc_list, listp); 3258 } 3259 } 3260 3261 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3262 { 3263 struct pnfs_layout_segment *lseg, *tmp; 3264 3265 /* Matched by references in pnfs_set_layoutcommit */ 3266 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3267 list_del_init(&lseg->pls_lc_list); 3268 pnfs_put_lseg(lseg); 3269 } 3270 3271 pnfs_clear_layoutcommitting(inode); 3272 } 3273 3274 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3275 { 3276 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3277 } 3278 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3279 3280 void 3281 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3282 loff_t end_pos) 3283 { 3284 struct nfs_inode *nfsi = NFS_I(inode); 3285 bool mark_as_dirty = false; 3286 3287 spin_lock(&inode->i_lock); 3288 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3289 nfsi->layout->plh_lwb = end_pos; 3290 mark_as_dirty = true; 3291 dprintk("%s: Set layoutcommit for inode %lu ", 3292 __func__, inode->i_ino); 3293 } else if (end_pos > nfsi->layout->plh_lwb) 3294 nfsi->layout->plh_lwb = end_pos; 3295 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3296 /* references matched in nfs4_layoutcommit_release */ 3297 pnfs_get_lseg(lseg); 3298 } 3299 spin_unlock(&inode->i_lock); 3300 dprintk("%s: lseg %p end_pos %llu\n", 3301 __func__, lseg, nfsi->layout->plh_lwb); 3302 3303 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3304 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3305 if (mark_as_dirty) 3306 mark_inode_dirty_sync(inode); 3307 } 3308 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3309 3310 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3311 { 3312 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3313 3314 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3315 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3316 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3317 } 3318 3319 /* 3320 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3321 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3322 * data to disk to allow the server to recover the data if it crashes. 3323 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3324 * is off, and a COMMIT is sent to a data server, or 3325 * if WRITEs to a data server return NFS_DATA_SYNC. 3326 */ 3327 int 3328 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3329 { 3330 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3331 struct nfs4_layoutcommit_data *data; 3332 struct nfs_inode *nfsi = NFS_I(inode); 3333 loff_t end_pos; 3334 int status; 3335 bool mark_as_dirty = false; 3336 3337 if (!pnfs_layoutcommit_outstanding(inode)) 3338 return 0; 3339 3340 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 3341 3342 status = -EAGAIN; 3343 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3344 if (!sync) 3345 goto out; 3346 status = wait_on_bit_lock_action(&nfsi->flags, 3347 NFS_INO_LAYOUTCOMMITTING, 3348 nfs_wait_bit_killable, 3349 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3350 if (status) 3351 goto out; 3352 } 3353 3354 status = -ENOMEM; 3355 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3356 data = kzalloc(sizeof(*data), nfs_io_gfp_mask()); 3357 if (!data) 3358 goto clear_layoutcommitting; 3359 3360 status = 0; 3361 spin_lock(&inode->i_lock); 3362 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3363 goto out_unlock; 3364 3365 INIT_LIST_HEAD(&data->lseg_list); 3366 pnfs_list_write_lseg(inode, &data->lseg_list); 3367 3368 end_pos = nfsi->layout->plh_lwb; 3369 3370 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3371 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3372 spin_unlock(&inode->i_lock); 3373 3374 data->args.inode = inode; 3375 nfs_fattr_init(&data->fattr); 3376 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3377 data->res.fattr = &data->fattr; 3378 if (end_pos != 0) 3379 data->args.lastbytewritten = end_pos - 1; 3380 else 3381 data->args.lastbytewritten = U64_MAX; 3382 data->res.server = NFS_SERVER(inode); 3383 3384 if (ld->prepare_layoutcommit) { 3385 status = ld->prepare_layoutcommit(&data->args); 3386 if (status) { 3387 if (status != -ENOSPC) 3388 put_cred(data->cred); 3389 spin_lock(&inode->i_lock); 3390 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3391 if (end_pos > nfsi->layout->plh_lwb) 3392 nfsi->layout->plh_lwb = end_pos; 3393 if (status != -ENOSPC) 3394 goto out_unlock; 3395 spin_unlock(&inode->i_lock); 3396 mark_as_dirty = true; 3397 } 3398 } 3399 3400 3401 status = nfs4_proc_layoutcommit(data, sync); 3402 out: 3403 if (status || mark_as_dirty) 3404 mark_inode_dirty_sync(inode); 3405 dprintk("<-- %s status %d\n", __func__, status); 3406 return status; 3407 out_unlock: 3408 spin_unlock(&inode->i_lock); 3409 kfree(data); 3410 clear_layoutcommitting: 3411 pnfs_clear_layoutcommitting(inode); 3412 goto out; 3413 } 3414 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3415 3416 int 3417 pnfs_generic_sync(struct inode *inode, bool datasync) 3418 { 3419 return pnfs_layoutcommit_inode(inode, true); 3420 } 3421 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3422 3423 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3424 { 3425 struct nfs4_threshold *thp; 3426 3427 thp = kzalloc(sizeof(*thp), nfs_io_gfp_mask()); 3428 if (!thp) { 3429 dprintk("%s mdsthreshold allocation failed\n", __func__); 3430 return NULL; 3431 } 3432 return thp; 3433 } 3434 3435 #if IS_ENABLED(CONFIG_NFS_V4_2) 3436 int 3437 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3438 { 3439 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3440 struct nfs_server *server = NFS_SERVER(inode); 3441 struct nfs_inode *nfsi = NFS_I(inode); 3442 struct nfs42_layoutstat_data *data; 3443 struct pnfs_layout_hdr *hdr; 3444 int status = 0; 3445 3446 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3447 goto out; 3448 3449 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3450 goto out; 3451 3452 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3453 goto out; 3454 3455 spin_lock(&inode->i_lock); 3456 if (!NFS_I(inode)->layout) { 3457 spin_unlock(&inode->i_lock); 3458 goto out_clear_layoutstats; 3459 } 3460 hdr = NFS_I(inode)->layout; 3461 pnfs_get_layout_hdr(hdr); 3462 spin_unlock(&inode->i_lock); 3463 3464 data = kzalloc(sizeof(*data), gfp_flags); 3465 if (!data) { 3466 status = -ENOMEM; 3467 goto out_put; 3468 } 3469 3470 data->args.fh = NFS_FH(inode); 3471 data->args.inode = inode; 3472 status = ld->prepare_layoutstats(&data->args); 3473 if (status) 3474 goto out_free; 3475 3476 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3477 3478 out: 3479 dprintk("%s returns %d\n", __func__, status); 3480 return status; 3481 3482 out_free: 3483 kfree(data); 3484 out_put: 3485 pnfs_put_layout_hdr(hdr); 3486 out_clear_layoutstats: 3487 smp_mb__before_atomic(); 3488 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3489 smp_mb__after_atomic(); 3490 goto out; 3491 } 3492 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3493 #endif 3494 3495 unsigned int layoutstats_timer; 3496 module_param(layoutstats_timer, uint, 0644); 3497 EXPORT_SYMBOL_GPL(layoutstats_timer); 3498