1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 436 if (err != -EBUSY) { 437 dst->seqid = lo->plh_stateid.seqid; 438 *dst_range = range; 439 ret = true; 440 } 441 } 442 out: 443 spin_unlock(&inode->i_lock); 444 pnfs_free_lseg_list(&head); 445 return ret; 446 } 447 448 /* 449 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 450 * 451 * In order to continue using the pnfs_layout_hdr, a full recovery 452 * is required. 453 * Note that caller must hold inode->i_lock. 454 */ 455 int 456 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 457 struct list_head *lseg_list) 458 { 459 struct pnfs_layout_range range = { 460 .iomode = IOMODE_ANY, 461 .offset = 0, 462 .length = NFS4_MAX_UINT64, 463 }; 464 struct pnfs_layout_segment *lseg, *next; 465 466 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 467 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); 468 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 469 pnfs_clear_lseg_state(lseg, lseg_list); 470 pnfs_clear_layoutreturn_info(lo); 471 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 472 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 473 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 474 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 475 pnfs_clear_layoutreturn_waitbit(lo); 476 return !list_empty(&lo->plh_segs); 477 } 478 479 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 480 struct list_head *lseg_list, 481 enum pnfs_iomode iomode, u32 seq) 482 { 483 struct pnfs_layout_range range = { 484 .iomode = iomode, 485 .length = NFS4_MAX_UINT64, 486 }; 487 488 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); 489 } 490 491 static int 492 pnfs_iomode_to_fail_bit(u32 iomode) 493 { 494 return iomode == IOMODE_RW ? 495 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 496 } 497 498 static void 499 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 500 { 501 lo->plh_retry_timestamp = jiffies; 502 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 503 refcount_inc(&lo->plh_refcount); 504 } 505 506 static void 507 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 508 { 509 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 510 refcount_dec(&lo->plh_refcount); 511 } 512 513 static void 514 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 515 { 516 struct inode *inode = lo->plh_inode; 517 struct pnfs_layout_range range = { 518 .iomode = iomode, 519 .offset = 0, 520 .length = NFS4_MAX_UINT64, 521 }; 522 LIST_HEAD(head); 523 524 spin_lock(&inode->i_lock); 525 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 526 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 527 spin_unlock(&inode->i_lock); 528 pnfs_free_lseg_list(&head); 529 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 530 iomode == IOMODE_RW ? "RW" : "READ"); 531 } 532 533 static bool 534 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 535 { 536 unsigned long start, end; 537 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 538 539 if (test_bit(fail_bit, &lo->plh_flags) == 0) 540 return false; 541 end = jiffies; 542 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 543 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 544 /* It is time to retry the failed layoutgets */ 545 pnfs_layout_clear_fail_bit(lo, fail_bit); 546 return false; 547 } 548 return true; 549 } 550 551 static void 552 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 553 const struct pnfs_layout_range *range, 554 const nfs4_stateid *stateid) 555 { 556 INIT_LIST_HEAD(&lseg->pls_list); 557 INIT_LIST_HEAD(&lseg->pls_lc_list); 558 INIT_LIST_HEAD(&lseg->pls_commits); 559 refcount_set(&lseg->pls_refcount, 1); 560 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 561 lseg->pls_layout = lo; 562 lseg->pls_range = *range; 563 lseg->pls_seq = be32_to_cpu(stateid->seqid); 564 } 565 566 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 567 { 568 if (lseg != NULL) { 569 struct inode *inode = lseg->pls_layout->plh_inode; 570 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 571 } 572 } 573 574 static void 575 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 576 struct pnfs_layout_segment *lseg) 577 { 578 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 579 list_del_init(&lseg->pls_list); 580 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 581 refcount_dec(&lo->plh_refcount); 582 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 583 return; 584 if (list_empty(&lo->plh_segs) && 585 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 586 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 587 if (atomic_read(&lo->plh_outstanding) == 0) 588 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 589 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 590 } 591 } 592 593 static bool 594 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 595 struct pnfs_layout_segment *lseg) 596 { 597 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 598 pnfs_layout_is_valid(lo)) { 599 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 600 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 601 return true; 602 } 603 return false; 604 } 605 606 void 607 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 608 { 609 struct pnfs_layout_hdr *lo; 610 struct inode *inode; 611 612 if (!lseg) 613 return; 614 615 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 616 refcount_read(&lseg->pls_refcount), 617 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 618 619 lo = lseg->pls_layout; 620 inode = lo->plh_inode; 621 622 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 623 pnfs_get_layout_hdr(lo); 624 pnfs_layout_remove_lseg(lo, lseg); 625 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 626 lseg = NULL; 627 spin_unlock(&inode->i_lock); 628 pnfs_free_lseg(lseg); 629 pnfs_put_layout_hdr(lo); 630 } 631 } 632 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 633 634 /* 635 * is l2 fully contained in l1? 636 * start1 end1 637 * [----------------------------------) 638 * start2 end2 639 * [----------------) 640 */ 641 static bool 642 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 643 const struct pnfs_layout_range *l2) 644 { 645 u64 start1 = l1->offset; 646 u64 end1 = pnfs_end_offset(start1, l1->length); 647 u64 start2 = l2->offset; 648 u64 end2 = pnfs_end_offset(start2, l2->length); 649 650 return (start1 <= start2) && (end1 >= end2); 651 } 652 653 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 654 struct list_head *tmp_list) 655 { 656 if (!refcount_dec_and_test(&lseg->pls_refcount)) 657 return false; 658 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 659 list_add(&lseg->pls_list, tmp_list); 660 return true; 661 } 662 663 /* Returns 1 if lseg is removed from list, 0 otherwise */ 664 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 665 struct list_head *tmp_list) 666 { 667 int rv = 0; 668 669 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 670 /* Remove the reference keeping the lseg in the 671 * list. It will now be removed when all 672 * outstanding io is finished. 673 */ 674 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 675 refcount_read(&lseg->pls_refcount)); 676 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 677 rv = 1; 678 } 679 return rv; 680 } 681 682 static bool 683 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 684 const struct pnfs_layout_range *recall_range) 685 { 686 return (recall_range->iomode == IOMODE_ANY || 687 lseg_range->iomode == recall_range->iomode) && 688 pnfs_lseg_range_intersecting(lseg_range, recall_range); 689 } 690 691 static bool 692 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 693 const struct pnfs_layout_range *recall_range, 694 u32 seq) 695 { 696 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 697 return false; 698 if (recall_range == NULL) 699 return true; 700 return pnfs_should_free_range(&lseg->pls_range, recall_range); 701 } 702 703 /** 704 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 705 * @lo: layout header containing the lsegs 706 * @tmp_list: list head where doomed lsegs should go 707 * @recall_range: optional recall range argument to match (may be NULL) 708 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 709 * 710 * Walk the list of lsegs in the layout header, and tear down any that should 711 * be destroyed. If "recall_range" is specified then the segment must match 712 * that range. If "seq" is non-zero, then only match segments that were handed 713 * out at or before that sequence. 714 * 715 * Returns number of matching invalid lsegs remaining in list after scanning 716 * it and purging them. 717 */ 718 int 719 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 720 struct list_head *tmp_list, 721 const struct pnfs_layout_range *recall_range, 722 u32 seq) 723 { 724 struct pnfs_layout_segment *lseg, *next; 725 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 726 int remaining = 0; 727 728 dprintk("%s:Begin lo %p\n", __func__, lo); 729 730 if (list_empty(&lo->plh_segs)) 731 return 0; 732 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 733 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 734 dprintk("%s: freeing lseg %p iomode %d seq %u " 735 "offset %llu length %llu\n", __func__, 736 lseg, lseg->pls_range.iomode, lseg->pls_seq, 737 lseg->pls_range.offset, lseg->pls_range.length); 738 if (mark_lseg_invalid(lseg, tmp_list)) 739 continue; 740 remaining++; 741 pnfs_lseg_cancel_io(server, lseg); 742 } 743 dprintk("%s:Return %i\n", __func__, remaining); 744 return remaining; 745 } 746 747 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 748 { 749 struct pnfs_layout_segment *lseg; 750 751 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 752 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 753 } 754 755 static void 756 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 757 struct list_head *free_me, 758 const struct pnfs_layout_range *range, 759 u32 seq) 760 { 761 struct pnfs_layout_segment *lseg, *next; 762 763 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 764 if (pnfs_match_lseg_recall(lseg, range, seq)) 765 list_move_tail(&lseg->pls_list, free_me); 766 } 767 } 768 769 /* note free_me must contain lsegs from a single layout_hdr */ 770 void 771 pnfs_free_lseg_list(struct list_head *free_me) 772 { 773 struct pnfs_layout_segment *lseg, *tmp; 774 775 if (list_empty(free_me)) 776 return; 777 778 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 779 list_del(&lseg->pls_list); 780 pnfs_free_lseg(lseg); 781 } 782 } 783 784 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 785 { 786 struct pnfs_layout_hdr *lo; 787 LIST_HEAD(tmp_list); 788 789 spin_lock(&nfsi->vfs_inode.i_lock); 790 lo = nfsi->layout; 791 if (lo) { 792 pnfs_get_layout_hdr(lo); 793 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 794 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 795 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 796 spin_unlock(&nfsi->vfs_inode.i_lock); 797 pnfs_free_lseg_list(&tmp_list); 798 nfs_commit_inode(&nfsi->vfs_inode, 0); 799 pnfs_put_layout_hdr(lo); 800 } else 801 spin_unlock(&nfsi->vfs_inode.i_lock); 802 return lo; 803 } 804 805 void pnfs_destroy_layout(struct nfs_inode *nfsi) 806 { 807 __pnfs_destroy_layout(nfsi); 808 } 809 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 810 811 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 812 { 813 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 814 struct inode *inode = &nfsi->vfs_inode; 815 816 if (lo) { 817 spin_lock(&inode->i_lock); 818 wait_var_event_spinlock(lo, nfsi->layout != lo, 819 &inode->i_lock); 820 spin_unlock(&inode->i_lock); 821 } 822 } 823 824 static bool 825 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 826 struct list_head *layout_list) 827 { 828 struct pnfs_layout_hdr *lo; 829 bool ret = false; 830 831 spin_lock(&inode->i_lock); 832 lo = NFS_I(inode)->layout; 833 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 834 pnfs_get_layout_hdr(lo); 835 list_add(&lo->plh_bulk_destroy, layout_list); 836 ret = true; 837 } 838 spin_unlock(&inode->i_lock); 839 return ret; 840 } 841 842 /* Caller must hold rcu_read_lock and clp->cl_lock */ 843 static int 844 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 845 struct nfs_server *server, 846 struct list_head *layout_list) 847 __must_hold(&clp->cl_lock) 848 __must_hold(RCU) 849 { 850 struct pnfs_layout_hdr *lo, *next; 851 struct inode *inode; 852 853 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 854 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 855 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 856 !list_empty(&lo->plh_bulk_destroy)) 857 continue; 858 /* If the sb is being destroyed, just bail */ 859 if (!nfs_sb_active(server->super)) 860 break; 861 inode = pnfs_grab_inode_layout_hdr(lo); 862 if (inode != NULL) { 863 if (pnfs_layout_add_bulk_destroy_list(inode, 864 layout_list)) 865 continue; 866 rcu_read_unlock(); 867 spin_unlock(&clp->cl_lock); 868 iput(inode); 869 } else { 870 rcu_read_unlock(); 871 spin_unlock(&clp->cl_lock); 872 } 873 nfs_sb_deactive(server->super); 874 spin_lock(&clp->cl_lock); 875 rcu_read_lock(); 876 return -EAGAIN; 877 } 878 return 0; 879 } 880 881 static int 882 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 883 enum pnfs_layout_destroy_mode mode) 884 { 885 struct pnfs_layout_hdr *lo; 886 struct inode *inode; 887 LIST_HEAD(lseg_list); 888 int ret = 0; 889 890 while (!list_empty(layout_list)) { 891 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 892 plh_bulk_destroy); 893 dprintk("%s freeing layout for inode %lu\n", __func__, 894 lo->plh_inode->i_ino); 895 inode = lo->plh_inode; 896 897 pnfs_layoutcommit_inode(inode, false); 898 899 spin_lock(&inode->i_lock); 900 list_del_init(&lo->plh_bulk_destroy); 901 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 902 pnfs_mark_layout_stateid_return(lo, &lseg_list, 903 IOMODE_ANY, 0); 904 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 905 if (mode == PNFS_LAYOUT_BULK_RETURN) 906 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 907 ret = -EAGAIN; 908 } 909 spin_unlock(&inode->i_lock); 910 pnfs_free_lseg_list(&lseg_list); 911 /* Free all lsegs that are attached to commit buckets */ 912 nfs_commit_inode(inode, 0); 913 pnfs_put_layout_hdr(lo); 914 nfs_iput_and_deactive(inode); 915 } 916 return ret; 917 } 918 919 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 920 enum pnfs_layout_destroy_mode mode) 921 { 922 struct nfs_server *server; 923 LIST_HEAD(layout_list); 924 925 spin_lock(&clp->cl_lock); 926 rcu_read_lock(); 927 restart: 928 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 929 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 930 continue; 931 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 932 server, 933 &layout_list) != 0) 934 goto restart; 935 } 936 rcu_read_unlock(); 937 spin_unlock(&clp->cl_lock); 938 939 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 940 } 941 942 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 943 struct list_head *list) 944 { 945 struct nfs_server *server; 946 947 spin_lock(&clp->cl_lock); 948 rcu_read_lock(); 949 restart: 950 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 951 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 952 list) != 0) 953 goto restart; 954 } 955 rcu_read_unlock(); 956 spin_unlock(&clp->cl_lock); 957 } 958 959 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 960 struct list_head *list, 961 enum pnfs_layout_destroy_mode mode) 962 { 963 pnfs_layout_build_destroy_list_byclient(clp, list); 964 return pnfs_layout_free_bulk_destroy_list(list, mode); 965 } 966 967 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 968 enum pnfs_layout_destroy_mode mode) 969 { 970 LIST_HEAD(layout_list); 971 972 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 973 } 974 975 /* 976 * Called by the state manager to remove all layouts established under an 977 * expired lease. 978 */ 979 void 980 pnfs_destroy_all_layouts(struct nfs_client *clp) 981 { 982 nfs4_deviceid_mark_client_invalid(clp); 983 nfs4_deviceid_purge_client(clp); 984 985 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 986 } 987 988 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 989 struct list_head *list) 990 { 991 struct nfs_server *server; 992 993 spin_lock(&clp->cl_lock); 994 rcu_read_lock(); 995 restart: 996 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 997 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 998 continue; 999 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 1000 list) != 0) 1001 goto restart; 1002 } 1003 rcu_read_unlock(); 1004 spin_unlock(&clp->cl_lock); 1005 } 1006 1007 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1008 { 1009 struct pnfs_layout_hdr *lo; 1010 struct nfs_server *server; 1011 int ret; 1012 1013 list_for_each_entry(lo, list, plh_bulk_destroy) { 1014 server = NFS_SERVER(lo->plh_inode); 1015 ret = pnfs_layout_return_on_reboot(lo); 1016 switch (ret) { 1017 case 0: 1018 continue; 1019 case -NFS4ERR_BAD_STATEID: 1020 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1021 break; 1022 case -NFS4ERR_NO_GRACE: 1023 break; 1024 default: 1025 goto err; 1026 } 1027 break; 1028 } 1029 return 0; 1030 err: 1031 return ret; 1032 } 1033 1034 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1035 { 1036 LIST_HEAD(list); 1037 int ret = 0, ret2; 1038 1039 pnfs_layout_build_recover_list_byclient(clp, &list); 1040 if (!list_empty(&list)) 1041 ret = pnfs_layout_bulk_list_reboot(&list); 1042 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1043 PNFS_LAYOUT_INVALIDATE); 1044 if (!ret) 1045 ret = ret2; 1046 return (ret == 0) ? 0 : -EAGAIN; 1047 } 1048 1049 static void 1050 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1051 { 1052 const struct cred *old; 1053 1054 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1055 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1056 put_cred(old); 1057 } 1058 } 1059 1060 /* update lo->plh_stateid with new if is more recent */ 1061 void 1062 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1063 const struct cred *cred, bool update_barrier) 1064 { 1065 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1066 u32 newseq = be32_to_cpu(new->seqid); 1067 1068 if (!pnfs_layout_is_valid(lo)) { 1069 pnfs_set_layout_cred(lo, cred); 1070 nfs4_stateid_copy(&lo->plh_stateid, new); 1071 lo->plh_barrier = newseq; 1072 pnfs_clear_layoutreturn_info(lo); 1073 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1074 return; 1075 } 1076 1077 if (pnfs_seqid_is_newer(newseq, oldseq)) 1078 nfs4_stateid_copy(&lo->plh_stateid, new); 1079 1080 if (update_barrier) { 1081 pnfs_barrier_update(lo, newseq); 1082 return; 1083 } 1084 /* 1085 * Because of wraparound, we want to keep the barrier 1086 * "close" to the current seqids. We really only want to 1087 * get here from a layoutget call. 1088 */ 1089 if (atomic_read(&lo->plh_outstanding) == 1) 1090 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1091 } 1092 1093 static bool 1094 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1095 const nfs4_stateid *stateid) 1096 { 1097 u32 seqid = be32_to_cpu(stateid->seqid); 1098 1099 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1100 } 1101 1102 /* lget is set to 1 if called from inside send_layoutget call chain */ 1103 static bool 1104 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1105 { 1106 return lo->plh_block_lgets || 1107 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1108 } 1109 1110 static struct nfs_server * 1111 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1112 { 1113 struct nfs_server *server; 1114 1115 if (inode) { 1116 server = NFS_SERVER(inode); 1117 } else { 1118 struct dentry *parent_dir = dget_parent(ctx->dentry); 1119 server = NFS_SERVER(parent_dir->d_inode); 1120 dput(parent_dir); 1121 } 1122 return server; 1123 } 1124 1125 static void nfs4_free_pages(struct page **pages, size_t size) 1126 { 1127 int i; 1128 1129 if (!pages) 1130 return; 1131 1132 for (i = 0; i < size; i++) { 1133 if (!pages[i]) 1134 break; 1135 __free_page(pages[i]); 1136 } 1137 kfree(pages); 1138 } 1139 1140 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1141 { 1142 struct page **pages; 1143 int i; 1144 1145 pages = kmalloc_array(size, sizeof(struct page *), gfp_flags); 1146 if (!pages) { 1147 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1148 return NULL; 1149 } 1150 1151 for (i = 0; i < size; i++) { 1152 pages[i] = alloc_page(gfp_flags); 1153 if (!pages[i]) { 1154 dprintk("%s: failed to allocate page\n", __func__); 1155 nfs4_free_pages(pages, i); 1156 return NULL; 1157 } 1158 } 1159 1160 return pages; 1161 } 1162 1163 static struct nfs4_layoutget * 1164 pnfs_alloc_init_layoutget_args(struct inode *ino, 1165 struct nfs_open_context *ctx, 1166 const nfs4_stateid *stateid, 1167 const struct pnfs_layout_range *range, 1168 gfp_t gfp_flags) 1169 { 1170 struct nfs_server *server = pnfs_find_server(ino, ctx); 1171 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1172 size_t max_pages = max_response_pages(server); 1173 struct nfs4_layoutget *lgp; 1174 1175 dprintk("--> %s\n", __func__); 1176 1177 lgp = kzalloc(sizeof(*lgp), gfp_flags); 1178 if (lgp == NULL) 1179 return NULL; 1180 1181 if (max_reply_sz) { 1182 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1183 if (npages < max_pages) 1184 max_pages = npages; 1185 } 1186 1187 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1188 if (!lgp->args.layout.pages) { 1189 kfree(lgp); 1190 return NULL; 1191 } 1192 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1193 lgp->res.layoutp = &lgp->args.layout; 1194 1195 /* Don't confuse uninitialised result and success */ 1196 lgp->res.status = -NFS4ERR_DELAY; 1197 1198 lgp->args.minlength = PAGE_SIZE; 1199 if (lgp->args.minlength > range->length) 1200 lgp->args.minlength = range->length; 1201 if (ino) { 1202 loff_t i_size = i_size_read(ino); 1203 1204 if (range->iomode == IOMODE_READ) { 1205 if (range->offset >= i_size) 1206 lgp->args.minlength = 0; 1207 else if (i_size - range->offset < lgp->args.minlength) 1208 lgp->args.minlength = i_size - range->offset; 1209 } 1210 } 1211 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 1212 pnfs_copy_range(&lgp->args.range, range); 1213 lgp->args.type = server->pnfs_curr_ld->id; 1214 lgp->args.inode = ino; 1215 lgp->args.ctx = get_nfs_open_context(ctx); 1216 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1217 lgp->gfp_flags = gfp_flags; 1218 lgp->cred = ctx->cred; 1219 return lgp; 1220 } 1221 1222 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1223 { 1224 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1225 1226 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1227 pnfs_put_layout_hdr(lgp->lo); 1228 put_nfs_open_context(lgp->args.ctx); 1229 kfree(lgp); 1230 } 1231 1232 static void pnfs_clear_layoutcommit(struct inode *inode, 1233 struct list_head *head) 1234 { 1235 struct nfs_inode *nfsi = NFS_I(inode); 1236 struct pnfs_layout_segment *lseg, *tmp; 1237 1238 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1239 return; 1240 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1241 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1242 continue; 1243 pnfs_lseg_dec_and_remove_zero(lseg, head); 1244 } 1245 } 1246 1247 static void 1248 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1249 const nfs4_stateid *arg_stateid, 1250 const struct pnfs_layout_range *range, 1251 struct list_head *freeme) 1252 { 1253 if (pnfs_layout_is_valid(lo) && 1254 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1255 pnfs_reset_return_info(lo); 1256 else 1257 pnfs_mark_layout_stateid_invalid(lo, freeme); 1258 pnfs_clear_layoutreturn_waitbit(lo); 1259 } 1260 1261 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1262 const nfs4_stateid *arg_stateid, 1263 const struct pnfs_layout_range *range) 1264 { 1265 struct inode *inode = lo->plh_inode; 1266 LIST_HEAD(freeme); 1267 1268 spin_lock(&inode->i_lock); 1269 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1270 spin_unlock(&inode->i_lock); 1271 pnfs_free_lseg_list(&freeme); 1272 } 1273 1274 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1275 const nfs4_stateid *arg_stateid, 1276 const struct pnfs_layout_range *range, 1277 const nfs4_stateid *stateid) 1278 { 1279 struct inode *inode = lo->plh_inode; 1280 LIST_HEAD(freeme); 1281 1282 spin_lock(&inode->i_lock); 1283 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1284 goto out_unlock; 1285 if (stateid && pnfs_layout_is_valid(lo)) { 1286 u32 seq = be32_to_cpu(arg_stateid->seqid); 1287 1288 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1289 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1290 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1291 pnfs_reset_return_info(lo); 1292 } else 1293 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1294 out_unlock: 1295 pnfs_clear_layoutreturn_waitbit(lo); 1296 spin_unlock(&inode->i_lock); 1297 pnfs_free_lseg_list(&freeme); 1298 1299 } 1300 1301 static bool 1302 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1303 nfs4_stateid *stateid, 1304 const struct cred **cred, 1305 enum pnfs_iomode *iomode) 1306 { 1307 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1308 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1309 return false; 1310 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1311 return false; 1312 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1313 pnfs_get_layout_hdr(lo); 1314 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1315 *cred = get_cred(lo->plh_lc_cred); 1316 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1317 if (lo->plh_return_seq != 0) 1318 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1319 if (iomode != NULL) 1320 *iomode = lo->plh_return_iomode; 1321 pnfs_clear_layoutreturn_info(lo); 1322 } else if (iomode != NULL) 1323 *iomode = IOMODE_ANY; 1324 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1325 return true; 1326 } 1327 1328 static void 1329 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1330 struct pnfs_layout_hdr *lo, 1331 const nfs4_stateid *stateid, 1332 enum pnfs_iomode iomode) 1333 { 1334 struct inode *inode = lo->plh_inode; 1335 1336 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1337 args->inode = inode; 1338 args->range.iomode = iomode; 1339 args->range.offset = 0; 1340 args->range.length = NFS4_MAX_UINT64; 1341 args->layout = lo; 1342 nfs4_stateid_copy(&args->stateid, stateid); 1343 } 1344 1345 static int 1346 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1347 const nfs4_stateid *stateid, 1348 const struct cred **pcred, 1349 enum pnfs_iomode iomode, 1350 unsigned int flags) 1351 { 1352 struct inode *ino = lo->plh_inode; 1353 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1354 struct nfs4_layoutreturn *lrp; 1355 const struct cred *cred = *pcred; 1356 int status = 0; 1357 1358 *pcred = NULL; 1359 lrp = kzalloc(sizeof(*lrp), nfs_io_gfp_mask()); 1360 if (unlikely(lrp == NULL)) { 1361 status = -ENOMEM; 1362 spin_lock(&ino->i_lock); 1363 pnfs_clear_layoutreturn_waitbit(lo); 1364 spin_unlock(&ino->i_lock); 1365 put_cred(cred); 1366 pnfs_put_layout_hdr(lo); 1367 goto out; 1368 } 1369 1370 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1371 lrp->args.ld_private = &lrp->ld_private; 1372 lrp->clp = NFS_SERVER(ino)->nfs_client; 1373 lrp->cred = cred; 1374 if (ld->prepare_layoutreturn) 1375 ld->prepare_layoutreturn(&lrp->args); 1376 1377 status = nfs4_proc_layoutreturn(lrp, flags); 1378 out: 1379 dprintk("<-- %s status: %d\n", __func__, status); 1380 return status; 1381 } 1382 1383 /* Return true if layoutreturn is needed */ 1384 static bool 1385 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1386 { 1387 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1388 return false; 1389 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1390 lo->plh_return_iomode, 1391 lo->plh_return_seq) != EBUSY; 1392 } 1393 1394 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1395 { 1396 struct inode *inode= lo->plh_inode; 1397 1398 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1399 return; 1400 spin_lock(&inode->i_lock); 1401 if (pnfs_layout_need_return(lo)) { 1402 const struct cred *cred; 1403 nfs4_stateid stateid; 1404 enum pnfs_iomode iomode; 1405 bool send; 1406 1407 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1408 spin_unlock(&inode->i_lock); 1409 if (send) { 1410 /* Send an async layoutreturn so we dont deadlock */ 1411 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1412 PNFS_FL_LAYOUTRETURN_ASYNC); 1413 } 1414 } else 1415 spin_unlock(&inode->i_lock); 1416 } 1417 1418 /* 1419 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1420 * when the layout segment list is empty. 1421 * 1422 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1423 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1424 * deviceid is marked invalid. 1425 */ 1426 int 1427 _pnfs_return_layout(struct inode *ino) 1428 { 1429 struct pnfs_layout_hdr *lo = NULL; 1430 struct nfs_inode *nfsi = NFS_I(ino); 1431 struct pnfs_layout_range range = { 1432 .iomode = IOMODE_ANY, 1433 .offset = 0, 1434 .length = NFS4_MAX_UINT64, 1435 }; 1436 LIST_HEAD(tmp_list); 1437 const struct cred *cred; 1438 nfs4_stateid stateid; 1439 int status = 0; 1440 bool send, valid_layout; 1441 1442 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1443 1444 spin_lock(&ino->i_lock); 1445 lo = nfsi->layout; 1446 if (!lo) { 1447 spin_unlock(&ino->i_lock); 1448 dprintk("NFS: %s no layout to return\n", __func__); 1449 goto out; 1450 } 1451 /* Reference matched in nfs4_layoutreturn_release */ 1452 pnfs_get_layout_hdr(lo); 1453 /* Is there an outstanding layoutreturn ? */ 1454 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1455 spin_unlock(&ino->i_lock); 1456 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1457 TASK_UNINTERRUPTIBLE)) 1458 goto out_put_layout_hdr; 1459 spin_lock(&ino->i_lock); 1460 } 1461 valid_layout = pnfs_layout_is_valid(lo); 1462 pnfs_clear_layoutcommit(ino, &tmp_list); 1463 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); 1464 1465 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1466 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1467 1468 /* Don't send a LAYOUTRETURN if list was initially empty */ 1469 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1470 !valid_layout) { 1471 spin_unlock(&ino->i_lock); 1472 dprintk("NFS: %s no layout segments to return\n", __func__); 1473 goto out_wait_layoutreturn; 1474 } 1475 1476 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1477 spin_unlock(&ino->i_lock); 1478 if (send) 1479 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1480 0); 1481 out_wait_layoutreturn: 1482 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1483 out_put_layout_hdr: 1484 pnfs_free_lseg_list(&tmp_list); 1485 pnfs_put_layout_hdr(lo); 1486 out: 1487 dprintk("<-- %s status: %d\n", __func__, status); 1488 return status; 1489 } 1490 1491 int 1492 pnfs_commit_and_return_layout(struct inode *inode) 1493 { 1494 struct pnfs_layout_hdr *lo; 1495 int ret; 1496 1497 spin_lock(&inode->i_lock); 1498 lo = NFS_I(inode)->layout; 1499 if (lo == NULL) { 1500 spin_unlock(&inode->i_lock); 1501 return 0; 1502 } 1503 pnfs_get_layout_hdr(lo); 1504 /* Block new layoutgets and read/write to ds */ 1505 lo->plh_block_lgets++; 1506 spin_unlock(&inode->i_lock); 1507 filemap_fdatawait(inode->i_mapping); 1508 ret = pnfs_layoutcommit_inode(inode, true); 1509 if (ret == 0) 1510 ret = _pnfs_return_layout(inode); 1511 spin_lock(&inode->i_lock); 1512 lo->plh_block_lgets--; 1513 spin_unlock(&inode->i_lock); 1514 pnfs_put_layout_hdr(lo); 1515 return ret; 1516 } 1517 1518 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1519 { 1520 struct inode *inode = lo->plh_inode; 1521 const struct cred *cred; 1522 1523 spin_lock(&inode->i_lock); 1524 if (!pnfs_layout_is_valid(lo)) { 1525 spin_unlock(&inode->i_lock); 1526 return 0; 1527 } 1528 cred = get_cred(lo->plh_lc_cred); 1529 pnfs_get_layout_hdr(lo); 1530 spin_unlock(&inode->i_lock); 1531 1532 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1533 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1534 } 1535 1536 bool pnfs_roc(struct inode *ino, 1537 struct nfs4_layoutreturn_args *args, 1538 struct nfs4_layoutreturn_res *res, 1539 const struct cred *cred) 1540 { 1541 struct nfs_inode *nfsi = NFS_I(ino); 1542 struct nfs_open_context *ctx; 1543 struct nfs4_state *state; 1544 struct pnfs_layout_hdr *lo; 1545 struct pnfs_layout_segment *lseg, *next; 1546 const struct cred *lc_cred; 1547 nfs4_stateid stateid; 1548 enum pnfs_iomode iomode = 0; 1549 bool layoutreturn = false, roc = false; 1550 bool skip_read = false; 1551 1552 if (!nfs_have_layout(ino)) 1553 return false; 1554 retry: 1555 rcu_read_lock(); 1556 spin_lock(&ino->i_lock); 1557 lo = nfsi->layout; 1558 if (!lo || !pnfs_layout_is_valid(lo) || 1559 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1560 lo = NULL; 1561 goto out_noroc; 1562 } 1563 pnfs_get_layout_hdr(lo); 1564 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1565 spin_unlock(&ino->i_lock); 1566 rcu_read_unlock(); 1567 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1568 TASK_UNINTERRUPTIBLE); 1569 pnfs_put_layout_hdr(lo); 1570 goto retry; 1571 } 1572 1573 /* no roc if we hold a delegation */ 1574 if (nfs4_check_delegation(ino, FMODE_READ)) { 1575 if (nfs4_check_delegation(ino, FMODE_WRITE)) 1576 goto out_noroc; 1577 skip_read = true; 1578 } 1579 1580 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1581 state = ctx->state; 1582 if (state == NULL) 1583 continue; 1584 /* Don't return layout if there is open file state */ 1585 if (state->state & FMODE_WRITE) 1586 goto out_noroc; 1587 if (state->state & FMODE_READ) 1588 skip_read = true; 1589 } 1590 1591 1592 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1593 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1594 continue; 1595 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1596 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1597 continue; 1598 /* 1599 * Note: mark lseg for return so pnfs_layout_remove_lseg 1600 * doesn't invalidate the layout for us. 1601 */ 1602 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1603 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1604 continue; 1605 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1606 } 1607 1608 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1609 goto out_noroc; 1610 1611 /* ROC in two conditions: 1612 * 1. there are ROC lsegs 1613 * 2. we don't send layoutreturn 1614 */ 1615 /* lo ref dropped in pnfs_roc_release() */ 1616 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1617 /* If the creds don't match, we can't compound the layoutreturn */ 1618 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1619 goto out_noroc; 1620 1621 roc = layoutreturn; 1622 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1623 res->lrs_present = 0; 1624 layoutreturn = false; 1625 put_cred(lc_cred); 1626 1627 out_noroc: 1628 spin_unlock(&ino->i_lock); 1629 rcu_read_unlock(); 1630 pnfs_layoutcommit_inode(ino, true); 1631 if (roc) { 1632 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1633 if (ld->prepare_layoutreturn) 1634 ld->prepare_layoutreturn(args); 1635 pnfs_put_layout_hdr(lo); 1636 return true; 1637 } 1638 if (layoutreturn) 1639 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1640 pnfs_put_layout_hdr(lo); 1641 return false; 1642 } 1643 1644 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1645 struct nfs4_layoutreturn_res **respp, int *ret) 1646 { 1647 struct nfs4_layoutreturn_args *arg = *argpp; 1648 int retval = -EAGAIN; 1649 1650 if (!arg) 1651 return 0; 1652 /* Handle Layoutreturn errors */ 1653 switch (*ret) { 1654 case 0: 1655 retval = 0; 1656 break; 1657 case -NFS4ERR_NOMATCHING_LAYOUT: 1658 /* Was there an RPC level error? If not, retry */ 1659 if (task->tk_rpc_status == 0) 1660 break; 1661 /* 1662 * Is there a fatal network level error? 1663 * If so release the layout, but flag the error. 1664 */ 1665 if ((task->tk_rpc_status == -ENETDOWN || 1666 task->tk_rpc_status == -ENETUNREACH) && 1667 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1668 *ret = 0; 1669 (*respp)->lrs_present = 0; 1670 retval = -EIO; 1671 break; 1672 } 1673 /* If the call was not sent, let caller handle it */ 1674 if (!RPC_WAS_SENT(task)) 1675 return 0; 1676 /* 1677 * Otherwise, assume the call succeeded and 1678 * that we need to release the layout 1679 */ 1680 *ret = 0; 1681 (*respp)->lrs_present = 0; 1682 retval = 0; 1683 break; 1684 case -NFS4ERR_DELAY: 1685 /* Let the caller handle the retry */ 1686 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1687 return 0; 1688 case -NFS4ERR_OLD_STATEID: 1689 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1690 &arg->range, arg->inode)) 1691 break; 1692 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1693 return -EAGAIN; 1694 } 1695 *argpp = NULL; 1696 *respp = NULL; 1697 return retval; 1698 } 1699 1700 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1701 struct nfs4_layoutreturn_res *res, int ret) 1702 { 1703 struct pnfs_layout_hdr *lo = args->layout; 1704 struct inode *inode = args->inode; 1705 const nfs4_stateid *res_stateid = NULL; 1706 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1707 LIST_HEAD(freeme); 1708 1709 switch (ret) { 1710 case -NFS4ERR_BADSESSION: 1711 case -NFS4ERR_DEADSESSION: 1712 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1713 case -NFS4ERR_NOMATCHING_LAYOUT: 1714 spin_lock(&inode->i_lock); 1715 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1716 &args->range, &freeme); 1717 spin_unlock(&inode->i_lock); 1718 pnfs_free_lseg_list(&freeme); 1719 break; 1720 case 0: 1721 if (res->lrs_present) 1722 res_stateid = &res->stateid; 1723 fallthrough; 1724 default: 1725 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1726 res_stateid); 1727 } 1728 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1729 if (ld_private && ld_private->ops && ld_private->ops->free) 1730 ld_private->ops->free(ld_private); 1731 pnfs_put_layout_hdr(lo); 1732 } 1733 1734 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1735 { 1736 struct nfs_inode *nfsi = NFS_I(ino); 1737 struct pnfs_layout_hdr *lo; 1738 bool sleep = false; 1739 1740 /* we might not have grabbed lo reference. so need to check under 1741 * i_lock */ 1742 spin_lock(&ino->i_lock); 1743 lo = nfsi->layout; 1744 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1745 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1746 sleep = true; 1747 } 1748 spin_unlock(&ino->i_lock); 1749 return sleep; 1750 } 1751 1752 /* 1753 * Compare two layout segments for sorting into layout cache. 1754 * We want to preferentially return RW over RO layouts, so ensure those 1755 * are seen first. 1756 */ 1757 static s64 1758 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1759 const struct pnfs_layout_range *l2) 1760 { 1761 s64 d; 1762 1763 /* high offset > low offset */ 1764 d = l1->offset - l2->offset; 1765 if (d) 1766 return d; 1767 1768 /* short length > long length */ 1769 d = l2->length - l1->length; 1770 if (d) 1771 return d; 1772 1773 /* read > read/write */ 1774 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1775 } 1776 1777 static bool 1778 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1779 const struct pnfs_layout_range *l2) 1780 { 1781 return pnfs_lseg_range_cmp(l1, l2) > 0; 1782 } 1783 1784 static bool 1785 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1786 struct pnfs_layout_segment *old) 1787 { 1788 return false; 1789 } 1790 1791 void 1792 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1793 struct pnfs_layout_segment *lseg, 1794 bool (*is_after)(const struct pnfs_layout_range *, 1795 const struct pnfs_layout_range *), 1796 bool (*do_merge)(struct pnfs_layout_segment *, 1797 struct pnfs_layout_segment *), 1798 struct list_head *free_me) 1799 { 1800 struct pnfs_layout_segment *lp, *tmp; 1801 1802 dprintk("%s:Begin\n", __func__); 1803 1804 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1805 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1806 continue; 1807 if (do_merge(lseg, lp)) { 1808 mark_lseg_invalid(lp, free_me); 1809 continue; 1810 } 1811 if (is_after(&lseg->pls_range, &lp->pls_range)) 1812 continue; 1813 list_add_tail(&lseg->pls_list, &lp->pls_list); 1814 dprintk("%s: inserted lseg %p " 1815 "iomode %d offset %llu length %llu before " 1816 "lp %p iomode %d offset %llu length %llu\n", 1817 __func__, lseg, lseg->pls_range.iomode, 1818 lseg->pls_range.offset, lseg->pls_range.length, 1819 lp, lp->pls_range.iomode, lp->pls_range.offset, 1820 lp->pls_range.length); 1821 goto out; 1822 } 1823 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1824 dprintk("%s: inserted lseg %p " 1825 "iomode %d offset %llu length %llu at tail\n", 1826 __func__, lseg, lseg->pls_range.iomode, 1827 lseg->pls_range.offset, lseg->pls_range.length); 1828 out: 1829 pnfs_get_layout_hdr(lo); 1830 1831 dprintk("%s:Return\n", __func__); 1832 } 1833 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1834 1835 static void 1836 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1837 struct pnfs_layout_segment *lseg, 1838 struct list_head *free_me) 1839 { 1840 struct inode *inode = lo->plh_inode; 1841 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1842 1843 if (ld->add_lseg != NULL) 1844 ld->add_lseg(lo, lseg, free_me); 1845 else 1846 pnfs_generic_layout_insert_lseg(lo, lseg, 1847 pnfs_lseg_range_is_after, 1848 pnfs_lseg_no_merge, 1849 free_me); 1850 } 1851 1852 static struct pnfs_layout_hdr * 1853 alloc_init_layout_hdr(struct inode *ino, 1854 struct nfs_open_context *ctx, 1855 gfp_t gfp_flags) 1856 { 1857 struct pnfs_layout_hdr *lo; 1858 1859 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1860 if (!lo) 1861 return NULL; 1862 refcount_set(&lo->plh_refcount, 1); 1863 INIT_LIST_HEAD(&lo->plh_layouts); 1864 INIT_LIST_HEAD(&lo->plh_segs); 1865 INIT_LIST_HEAD(&lo->plh_return_segs); 1866 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1867 lo->plh_inode = ino; 1868 lo->plh_lc_cred = get_cred(ctx->cred); 1869 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1870 return lo; 1871 } 1872 1873 static struct pnfs_layout_hdr * 1874 pnfs_find_alloc_layout(struct inode *ino, 1875 struct nfs_open_context *ctx, 1876 gfp_t gfp_flags) 1877 __releases(&ino->i_lock) 1878 __acquires(&ino->i_lock) 1879 { 1880 struct nfs_inode *nfsi = NFS_I(ino); 1881 struct pnfs_layout_hdr *new = NULL; 1882 1883 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1884 1885 if (nfsi->layout != NULL) 1886 goto out_existing; 1887 spin_unlock(&ino->i_lock); 1888 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1889 spin_lock(&ino->i_lock); 1890 1891 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1892 nfsi->layout = new; 1893 return new; 1894 } else if (new != NULL) 1895 pnfs_free_layout_hdr(new); 1896 out_existing: 1897 pnfs_get_layout_hdr(nfsi->layout); 1898 return nfsi->layout; 1899 } 1900 1901 /* 1902 * iomode matching rules: 1903 * iomode lseg strict match 1904 * iomode 1905 * ----- ----- ------ ----- 1906 * ANY READ N/A true 1907 * ANY RW N/A true 1908 * RW READ N/A false 1909 * RW RW N/A true 1910 * READ READ N/A true 1911 * READ RW true false 1912 * READ RW false true 1913 */ 1914 static bool 1915 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1916 const struct pnfs_layout_range *range, 1917 bool strict_iomode) 1918 { 1919 struct pnfs_layout_range range1; 1920 1921 if ((range->iomode == IOMODE_RW && 1922 ls_range->iomode != IOMODE_RW) || 1923 (range->iomode != ls_range->iomode && 1924 strict_iomode) || 1925 !pnfs_lseg_range_intersecting(ls_range, range)) 1926 return false; 1927 1928 /* range1 covers only the first byte in the range */ 1929 range1 = *range; 1930 range1.length = 1; 1931 return pnfs_lseg_range_contained(ls_range, &range1); 1932 } 1933 1934 /* 1935 * lookup range in layout 1936 */ 1937 static struct pnfs_layout_segment * 1938 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1939 struct pnfs_layout_range *range, 1940 bool strict_iomode) 1941 { 1942 struct pnfs_layout_segment *lseg, *ret = NULL; 1943 1944 dprintk("%s:Begin\n", __func__); 1945 1946 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1947 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1948 pnfs_lseg_range_match(&lseg->pls_range, range, 1949 strict_iomode)) { 1950 ret = pnfs_get_lseg(lseg); 1951 break; 1952 } 1953 } 1954 1955 dprintk("%s:Return lseg %p ref %d\n", 1956 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 1957 return ret; 1958 } 1959 1960 /* 1961 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1962 * to the MDS or over pNFS 1963 * 1964 * The nfs_inode read_io and write_io fields are cumulative counters reset 1965 * when there are no layout segments. Note that in pnfs_update_layout iomode 1966 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1967 * WRITE request. 1968 * 1969 * A return of true means use MDS I/O. 1970 * 1971 * From rfc 5661: 1972 * If a file's size is smaller than the file size threshold, data accesses 1973 * SHOULD be sent to the metadata server. If an I/O request has a length that 1974 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1975 * server. If both file size and I/O size are provided, the client SHOULD 1976 * reach or exceed both thresholds before sending its read or write 1977 * requests to the data server. 1978 */ 1979 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1980 struct inode *ino, int iomode) 1981 { 1982 struct nfs4_threshold *t = ctx->mdsthreshold; 1983 struct nfs_inode *nfsi = NFS_I(ino); 1984 loff_t fsize = i_size_read(ino); 1985 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1986 1987 if (t == NULL) 1988 return ret; 1989 1990 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1991 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1992 1993 switch (iomode) { 1994 case IOMODE_READ: 1995 if (t->bm & THRESHOLD_RD) { 1996 dprintk("%s fsize %llu\n", __func__, fsize); 1997 size_set = true; 1998 if (fsize < t->rd_sz) 1999 size = true; 2000 } 2001 if (t->bm & THRESHOLD_RD_IO) { 2002 dprintk("%s nfsi->read_io %llu\n", __func__, 2003 nfsi->read_io); 2004 io_set = true; 2005 if (nfsi->read_io < t->rd_io_sz) 2006 io = true; 2007 } 2008 break; 2009 case IOMODE_RW: 2010 if (t->bm & THRESHOLD_WR) { 2011 dprintk("%s fsize %llu\n", __func__, fsize); 2012 size_set = true; 2013 if (fsize < t->wr_sz) 2014 size = true; 2015 } 2016 if (t->bm & THRESHOLD_WR_IO) { 2017 dprintk("%s nfsi->write_io %llu\n", __func__, 2018 nfsi->write_io); 2019 io_set = true; 2020 if (nfsi->write_io < t->wr_io_sz) 2021 io = true; 2022 } 2023 break; 2024 } 2025 if (size_set && io_set) { 2026 if (size && io) 2027 ret = true; 2028 } else if (size || io) 2029 ret = true; 2030 2031 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2032 return ret; 2033 } 2034 2035 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2036 { 2037 /* 2038 * send layoutcommit as it can hold up layoutreturn due to lseg 2039 * reference 2040 */ 2041 pnfs_layoutcommit_inode(lo->plh_inode, false); 2042 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2043 nfs_wait_bit_killable, 2044 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2045 } 2046 2047 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2048 { 2049 atomic_inc(&lo->plh_outstanding); 2050 } 2051 2052 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2053 { 2054 if (atomic_dec_and_test(&lo->plh_outstanding) && 2055 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2056 smp_mb__after_atomic(); 2057 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2058 } 2059 } 2060 2061 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2062 { 2063 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2064 } 2065 2066 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 2067 { 2068 unsigned long *bitlock = &lo->plh_flags; 2069 2070 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 2071 smp_mb__after_atomic(); 2072 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 2073 } 2074 2075 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2076 struct nfs_server *server) 2077 { 2078 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2079 struct nfs_client *clp = server->nfs_client; 2080 2081 /* The lo must be on the clp list if there is any 2082 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2083 */ 2084 spin_lock(&clp->cl_lock); 2085 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2086 spin_unlock(&clp->cl_lock); 2087 } 2088 } 2089 2090 /* 2091 * Layout segment is retreived from the server if not cached. 2092 * The appropriate layout segment is referenced and returned to the caller. 2093 */ 2094 struct pnfs_layout_segment * 2095 pnfs_update_layout(struct inode *ino, 2096 struct nfs_open_context *ctx, 2097 loff_t pos, 2098 u64 count, 2099 enum pnfs_iomode iomode, 2100 bool strict_iomode, 2101 gfp_t gfp_flags) 2102 { 2103 struct pnfs_layout_range arg = { 2104 .iomode = iomode, 2105 .offset = pos, 2106 .length = count, 2107 }; 2108 unsigned pg_offset; 2109 struct nfs_server *server = NFS_SERVER(ino); 2110 struct nfs_client *clp = server->nfs_client; 2111 struct pnfs_layout_hdr *lo = NULL; 2112 struct pnfs_layout_segment *lseg = NULL; 2113 struct nfs4_layoutget *lgp; 2114 nfs4_stateid stateid; 2115 struct nfs4_exception exception = { 2116 .inode = ino, 2117 }; 2118 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2119 bool first; 2120 2121 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2122 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2123 PNFS_UPDATE_LAYOUT_NO_PNFS); 2124 goto out; 2125 } 2126 2127 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2128 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2129 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2130 goto out; 2131 } 2132 2133 lookup_again: 2134 if (!nfs4_valid_open_stateid(ctx->state)) { 2135 trace_pnfs_update_layout(ino, pos, count, 2136 iomode, lo, lseg, 2137 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2138 lseg = ERR_PTR(-EIO); 2139 goto out; 2140 } 2141 2142 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2143 if (IS_ERR(lseg)) 2144 goto out; 2145 first = false; 2146 spin_lock(&ino->i_lock); 2147 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2148 if (lo == NULL) { 2149 spin_unlock(&ino->i_lock); 2150 lseg = ERR_PTR(-ENOMEM); 2151 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2152 PNFS_UPDATE_LAYOUT_NOMEM); 2153 goto out; 2154 } 2155 2156 /* Do we even need to bother with this? */ 2157 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2158 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2159 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2160 dprintk("%s matches recall, use MDS\n", __func__); 2161 goto out_unlock; 2162 } 2163 2164 /* if LAYOUTGET already failed once we don't try again */ 2165 if (pnfs_layout_io_test_failed(lo, iomode)) { 2166 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2167 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2168 goto out_unlock; 2169 } 2170 2171 /* 2172 * If the layout segment list is empty, but there are outstanding 2173 * layoutget calls, then they might be subject to a layoutrecall. 2174 */ 2175 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2176 atomic_read(&lo->plh_outstanding) != 0) { 2177 spin_unlock(&ino->i_lock); 2178 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2179 TASK_KILLABLE)); 2180 if (IS_ERR(lseg)) 2181 goto out_put_layout_hdr; 2182 pnfs_put_layout_hdr(lo); 2183 goto lookup_again; 2184 } 2185 2186 /* 2187 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2188 * for LAYOUTRETURN. 2189 */ 2190 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2191 spin_unlock(&ino->i_lock); 2192 dprintk("%s wait for layoutreturn\n", __func__); 2193 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2194 if (!IS_ERR(lseg)) { 2195 pnfs_put_layout_hdr(lo); 2196 dprintk("%s retrying\n", __func__); 2197 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2198 lseg, 2199 PNFS_UPDATE_LAYOUT_RETRY); 2200 goto lookup_again; 2201 } 2202 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2203 PNFS_UPDATE_LAYOUT_RETURN); 2204 goto out_put_layout_hdr; 2205 } 2206 2207 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2208 if (lseg) { 2209 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2210 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2211 goto out_unlock; 2212 } 2213 2214 /* 2215 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2216 * stateid, or it has been invalidated, then we must use the open 2217 * stateid. 2218 */ 2219 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2220 int status; 2221 2222 /* 2223 * The first layoutget for the file. Need to serialize per 2224 * RFC 5661 Errata 3208. 2225 */ 2226 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2227 &lo->plh_flags)) { 2228 spin_unlock(&ino->i_lock); 2229 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2230 NFS_LAYOUT_FIRST_LAYOUTGET, 2231 TASK_KILLABLE)); 2232 if (IS_ERR(lseg)) 2233 goto out_put_layout_hdr; 2234 pnfs_put_layout_hdr(lo); 2235 dprintk("%s retrying\n", __func__); 2236 goto lookup_again; 2237 } 2238 2239 spin_unlock(&ino->i_lock); 2240 first = true; 2241 status = nfs4_select_rw_stateid(ctx->state, 2242 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2243 NULL, &stateid, NULL); 2244 if (status != 0) { 2245 lseg = ERR_PTR(status); 2246 trace_pnfs_update_layout(ino, pos, count, 2247 iomode, lo, lseg, 2248 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2249 nfs4_schedule_stateid_recovery(server, ctx->state); 2250 pnfs_clear_first_layoutget(lo); 2251 pnfs_put_layout_hdr(lo); 2252 goto lookup_again; 2253 } 2254 spin_lock(&ino->i_lock); 2255 } else { 2256 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2257 } 2258 2259 if (pnfs_layoutgets_blocked(lo)) { 2260 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2261 PNFS_UPDATE_LAYOUT_BLOCKED); 2262 goto out_unlock; 2263 } 2264 nfs_layoutget_begin(lo); 2265 spin_unlock(&ino->i_lock); 2266 2267 _add_to_server_list(lo, server); 2268 2269 pg_offset = arg.offset & ~PAGE_MASK; 2270 if (pg_offset) { 2271 arg.offset -= pg_offset; 2272 arg.length += pg_offset; 2273 } 2274 if (arg.length != NFS4_MAX_UINT64) 2275 arg.length = PAGE_ALIGN(arg.length); 2276 2277 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); 2278 if (!lgp) { 2279 lseg = ERR_PTR(-ENOMEM); 2280 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2281 PNFS_UPDATE_LAYOUT_NOMEM); 2282 nfs_layoutget_end(lo); 2283 goto out_put_layout_hdr; 2284 } 2285 2286 lgp->lo = lo; 2287 pnfs_get_layout_hdr(lo); 2288 2289 lseg = nfs4_proc_layoutget(lgp, &exception); 2290 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2291 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2292 nfs_layoutget_end(lo); 2293 if (IS_ERR(lseg)) { 2294 switch(PTR_ERR(lseg)) { 2295 case -EBUSY: 2296 if (time_after(jiffies, giveup)) 2297 lseg = NULL; 2298 break; 2299 case -ERECALLCONFLICT: 2300 case -EAGAIN: 2301 break; 2302 case -ENODATA: 2303 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2304 pnfs_layout_set_fail_bit( 2305 lo, pnfs_iomode_to_fail_bit(iomode)); 2306 lseg = NULL; 2307 goto out_put_layout_hdr; 2308 default: 2309 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2310 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2311 lseg = NULL; 2312 } 2313 goto out_put_layout_hdr; 2314 } 2315 if (lseg) { 2316 if (!exception.retry) 2317 goto out_put_layout_hdr; 2318 if (first) 2319 pnfs_clear_first_layoutget(lo); 2320 trace_pnfs_update_layout(ino, pos, count, 2321 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2322 pnfs_put_layout_hdr(lo); 2323 goto lookup_again; 2324 } 2325 } else { 2326 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2327 } 2328 2329 out_put_layout_hdr: 2330 if (first) 2331 pnfs_clear_first_layoutget(lo); 2332 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2333 PNFS_UPDATE_LAYOUT_EXIT); 2334 pnfs_put_layout_hdr(lo); 2335 out: 2336 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2337 "(%s, offset: %llu, length: %llu)\n", 2338 __func__, ino->i_sb->s_id, 2339 (unsigned long long)NFS_FILEID(ino), 2340 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2341 iomode==IOMODE_RW ? "read/write" : "read-only", 2342 (unsigned long long)pos, 2343 (unsigned long long)count); 2344 return lseg; 2345 out_unlock: 2346 spin_unlock(&ino->i_lock); 2347 goto out_put_layout_hdr; 2348 } 2349 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2350 2351 static bool 2352 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2353 { 2354 switch (range->iomode) { 2355 case IOMODE_READ: 2356 case IOMODE_RW: 2357 break; 2358 default: 2359 return false; 2360 } 2361 if (range->offset == NFS4_MAX_UINT64) 2362 return false; 2363 if (range->length == 0) 2364 return false; 2365 if (range->length != NFS4_MAX_UINT64 && 2366 range->length > NFS4_MAX_UINT64 - range->offset) 2367 return false; 2368 return true; 2369 } 2370 2371 static struct pnfs_layout_hdr * 2372 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2373 { 2374 struct pnfs_layout_hdr *lo; 2375 2376 spin_lock(&ino->i_lock); 2377 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2378 if (!lo) 2379 goto out_unlock; 2380 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2381 goto out_unlock; 2382 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2383 goto out_unlock; 2384 if (pnfs_layoutgets_blocked(lo)) 2385 goto out_unlock; 2386 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2387 goto out_unlock; 2388 nfs_layoutget_begin(lo); 2389 spin_unlock(&ino->i_lock); 2390 _add_to_server_list(lo, NFS_SERVER(ino)); 2391 return lo; 2392 2393 out_unlock: 2394 spin_unlock(&ino->i_lock); 2395 pnfs_put_layout_hdr(lo); 2396 return NULL; 2397 } 2398 2399 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2400 struct nfs_open_context *ctx) 2401 { 2402 struct inode *ino = data->dentry->d_inode; 2403 struct pnfs_layout_range rng = { 2404 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2405 IOMODE_RW: IOMODE_READ, 2406 .offset = 0, 2407 .length = NFS4_MAX_UINT64, 2408 }; 2409 struct nfs4_layoutget *lgp; 2410 struct pnfs_layout_hdr *lo; 2411 2412 /* Heuristic: don't send layoutget if we have cached data */ 2413 if (rng.iomode == IOMODE_READ && 2414 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2415 return; 2416 2417 lo = _pnfs_grab_empty_layout(ino, ctx); 2418 if (!lo) 2419 return; 2420 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2421 nfs_io_gfp_mask()); 2422 if (!lgp) { 2423 pnfs_clear_first_layoutget(lo); 2424 nfs_layoutget_end(lo); 2425 pnfs_put_layout_hdr(lo); 2426 return; 2427 } 2428 lgp->lo = lo; 2429 data->lgp = lgp; 2430 data->o_arg.lg_args = &lgp->args; 2431 data->o_res.lg_res = &lgp->res; 2432 } 2433 2434 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2435 struct nfs_open_context *ctx) 2436 { 2437 struct inode *ino = data->dentry->d_inode; 2438 struct pnfs_layout_range rng = { 2439 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2440 IOMODE_RW: IOMODE_READ, 2441 .offset = 0, 2442 .length = NFS4_MAX_UINT64, 2443 }; 2444 struct nfs4_layoutget *lgp; 2445 2446 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2447 nfs_io_gfp_mask()); 2448 if (!lgp) 2449 return; 2450 data->lgp = lgp; 2451 data->o_arg.lg_args = &lgp->args; 2452 data->o_res.lg_res = &lgp->res; 2453 } 2454 2455 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2456 struct nfs_open_context *ctx) 2457 { 2458 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2459 2460 if (!(pnfs_enabled_sb(server) && 2461 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2462 return; 2463 /* Could check on max_ops, but currently hardcoded high enough */ 2464 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2465 return; 2466 if (data->lgp) 2467 return; 2468 if (data->state) 2469 _lgopen_prepare_attached(data, ctx); 2470 else 2471 _lgopen_prepare_floating(data, ctx); 2472 } 2473 2474 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2475 struct nfs_open_context *ctx) 2476 { 2477 struct pnfs_layout_hdr *lo; 2478 struct pnfs_layout_segment *lseg; 2479 struct nfs_server *srv = NFS_SERVER(ino); 2480 u32 iomode; 2481 2482 if (!lgp) 2483 return; 2484 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2485 if (lgp->res.status) { 2486 switch (lgp->res.status) { 2487 default: 2488 break; 2489 /* 2490 * Halt lgopen attempts if the server doesn't recognise 2491 * the "current stateid" value, the layout type, or the 2492 * layoutget operation as being valid. 2493 * Also if it complains about too many ops in the compound 2494 * or of the request/reply being too big. 2495 */ 2496 case -NFS4ERR_BAD_STATEID: 2497 case -NFS4ERR_NOTSUPP: 2498 case -NFS4ERR_REP_TOO_BIG: 2499 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2500 case -NFS4ERR_REQ_TOO_BIG: 2501 case -NFS4ERR_TOO_MANY_OPS: 2502 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2503 srv->caps &= ~NFS_CAP_LGOPEN; 2504 } 2505 return; 2506 } 2507 if (!lgp->lo) { 2508 lo = _pnfs_grab_empty_layout(ino, ctx); 2509 if (!lo) 2510 return; 2511 lgp->lo = lo; 2512 } else 2513 lo = lgp->lo; 2514 2515 lseg = pnfs_layout_process(lgp); 2516 if (!IS_ERR(lseg)) { 2517 iomode = lgp->args.range.iomode; 2518 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2519 pnfs_put_lseg(lseg); 2520 } 2521 } 2522 2523 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2524 { 2525 if (lgp != NULL) { 2526 if (lgp->lo) { 2527 pnfs_clear_first_layoutget(lgp->lo); 2528 nfs_layoutget_end(lgp->lo); 2529 } 2530 pnfs_layoutget_free(lgp); 2531 } 2532 } 2533 2534 struct pnfs_layout_segment * 2535 pnfs_layout_process(struct nfs4_layoutget *lgp) 2536 { 2537 struct pnfs_layout_hdr *lo = lgp->lo; 2538 struct nfs4_layoutget_res *res = &lgp->res; 2539 struct pnfs_layout_segment *lseg; 2540 struct inode *ino = lo->plh_inode; 2541 LIST_HEAD(free_me); 2542 2543 if (!pnfs_sanity_check_layout_range(&res->range)) 2544 return ERR_PTR(-EINVAL); 2545 2546 /* Inject layout blob into I/O device driver */ 2547 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2548 if (IS_ERR_OR_NULL(lseg)) { 2549 if (!lseg) 2550 lseg = ERR_PTR(-ENOMEM); 2551 2552 dprintk("%s: Could not allocate layout: error %ld\n", 2553 __func__, PTR_ERR(lseg)); 2554 return lseg; 2555 } 2556 2557 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2558 2559 spin_lock(&ino->i_lock); 2560 if (pnfs_layoutgets_blocked(lo)) { 2561 dprintk("%s forget reply due to state\n", __func__); 2562 goto out_forget; 2563 } 2564 2565 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2566 !pnfs_is_first_layoutget(lo)) 2567 goto out_forget; 2568 2569 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2570 /* existing state ID, make sure the sequence number matches. */ 2571 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2572 if (!pnfs_layout_is_valid(lo)) 2573 lo->plh_barrier = 0; 2574 dprintk("%s forget reply due to sequence\n", __func__); 2575 goto out_forget; 2576 } 2577 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2578 } else if (pnfs_layout_is_valid(lo)) { 2579 /* 2580 * We got an entirely new state ID. Mark all segments for the 2581 * inode invalid, and retry the layoutget 2582 */ 2583 struct pnfs_layout_range range = { 2584 .iomode = IOMODE_ANY, 2585 .length = NFS4_MAX_UINT64, 2586 }; 2587 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); 2588 goto out_forget; 2589 } else { 2590 /* We have a completely new layout */ 2591 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2592 } 2593 2594 pnfs_get_lseg(lseg); 2595 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2596 2597 2598 if (res->return_on_close) 2599 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2600 2601 spin_unlock(&ino->i_lock); 2602 pnfs_free_lseg_list(&free_me); 2603 return lseg; 2604 2605 out_forget: 2606 spin_unlock(&ino->i_lock); 2607 lseg->pls_layout = lo; 2608 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2609 return ERR_PTR(-EAGAIN); 2610 } 2611 2612 /** 2613 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2614 * @lo: pointer to layout header 2615 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2616 * @return_range: describe layout segment ranges to be returned 2617 * @seq: stateid seqid to match 2618 * 2619 * This function is mainly intended for use by layoutrecall. It attempts 2620 * to free the layout segment immediately, or else to mark it for return 2621 * as soon as its reference count drops to zero. 2622 * 2623 * Returns 2624 * - 0: a layoutreturn needs to be scheduled. 2625 * - EBUSY: there are layout segment that are still in use. 2626 * - ENOENT: there are no layout segments that need to be returned. 2627 */ 2628 int 2629 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2630 struct list_head *tmp_list, 2631 const struct pnfs_layout_range *return_range, 2632 u32 seq) 2633 { 2634 struct pnfs_layout_segment *lseg, *next; 2635 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2636 int remaining = 0; 2637 2638 dprintk("%s:Begin lo %p\n", __func__, lo); 2639 2640 assert_spin_locked(&lo->plh_inode->i_lock); 2641 2642 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2643 tmp_list = &lo->plh_return_segs; 2644 2645 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2646 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2647 dprintk("%s: marking lseg %p iomode %d " 2648 "offset %llu length %llu\n", __func__, 2649 lseg, lseg->pls_range.iomode, 2650 lseg->pls_range.offset, 2651 lseg->pls_range.length); 2652 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2653 tmp_list = &lo->plh_return_segs; 2654 if (mark_lseg_invalid(lseg, tmp_list)) 2655 continue; 2656 remaining++; 2657 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2658 pnfs_lseg_cancel_io(server, lseg); 2659 } 2660 2661 if (remaining) { 2662 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2663 return -EBUSY; 2664 } 2665 2666 if (!list_empty(&lo->plh_return_segs)) { 2667 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2668 return 0; 2669 } 2670 2671 return -ENOENT; 2672 } 2673 2674 static void 2675 pnfs_mark_layout_for_return(struct inode *inode, 2676 const struct pnfs_layout_range *range) 2677 { 2678 struct pnfs_layout_hdr *lo; 2679 bool return_now = false; 2680 2681 spin_lock(&inode->i_lock); 2682 lo = NFS_I(inode)->layout; 2683 if (!pnfs_layout_is_valid(lo)) { 2684 spin_unlock(&inode->i_lock); 2685 return; 2686 } 2687 pnfs_set_plh_return_info(lo, range->iomode, 0); 2688 /* 2689 * mark all matching lsegs so that we are sure to have no live 2690 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2691 * for how it works. 2692 */ 2693 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { 2694 const struct cred *cred; 2695 nfs4_stateid stateid; 2696 enum pnfs_iomode iomode; 2697 2698 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2699 spin_unlock(&inode->i_lock); 2700 if (return_now) 2701 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2702 PNFS_FL_LAYOUTRETURN_ASYNC); 2703 } else { 2704 spin_unlock(&inode->i_lock); 2705 nfs_commit_inode(inode, 0); 2706 } 2707 } 2708 2709 void pnfs_error_mark_layout_for_return(struct inode *inode, 2710 struct pnfs_layout_segment *lseg) 2711 { 2712 struct pnfs_layout_range range = { 2713 .iomode = lseg->pls_range.iomode, 2714 .offset = 0, 2715 .length = NFS4_MAX_UINT64, 2716 }; 2717 2718 pnfs_mark_layout_for_return(inode, &range); 2719 } 2720 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2721 2722 static bool 2723 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2724 { 2725 return pnfs_layout_is_valid(lo) && 2726 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2727 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2728 } 2729 2730 static struct pnfs_layout_segment * 2731 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2732 const struct pnfs_layout_range *range, 2733 enum pnfs_iomode iomode) 2734 { 2735 struct pnfs_layout_segment *lseg; 2736 2737 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2738 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2739 continue; 2740 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2741 continue; 2742 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2743 continue; 2744 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2745 return lseg; 2746 } 2747 return NULL; 2748 } 2749 2750 /* Find open file states whose mode matches that of the range */ 2751 static bool 2752 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2753 const struct pnfs_layout_range *range) 2754 { 2755 struct list_head *head; 2756 struct nfs_open_context *ctx; 2757 fmode_t mode = 0; 2758 2759 if (!pnfs_layout_can_be_returned(lo) || 2760 !pnfs_find_first_lseg(lo, range, range->iomode)) 2761 return false; 2762 2763 head = &NFS_I(lo->plh_inode)->open_files; 2764 list_for_each_entry_rcu(ctx, head, list) { 2765 if (ctx->state) 2766 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2767 } 2768 2769 switch (range->iomode) { 2770 default: 2771 break; 2772 case IOMODE_READ: 2773 mode &= ~FMODE_WRITE; 2774 break; 2775 case IOMODE_RW: 2776 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2777 mode &= ~FMODE_READ; 2778 } 2779 return mode == 0; 2780 } 2781 2782 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2783 void *data) 2784 { 2785 const struct pnfs_layout_range *range = data; 2786 const struct cred *cred; 2787 struct pnfs_layout_hdr *lo; 2788 struct inode *inode; 2789 nfs4_stateid stateid; 2790 enum pnfs_iomode iomode; 2791 2792 restart: 2793 rcu_read_lock(); 2794 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2795 inode = lo->plh_inode; 2796 if (!inode || !pnfs_layout_can_be_returned(lo) || 2797 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2798 continue; 2799 spin_lock(&inode->i_lock); 2800 if (!lo->plh_inode || 2801 !pnfs_should_return_unused_layout(lo, range)) { 2802 spin_unlock(&inode->i_lock); 2803 continue; 2804 } 2805 pnfs_get_layout_hdr(lo); 2806 pnfs_set_plh_return_info(lo, range->iomode, 0); 2807 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2808 range, 0) != 0 || 2809 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2810 spin_unlock(&inode->i_lock); 2811 rcu_read_unlock(); 2812 pnfs_put_layout_hdr(lo); 2813 cond_resched(); 2814 goto restart; 2815 } 2816 spin_unlock(&inode->i_lock); 2817 rcu_read_unlock(); 2818 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2819 PNFS_FL_LAYOUTRETURN_ASYNC); 2820 pnfs_put_layout_hdr(lo); 2821 cond_resched(); 2822 goto restart; 2823 } 2824 rcu_read_unlock(); 2825 return 0; 2826 } 2827 2828 void 2829 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2830 enum pnfs_iomode iomode) 2831 { 2832 struct pnfs_layout_range range = { 2833 .iomode = iomode, 2834 .offset = 0, 2835 .length = NFS4_MAX_UINT64, 2836 }; 2837 2838 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2839 &range); 2840 } 2841 2842 /* Check if we have we have a valid layout but if there isn't an intersection 2843 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 2844 */ 2845 void 2846 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 2847 struct nfs_page *req) 2848 { 2849 if (pgio->pg_lseg == NULL || 2850 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 2851 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 2852 return; 2853 pnfs_put_lseg(pgio->pg_lseg); 2854 pgio->pg_lseg = NULL; 2855 } 2856 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2857 2858 void 2859 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2860 { 2861 u64 rd_size; 2862 2863 pnfs_generic_pg_check_layout(pgio, req); 2864 if (pgio->pg_lseg == NULL) { 2865 if (pgio->pg_dreq == NULL) 2866 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2867 else 2868 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 2869 req_offset(req)); 2870 2871 pgio->pg_lseg = 2872 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2873 req_offset(req), rd_size, 2874 IOMODE_READ, false, 2875 nfs_io_gfp_mask()); 2876 if (IS_ERR(pgio->pg_lseg)) { 2877 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2878 pgio->pg_lseg = NULL; 2879 return; 2880 } 2881 } 2882 /* If no lseg, fall back to read through mds */ 2883 if (pgio->pg_lseg == NULL) 2884 nfs_pageio_reset_read_mds(pgio); 2885 2886 } 2887 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2888 2889 void 2890 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2891 struct nfs_page *req, u64 wb_size) 2892 { 2893 pnfs_generic_pg_check_layout(pgio, req); 2894 if (pgio->pg_lseg == NULL) { 2895 pgio->pg_lseg = 2896 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2897 req_offset(req), wb_size, IOMODE_RW, 2898 false, nfs_io_gfp_mask()); 2899 if (IS_ERR(pgio->pg_lseg)) { 2900 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2901 pgio->pg_lseg = NULL; 2902 return; 2903 } 2904 } 2905 /* If no lseg, fall back to write through mds */ 2906 if (pgio->pg_lseg == NULL) 2907 nfs_pageio_reset_write_mds(pgio); 2908 } 2909 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2910 2911 void 2912 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2913 { 2914 if (desc->pg_lseg) { 2915 pnfs_put_lseg(desc->pg_lseg); 2916 desc->pg_lseg = NULL; 2917 } 2918 } 2919 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2920 2921 /* 2922 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2923 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2924 */ 2925 size_t 2926 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2927 struct nfs_page *prev, struct nfs_page *req) 2928 { 2929 unsigned int size; 2930 u64 seg_end, req_start, seg_left; 2931 2932 size = nfs_generic_pg_test(pgio, prev, req); 2933 if (!size) 2934 return 0; 2935 2936 /* 2937 * 'size' contains the number of bytes left in the current page (up 2938 * to the original size asked for in @req->wb_bytes). 2939 * 2940 * Calculate how many bytes are left in the layout segment 2941 * and if there are less bytes than 'size', return that instead. 2942 * 2943 * Please also note that 'end_offset' is actually the offset of the 2944 * first byte that lies outside the pnfs_layout_range. FIXME? 2945 * 2946 */ 2947 if (pgio->pg_lseg) { 2948 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2949 pgio->pg_lseg->pls_range.length); 2950 req_start = req_offset(req); 2951 2952 /* start of request is past the last byte of this segment */ 2953 if (req_start >= seg_end) 2954 return 0; 2955 2956 /* adjust 'size' iff there are fewer bytes left in the 2957 * segment than what nfs_generic_pg_test returned */ 2958 seg_left = seg_end - req_start; 2959 if (seg_left < size) 2960 size = (unsigned int)seg_left; 2961 } 2962 2963 return size; 2964 } 2965 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2966 2967 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2968 { 2969 struct nfs_pageio_descriptor pgio; 2970 2971 /* Resend all requests through the MDS */ 2972 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2973 hdr->completion_ops); 2974 return nfs_pageio_resend(&pgio, hdr); 2975 } 2976 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 2977 2978 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 2979 { 2980 2981 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 2982 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2983 PNFS_LAYOUTRET_ON_ERROR) { 2984 pnfs_return_layout(hdr->inode); 2985 } 2986 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2987 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 2988 } 2989 2990 /* 2991 * Called by non rpc-based layout drivers 2992 */ 2993 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 2994 { 2995 if (likely(!hdr->pnfs_error)) { 2996 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 2997 hdr->mds_offset + hdr->res.count); 2998 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2999 } 3000 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3001 if (unlikely(hdr->pnfs_error)) 3002 pnfs_ld_handle_write_error(hdr); 3003 hdr->mds_ops->rpc_release(hdr); 3004 } 3005 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3006 3007 static void 3008 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3009 struct nfs_pgio_header *hdr) 3010 { 3011 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3012 3013 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3014 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3015 nfs_pageio_reset_write_mds(desc); 3016 mirror->pg_recoalesce = 1; 3017 } 3018 hdr->completion_ops->completion(hdr); 3019 } 3020 3021 static enum pnfs_try_status 3022 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3023 const struct rpc_call_ops *call_ops, 3024 struct pnfs_layout_segment *lseg, 3025 int how) 3026 { 3027 struct inode *inode = hdr->inode; 3028 enum pnfs_try_status trypnfs; 3029 struct nfs_server *nfss = NFS_SERVER(inode); 3030 3031 hdr->mds_ops = call_ops; 3032 3033 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 3034 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3035 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3036 if (trypnfs != PNFS_NOT_ATTEMPTED) 3037 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3038 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3039 return trypnfs; 3040 } 3041 3042 static void 3043 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3044 struct nfs_pgio_header *hdr, int how) 3045 { 3046 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3047 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3048 enum pnfs_try_status trypnfs; 3049 3050 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3051 switch (trypnfs) { 3052 case PNFS_NOT_ATTEMPTED: 3053 pnfs_write_through_mds(desc, hdr); 3054 break; 3055 case PNFS_ATTEMPTED: 3056 break; 3057 case PNFS_TRY_AGAIN: 3058 /* cleanup hdr and prepare to redo pnfs */ 3059 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3060 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3061 list_splice_init(&hdr->pages, &mirror->pg_list); 3062 mirror->pg_recoalesce = 1; 3063 } 3064 hdr->mds_ops->rpc_release(hdr); 3065 } 3066 } 3067 3068 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3069 { 3070 pnfs_put_lseg(hdr->lseg); 3071 nfs_pgio_header_free(hdr); 3072 } 3073 3074 int 3075 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3076 { 3077 struct nfs_pgio_header *hdr; 3078 int ret; 3079 3080 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3081 if (!hdr) { 3082 desc->pg_error = -ENOMEM; 3083 return desc->pg_error; 3084 } 3085 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3086 3087 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3088 ret = nfs_generic_pgio(desc, hdr); 3089 if (!ret) 3090 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3091 3092 return ret; 3093 } 3094 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3095 3096 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3097 { 3098 struct nfs_pageio_descriptor pgio; 3099 3100 /* Resend all requests through the MDS */ 3101 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3102 return nfs_pageio_resend(&pgio, hdr); 3103 } 3104 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3105 3106 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3107 { 3108 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3109 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3110 PNFS_LAYOUTRET_ON_ERROR) { 3111 pnfs_return_layout(hdr->inode); 3112 } 3113 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3114 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3115 } 3116 3117 /* 3118 * Called by non rpc-based layout drivers 3119 */ 3120 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3121 { 3122 if (likely(!hdr->pnfs_error)) 3123 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3124 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3125 if (unlikely(hdr->pnfs_error)) 3126 pnfs_ld_handle_read_error(hdr); 3127 hdr->mds_ops->rpc_release(hdr); 3128 } 3129 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3130 3131 static void 3132 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3133 struct nfs_pgio_header *hdr) 3134 { 3135 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3136 3137 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3138 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3139 nfs_pageio_reset_read_mds(desc); 3140 mirror->pg_recoalesce = 1; 3141 } 3142 hdr->completion_ops->completion(hdr); 3143 } 3144 3145 /* 3146 * Call the appropriate parallel I/O subsystem read function. 3147 */ 3148 static enum pnfs_try_status 3149 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3150 const struct rpc_call_ops *call_ops, 3151 struct pnfs_layout_segment *lseg) 3152 { 3153 struct inode *inode = hdr->inode; 3154 struct nfs_server *nfss = NFS_SERVER(inode); 3155 enum pnfs_try_status trypnfs; 3156 3157 hdr->mds_ops = call_ops; 3158 3159 dprintk("%s: Reading ino:%lu %u@%llu\n", 3160 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3161 3162 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3163 if (trypnfs != PNFS_NOT_ATTEMPTED) 3164 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3165 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3166 return trypnfs; 3167 } 3168 3169 /* Resend all requests through pnfs. */ 3170 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3171 unsigned int mirror_idx) 3172 { 3173 struct nfs_pageio_descriptor pgio; 3174 3175 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3176 /* Prevent deadlocks with layoutreturn! */ 3177 pnfs_put_lseg(hdr->lseg); 3178 hdr->lseg = NULL; 3179 3180 nfs_pageio_init_read(&pgio, hdr->inode, false, 3181 hdr->completion_ops); 3182 pgio.pg_mirror_idx = mirror_idx; 3183 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3184 } 3185 } 3186 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3187 3188 static void 3189 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3190 { 3191 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3192 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3193 enum pnfs_try_status trypnfs; 3194 3195 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3196 switch (trypnfs) { 3197 case PNFS_NOT_ATTEMPTED: 3198 pnfs_read_through_mds(desc, hdr); 3199 break; 3200 case PNFS_ATTEMPTED: 3201 break; 3202 case PNFS_TRY_AGAIN: 3203 /* cleanup hdr and prepare to redo pnfs */ 3204 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3205 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3206 list_splice_init(&hdr->pages, &mirror->pg_list); 3207 mirror->pg_recoalesce = 1; 3208 } 3209 hdr->mds_ops->rpc_release(hdr); 3210 } 3211 } 3212 3213 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3214 { 3215 pnfs_put_lseg(hdr->lseg); 3216 nfs_pgio_header_free(hdr); 3217 } 3218 3219 int 3220 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3221 { 3222 struct nfs_pgio_header *hdr; 3223 int ret; 3224 3225 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3226 if (!hdr) { 3227 desc->pg_error = -ENOMEM; 3228 return desc->pg_error; 3229 } 3230 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3231 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3232 ret = nfs_generic_pgio(desc, hdr); 3233 if (!ret) 3234 pnfs_do_read(desc, hdr); 3235 return ret; 3236 } 3237 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3238 3239 static void pnfs_clear_layoutcommitting(struct inode *inode) 3240 { 3241 unsigned long *bitlock = &NFS_I(inode)->flags; 3242 3243 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 3244 smp_mb__after_atomic(); 3245 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 3246 } 3247 3248 /* 3249 * There can be multiple RW segments. 3250 */ 3251 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3252 { 3253 struct pnfs_layout_segment *lseg; 3254 3255 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3256 if (lseg->pls_range.iomode == IOMODE_RW && 3257 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3258 list_add(&lseg->pls_lc_list, listp); 3259 } 3260 } 3261 3262 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3263 { 3264 struct pnfs_layout_segment *lseg, *tmp; 3265 3266 /* Matched by references in pnfs_set_layoutcommit */ 3267 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3268 list_del_init(&lseg->pls_lc_list); 3269 pnfs_put_lseg(lseg); 3270 } 3271 3272 pnfs_clear_layoutcommitting(inode); 3273 } 3274 3275 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3276 { 3277 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3278 } 3279 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3280 3281 void 3282 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3283 loff_t end_pos) 3284 { 3285 struct nfs_inode *nfsi = NFS_I(inode); 3286 bool mark_as_dirty = false; 3287 3288 spin_lock(&inode->i_lock); 3289 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3290 nfsi->layout->plh_lwb = end_pos; 3291 mark_as_dirty = true; 3292 dprintk("%s: Set layoutcommit for inode %lu ", 3293 __func__, inode->i_ino); 3294 } else if (end_pos > nfsi->layout->plh_lwb) 3295 nfsi->layout->plh_lwb = end_pos; 3296 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3297 /* references matched in nfs4_layoutcommit_release */ 3298 pnfs_get_lseg(lseg); 3299 } 3300 spin_unlock(&inode->i_lock); 3301 dprintk("%s: lseg %p end_pos %llu\n", 3302 __func__, lseg, nfsi->layout->plh_lwb); 3303 3304 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3305 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3306 if (mark_as_dirty) 3307 mark_inode_dirty_sync(inode); 3308 } 3309 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3310 3311 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3312 { 3313 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3314 3315 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3316 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3317 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3318 } 3319 3320 /* 3321 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3322 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3323 * data to disk to allow the server to recover the data if it crashes. 3324 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3325 * is off, and a COMMIT is sent to a data server, or 3326 * if WRITEs to a data server return NFS_DATA_SYNC. 3327 */ 3328 int 3329 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3330 { 3331 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3332 struct nfs4_layoutcommit_data *data; 3333 struct nfs_inode *nfsi = NFS_I(inode); 3334 loff_t end_pos; 3335 int status; 3336 bool mark_as_dirty = false; 3337 3338 if (!pnfs_layoutcommit_outstanding(inode)) 3339 return 0; 3340 3341 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 3342 3343 status = -EAGAIN; 3344 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3345 if (!sync) 3346 goto out; 3347 status = wait_on_bit_lock_action(&nfsi->flags, 3348 NFS_INO_LAYOUTCOMMITTING, 3349 nfs_wait_bit_killable, 3350 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3351 if (status) 3352 goto out; 3353 } 3354 3355 status = -ENOMEM; 3356 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3357 data = kzalloc(sizeof(*data), nfs_io_gfp_mask()); 3358 if (!data) 3359 goto clear_layoutcommitting; 3360 3361 status = 0; 3362 spin_lock(&inode->i_lock); 3363 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3364 goto out_unlock; 3365 3366 INIT_LIST_HEAD(&data->lseg_list); 3367 pnfs_list_write_lseg(inode, &data->lseg_list); 3368 3369 end_pos = nfsi->layout->plh_lwb; 3370 3371 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3372 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3373 spin_unlock(&inode->i_lock); 3374 3375 data->args.inode = inode; 3376 nfs_fattr_init(&data->fattr); 3377 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3378 data->res.fattr = &data->fattr; 3379 if (end_pos != 0) 3380 data->args.lastbytewritten = end_pos - 1; 3381 else 3382 data->args.lastbytewritten = U64_MAX; 3383 data->res.server = NFS_SERVER(inode); 3384 3385 if (ld->prepare_layoutcommit) { 3386 status = ld->prepare_layoutcommit(&data->args); 3387 if (status) { 3388 if (status != -ENOSPC) 3389 put_cred(data->cred); 3390 spin_lock(&inode->i_lock); 3391 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3392 if (end_pos > nfsi->layout->plh_lwb) 3393 nfsi->layout->plh_lwb = end_pos; 3394 if (status != -ENOSPC) 3395 goto out_unlock; 3396 spin_unlock(&inode->i_lock); 3397 mark_as_dirty = true; 3398 } 3399 } 3400 3401 3402 status = nfs4_proc_layoutcommit(data, sync); 3403 out: 3404 if (status || mark_as_dirty) 3405 mark_inode_dirty_sync(inode); 3406 dprintk("<-- %s status %d\n", __func__, status); 3407 return status; 3408 out_unlock: 3409 spin_unlock(&inode->i_lock); 3410 kfree(data); 3411 clear_layoutcommitting: 3412 pnfs_clear_layoutcommitting(inode); 3413 goto out; 3414 } 3415 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3416 3417 int 3418 pnfs_generic_sync(struct inode *inode, bool datasync) 3419 { 3420 return pnfs_layoutcommit_inode(inode, true); 3421 } 3422 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3423 3424 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3425 { 3426 struct nfs4_threshold *thp; 3427 3428 thp = kzalloc(sizeof(*thp), nfs_io_gfp_mask()); 3429 if (!thp) { 3430 dprintk("%s mdsthreshold allocation failed\n", __func__); 3431 return NULL; 3432 } 3433 return thp; 3434 } 3435 3436 #if IS_ENABLED(CONFIG_NFS_V4_2) 3437 int 3438 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3439 { 3440 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3441 struct nfs_server *server = NFS_SERVER(inode); 3442 struct nfs_inode *nfsi = NFS_I(inode); 3443 struct nfs42_layoutstat_data *data; 3444 struct pnfs_layout_hdr *hdr; 3445 int status = 0; 3446 3447 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3448 goto out; 3449 3450 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3451 goto out; 3452 3453 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3454 goto out; 3455 3456 spin_lock(&inode->i_lock); 3457 if (!NFS_I(inode)->layout) { 3458 spin_unlock(&inode->i_lock); 3459 goto out_clear_layoutstats; 3460 } 3461 hdr = NFS_I(inode)->layout; 3462 pnfs_get_layout_hdr(hdr); 3463 spin_unlock(&inode->i_lock); 3464 3465 data = kzalloc(sizeof(*data), gfp_flags); 3466 if (!data) { 3467 status = -ENOMEM; 3468 goto out_put; 3469 } 3470 3471 data->args.fh = NFS_FH(inode); 3472 data->args.inode = inode; 3473 status = ld->prepare_layoutstats(&data->args); 3474 if (status) 3475 goto out_free; 3476 3477 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3478 3479 out: 3480 dprintk("%s returns %d\n", __func__, status); 3481 return status; 3482 3483 out_free: 3484 kfree(data); 3485 out_put: 3486 pnfs_put_layout_hdr(hdr); 3487 out_clear_layoutstats: 3488 smp_mb__before_atomic(); 3489 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3490 smp_mb__after_atomic(); 3491 goto out; 3492 } 3493 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3494 #endif 3495 3496 unsigned int layoutstats_timer; 3497 module_param(layoutstats_timer, uint, 0644); 3498 EXPORT_SYMBOL_GPL(layoutstats_timer); 3499