1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 436 if (err != -EBUSY) { 437 dst->seqid = lo->plh_stateid.seqid; 438 *dst_range = range; 439 ret = true; 440 } 441 } 442 out: 443 spin_unlock(&inode->i_lock); 444 pnfs_free_lseg_list(&head); 445 return ret; 446 } 447 448 /* 449 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 450 * 451 * In order to continue using the pnfs_layout_hdr, a full recovery 452 * is required. 453 * Note that caller must hold inode->i_lock. 454 */ 455 int 456 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 457 struct list_head *lseg_list) 458 { 459 struct pnfs_layout_range range = { 460 .iomode = IOMODE_ANY, 461 .offset = 0, 462 .length = NFS4_MAX_UINT64, 463 }; 464 struct pnfs_layout_segment *lseg, *next; 465 466 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 467 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); 468 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 469 pnfs_clear_lseg_state(lseg, lseg_list); 470 pnfs_clear_layoutreturn_info(lo); 471 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 472 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 473 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 474 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 475 pnfs_clear_layoutreturn_waitbit(lo); 476 return !list_empty(&lo->plh_segs); 477 } 478 479 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 480 struct list_head *lseg_list, 481 enum pnfs_iomode iomode, u32 seq) 482 { 483 struct pnfs_layout_range range = { 484 .iomode = iomode, 485 .length = NFS4_MAX_UINT64, 486 }; 487 488 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); 489 } 490 491 static int 492 pnfs_iomode_to_fail_bit(u32 iomode) 493 { 494 return iomode == IOMODE_RW ? 495 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 496 } 497 498 static void 499 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 500 { 501 lo->plh_retry_timestamp = jiffies; 502 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 503 refcount_inc(&lo->plh_refcount); 504 } 505 506 static void 507 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 508 { 509 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 510 refcount_dec(&lo->plh_refcount); 511 } 512 513 static void 514 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 515 { 516 struct inode *inode = lo->plh_inode; 517 struct pnfs_layout_range range = { 518 .iomode = iomode, 519 .offset = 0, 520 .length = NFS4_MAX_UINT64, 521 }; 522 LIST_HEAD(head); 523 524 spin_lock(&inode->i_lock); 525 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 526 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 527 spin_unlock(&inode->i_lock); 528 pnfs_free_lseg_list(&head); 529 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 530 iomode == IOMODE_RW ? "RW" : "READ"); 531 } 532 533 static bool 534 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 535 { 536 unsigned long start, end; 537 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 538 539 if (test_bit(fail_bit, &lo->plh_flags) == 0) 540 return false; 541 end = jiffies; 542 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 543 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 544 /* It is time to retry the failed layoutgets */ 545 pnfs_layout_clear_fail_bit(lo, fail_bit); 546 return false; 547 } 548 return true; 549 } 550 551 static void 552 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 553 const struct pnfs_layout_range *range, 554 const nfs4_stateid *stateid) 555 { 556 INIT_LIST_HEAD(&lseg->pls_list); 557 INIT_LIST_HEAD(&lseg->pls_lc_list); 558 INIT_LIST_HEAD(&lseg->pls_commits); 559 refcount_set(&lseg->pls_refcount, 1); 560 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 561 lseg->pls_layout = lo; 562 lseg->pls_range = *range; 563 lseg->pls_seq = be32_to_cpu(stateid->seqid); 564 } 565 566 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 567 { 568 if (lseg != NULL) { 569 struct inode *inode = lseg->pls_layout->plh_inode; 570 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 571 } 572 } 573 574 static void 575 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 576 struct pnfs_layout_segment *lseg) 577 { 578 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 579 list_del_init(&lseg->pls_list); 580 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 581 refcount_dec(&lo->plh_refcount); 582 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 583 return; 584 if (list_empty(&lo->plh_segs) && 585 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 586 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 587 if (atomic_read(&lo->plh_outstanding) == 0) 588 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 589 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 590 } 591 } 592 593 static bool 594 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 595 struct pnfs_layout_segment *lseg) 596 { 597 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 598 pnfs_layout_is_valid(lo)) { 599 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 600 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 601 return true; 602 } 603 return false; 604 } 605 606 void 607 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 608 { 609 struct pnfs_layout_hdr *lo; 610 struct inode *inode; 611 612 if (!lseg) 613 return; 614 615 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 616 refcount_read(&lseg->pls_refcount), 617 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 618 619 lo = lseg->pls_layout; 620 inode = lo->plh_inode; 621 622 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 623 pnfs_get_layout_hdr(lo); 624 pnfs_layout_remove_lseg(lo, lseg); 625 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 626 lseg = NULL; 627 spin_unlock(&inode->i_lock); 628 pnfs_free_lseg(lseg); 629 pnfs_put_layout_hdr(lo); 630 } 631 } 632 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 633 634 /* 635 * is l2 fully contained in l1? 636 * start1 end1 637 * [----------------------------------) 638 * start2 end2 639 * [----------------) 640 */ 641 static bool 642 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 643 const struct pnfs_layout_range *l2) 644 { 645 u64 start1 = l1->offset; 646 u64 end1 = pnfs_end_offset(start1, l1->length); 647 u64 start2 = l2->offset; 648 u64 end2 = pnfs_end_offset(start2, l2->length); 649 650 return (start1 <= start2) && (end1 >= end2); 651 } 652 653 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 654 struct list_head *tmp_list) 655 { 656 if (!refcount_dec_and_test(&lseg->pls_refcount)) 657 return false; 658 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 659 list_add(&lseg->pls_list, tmp_list); 660 return true; 661 } 662 663 /* Returns 1 if lseg is removed from list, 0 otherwise */ 664 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 665 struct list_head *tmp_list) 666 { 667 int rv = 0; 668 669 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 670 /* Remove the reference keeping the lseg in the 671 * list. It will now be removed when all 672 * outstanding io is finished. 673 */ 674 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 675 refcount_read(&lseg->pls_refcount)); 676 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 677 rv = 1; 678 } 679 return rv; 680 } 681 682 static bool 683 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 684 const struct pnfs_layout_range *recall_range) 685 { 686 return (recall_range->iomode == IOMODE_ANY || 687 lseg_range->iomode == recall_range->iomode) && 688 pnfs_lseg_range_intersecting(lseg_range, recall_range); 689 } 690 691 static bool 692 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 693 const struct pnfs_layout_range *recall_range, 694 u32 seq) 695 { 696 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 697 return false; 698 if (recall_range == NULL) 699 return true; 700 return pnfs_should_free_range(&lseg->pls_range, recall_range); 701 } 702 703 /** 704 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 705 * @lo: layout header containing the lsegs 706 * @tmp_list: list head where doomed lsegs should go 707 * @recall_range: optional recall range argument to match (may be NULL) 708 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 709 * 710 * Walk the list of lsegs in the layout header, and tear down any that should 711 * be destroyed. If "recall_range" is specified then the segment must match 712 * that range. If "seq" is non-zero, then only match segments that were handed 713 * out at or before that sequence. 714 * 715 * Returns number of matching invalid lsegs remaining in list after scanning 716 * it and purging them. 717 */ 718 int 719 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 720 struct list_head *tmp_list, 721 const struct pnfs_layout_range *recall_range, 722 u32 seq) 723 { 724 struct pnfs_layout_segment *lseg, *next; 725 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 726 int remaining = 0; 727 728 dprintk("%s:Begin lo %p\n", __func__, lo); 729 730 if (list_empty(&lo->plh_segs)) 731 return 0; 732 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 733 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 734 dprintk("%s: freeing lseg %p iomode %d seq %u " 735 "offset %llu length %llu\n", __func__, 736 lseg, lseg->pls_range.iomode, lseg->pls_seq, 737 lseg->pls_range.offset, lseg->pls_range.length); 738 if (mark_lseg_invalid(lseg, tmp_list)) 739 continue; 740 remaining++; 741 pnfs_lseg_cancel_io(server, lseg); 742 } 743 dprintk("%s:Return %i\n", __func__, remaining); 744 return remaining; 745 } 746 747 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 748 { 749 struct pnfs_layout_segment *lseg; 750 751 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 752 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 753 } 754 755 static void 756 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 757 struct list_head *free_me, 758 const struct pnfs_layout_range *range, 759 u32 seq) 760 { 761 struct pnfs_layout_segment *lseg, *next; 762 763 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 764 if (pnfs_match_lseg_recall(lseg, range, seq)) 765 list_move_tail(&lseg->pls_list, free_me); 766 } 767 } 768 769 /* note free_me must contain lsegs from a single layout_hdr */ 770 void 771 pnfs_free_lseg_list(struct list_head *free_me) 772 { 773 struct pnfs_layout_segment *lseg, *tmp; 774 775 if (list_empty(free_me)) 776 return; 777 778 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 779 list_del(&lseg->pls_list); 780 pnfs_free_lseg(lseg); 781 } 782 } 783 784 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 785 { 786 struct pnfs_layout_hdr *lo; 787 LIST_HEAD(tmp_list); 788 789 spin_lock(&nfsi->vfs_inode.i_lock); 790 lo = nfsi->layout; 791 if (lo) { 792 pnfs_get_layout_hdr(lo); 793 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 794 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 795 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 796 spin_unlock(&nfsi->vfs_inode.i_lock); 797 pnfs_free_lseg_list(&tmp_list); 798 nfs_commit_inode(&nfsi->vfs_inode, 0); 799 pnfs_put_layout_hdr(lo); 800 } else 801 spin_unlock(&nfsi->vfs_inode.i_lock); 802 return lo; 803 } 804 805 void pnfs_destroy_layout(struct nfs_inode *nfsi) 806 { 807 __pnfs_destroy_layout(nfsi); 808 } 809 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 810 811 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 812 { 813 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 814 struct inode *inode = &nfsi->vfs_inode; 815 816 if (lo) { 817 spin_lock(&inode->i_lock); 818 wait_var_event_spinlock(lo, nfsi->layout != lo, 819 &inode->i_lock); 820 spin_unlock(&inode->i_lock); 821 } 822 } 823 824 static bool 825 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 826 struct list_head *layout_list) 827 { 828 struct pnfs_layout_hdr *lo; 829 bool ret = false; 830 831 spin_lock(&inode->i_lock); 832 lo = NFS_I(inode)->layout; 833 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 834 pnfs_get_layout_hdr(lo); 835 list_add(&lo->plh_bulk_destroy, layout_list); 836 ret = true; 837 } 838 spin_unlock(&inode->i_lock); 839 return ret; 840 } 841 842 /* Caller must hold rcu_read_lock and clp->cl_lock */ 843 static int 844 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 845 struct nfs_server *server, 846 struct list_head *layout_list) 847 __must_hold(&clp->cl_lock) 848 __must_hold(RCU) 849 { 850 struct pnfs_layout_hdr *lo, *next; 851 struct inode *inode; 852 853 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 854 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 855 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 856 !list_empty(&lo->plh_bulk_destroy)) 857 continue; 858 /* If the sb is being destroyed, just bail */ 859 if (!nfs_sb_active(server->super)) 860 break; 861 inode = pnfs_grab_inode_layout_hdr(lo); 862 if (inode != NULL) { 863 if (pnfs_layout_add_bulk_destroy_list(inode, 864 layout_list)) 865 continue; 866 rcu_read_unlock(); 867 spin_unlock(&clp->cl_lock); 868 iput(inode); 869 } else { 870 rcu_read_unlock(); 871 spin_unlock(&clp->cl_lock); 872 } 873 nfs_sb_deactive(server->super); 874 spin_lock(&clp->cl_lock); 875 rcu_read_lock(); 876 return -EAGAIN; 877 } 878 return 0; 879 } 880 881 static int 882 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 883 enum pnfs_layout_destroy_mode mode) 884 { 885 struct pnfs_layout_hdr *lo; 886 struct inode *inode; 887 LIST_HEAD(lseg_list); 888 int ret = 0; 889 890 while (!list_empty(layout_list)) { 891 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 892 plh_bulk_destroy); 893 dprintk("%s freeing layout for inode %lu\n", __func__, 894 lo->plh_inode->i_ino); 895 inode = lo->plh_inode; 896 897 pnfs_layoutcommit_inode(inode, false); 898 899 spin_lock(&inode->i_lock); 900 list_del_init(&lo->plh_bulk_destroy); 901 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 902 pnfs_mark_layout_stateid_return(lo, &lseg_list, 903 IOMODE_ANY, 0); 904 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 905 if (mode == PNFS_LAYOUT_BULK_RETURN) 906 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 907 ret = -EAGAIN; 908 } 909 spin_unlock(&inode->i_lock); 910 pnfs_free_lseg_list(&lseg_list); 911 /* Free all lsegs that are attached to commit buckets */ 912 nfs_commit_inode(inode, 0); 913 pnfs_put_layout_hdr(lo); 914 nfs_iput_and_deactive(inode); 915 } 916 return ret; 917 } 918 919 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 920 enum pnfs_layout_destroy_mode mode) 921 { 922 struct nfs_server *server; 923 LIST_HEAD(layout_list); 924 925 spin_lock(&clp->cl_lock); 926 rcu_read_lock(); 927 restart: 928 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 929 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 930 continue; 931 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 932 server, 933 &layout_list) != 0) 934 goto restart; 935 } 936 rcu_read_unlock(); 937 spin_unlock(&clp->cl_lock); 938 939 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 940 } 941 942 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 943 struct list_head *list) 944 { 945 struct nfs_server *server; 946 947 spin_lock(&clp->cl_lock); 948 rcu_read_lock(); 949 restart: 950 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 951 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 952 list) != 0) 953 goto restart; 954 } 955 rcu_read_unlock(); 956 spin_unlock(&clp->cl_lock); 957 } 958 959 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 960 struct list_head *list, 961 enum pnfs_layout_destroy_mode mode) 962 { 963 pnfs_layout_build_destroy_list_byclient(clp, list); 964 return pnfs_layout_free_bulk_destroy_list(list, mode); 965 } 966 967 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 968 enum pnfs_layout_destroy_mode mode) 969 { 970 LIST_HEAD(layout_list); 971 972 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 973 } 974 975 /* 976 * Called by the state manager to remove all layouts established under an 977 * expired lease. 978 */ 979 void 980 pnfs_destroy_all_layouts(struct nfs_client *clp) 981 { 982 nfs4_deviceid_mark_client_invalid(clp); 983 nfs4_deviceid_purge_client(clp); 984 985 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 986 } 987 988 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 989 struct list_head *list) 990 { 991 struct nfs_server *server; 992 993 spin_lock(&clp->cl_lock); 994 rcu_read_lock(); 995 restart: 996 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 997 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 998 continue; 999 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 1000 list) != 0) 1001 goto restart; 1002 } 1003 rcu_read_unlock(); 1004 spin_unlock(&clp->cl_lock); 1005 } 1006 1007 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1008 { 1009 struct pnfs_layout_hdr *lo; 1010 struct nfs_server *server; 1011 int ret; 1012 1013 list_for_each_entry(lo, list, plh_bulk_destroy) { 1014 server = NFS_SERVER(lo->plh_inode); 1015 ret = pnfs_layout_return_on_reboot(lo); 1016 switch (ret) { 1017 case 0: 1018 continue; 1019 case -NFS4ERR_BAD_STATEID: 1020 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1021 break; 1022 case -NFS4ERR_NO_GRACE: 1023 break; 1024 default: 1025 goto err; 1026 } 1027 break; 1028 } 1029 return 0; 1030 err: 1031 return ret; 1032 } 1033 1034 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1035 { 1036 LIST_HEAD(list); 1037 int ret = 0, ret2; 1038 1039 pnfs_layout_build_recover_list_byclient(clp, &list); 1040 if (!list_empty(&list)) 1041 ret = pnfs_layout_bulk_list_reboot(&list); 1042 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1043 PNFS_LAYOUT_INVALIDATE); 1044 if (!ret) 1045 ret = ret2; 1046 return (ret == 0) ? 0 : -EAGAIN; 1047 } 1048 1049 static void 1050 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1051 { 1052 const struct cred *old; 1053 1054 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1055 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1056 put_cred(old); 1057 } 1058 } 1059 1060 /* update lo->plh_stateid with new if is more recent */ 1061 void 1062 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1063 const struct cred *cred, bool update_barrier) 1064 { 1065 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1066 u32 newseq = be32_to_cpu(new->seqid); 1067 1068 if (!pnfs_layout_is_valid(lo)) { 1069 pnfs_set_layout_cred(lo, cred); 1070 nfs4_stateid_copy(&lo->plh_stateid, new); 1071 lo->plh_barrier = newseq; 1072 pnfs_clear_layoutreturn_info(lo); 1073 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1074 return; 1075 } 1076 1077 if (pnfs_seqid_is_newer(newseq, oldseq)) 1078 nfs4_stateid_copy(&lo->plh_stateid, new); 1079 1080 if (update_barrier) { 1081 pnfs_barrier_update(lo, newseq); 1082 return; 1083 } 1084 /* 1085 * Because of wraparound, we want to keep the barrier 1086 * "close" to the current seqids. We really only want to 1087 * get here from a layoutget call. 1088 */ 1089 if (atomic_read(&lo->plh_outstanding) == 1) 1090 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1091 } 1092 1093 static bool 1094 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1095 const nfs4_stateid *stateid) 1096 { 1097 u32 seqid = be32_to_cpu(stateid->seqid); 1098 1099 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1100 } 1101 1102 /* lget is set to 1 if called from inside send_layoutget call chain */ 1103 static bool 1104 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1105 { 1106 return lo->plh_block_lgets || 1107 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1108 } 1109 1110 static struct nfs_server * 1111 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1112 { 1113 struct nfs_server *server; 1114 1115 if (inode) { 1116 server = NFS_SERVER(inode); 1117 } else { 1118 struct dentry *parent_dir = dget_parent(ctx->dentry); 1119 server = NFS_SERVER(parent_dir->d_inode); 1120 dput(parent_dir); 1121 } 1122 return server; 1123 } 1124 1125 static void nfs4_free_pages(struct page **pages, size_t size) 1126 { 1127 int i; 1128 1129 if (!pages) 1130 return; 1131 1132 for (i = 0; i < size; i++) { 1133 if (!pages[i]) 1134 break; 1135 __free_page(pages[i]); 1136 } 1137 kfree(pages); 1138 } 1139 1140 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1141 { 1142 struct page **pages; 1143 int i; 1144 1145 pages = kmalloc_array(size, sizeof(struct page *), gfp_flags); 1146 if (!pages) { 1147 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1148 return NULL; 1149 } 1150 1151 for (i = 0; i < size; i++) { 1152 pages[i] = alloc_page(gfp_flags); 1153 if (!pages[i]) { 1154 dprintk("%s: failed to allocate page\n", __func__); 1155 nfs4_free_pages(pages, i); 1156 return NULL; 1157 } 1158 } 1159 1160 return pages; 1161 } 1162 1163 static struct nfs4_layoutget * 1164 pnfs_alloc_init_layoutget_args(struct inode *ino, 1165 struct nfs_open_context *ctx, 1166 const nfs4_stateid *stateid, 1167 const struct pnfs_layout_range *range, 1168 gfp_t gfp_flags) 1169 { 1170 struct nfs_server *server = pnfs_find_server(ino, ctx); 1171 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1172 size_t max_pages = max_response_pages(server); 1173 struct nfs4_layoutget *lgp; 1174 1175 dprintk("--> %s\n", __func__); 1176 1177 lgp = kzalloc(sizeof(*lgp), gfp_flags); 1178 if (lgp == NULL) 1179 return NULL; 1180 1181 if (max_reply_sz) { 1182 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1183 if (npages < max_pages) 1184 max_pages = npages; 1185 } 1186 1187 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1188 if (!lgp->args.layout.pages) { 1189 kfree(lgp); 1190 return NULL; 1191 } 1192 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1193 lgp->res.layoutp = &lgp->args.layout; 1194 1195 /* Don't confuse uninitialised result and success */ 1196 lgp->res.status = -NFS4ERR_DELAY; 1197 1198 lgp->args.minlength = PAGE_SIZE; 1199 if (lgp->args.minlength > range->length) 1200 lgp->args.minlength = range->length; 1201 if (ino) { 1202 loff_t i_size = i_size_read(ino); 1203 1204 if (range->iomode == IOMODE_READ) { 1205 if (range->offset >= i_size) 1206 lgp->args.minlength = 0; 1207 else if (i_size - range->offset < lgp->args.minlength) 1208 lgp->args.minlength = i_size - range->offset; 1209 } 1210 } 1211 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 1212 pnfs_copy_range(&lgp->args.range, range); 1213 lgp->args.type = server->pnfs_curr_ld->id; 1214 lgp->args.inode = ino; 1215 lgp->args.ctx = get_nfs_open_context(ctx); 1216 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1217 lgp->gfp_flags = gfp_flags; 1218 lgp->cred = ctx->cred; 1219 return lgp; 1220 } 1221 1222 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1223 { 1224 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1225 1226 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1227 pnfs_put_layout_hdr(lgp->lo); 1228 put_nfs_open_context(lgp->args.ctx); 1229 kfree(lgp); 1230 } 1231 1232 static void pnfs_clear_layoutcommit(struct inode *inode, 1233 struct list_head *head) 1234 { 1235 struct nfs_inode *nfsi = NFS_I(inode); 1236 struct pnfs_layout_segment *lseg, *tmp; 1237 1238 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1239 return; 1240 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1241 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1242 continue; 1243 pnfs_lseg_dec_and_remove_zero(lseg, head); 1244 } 1245 } 1246 1247 static void 1248 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1249 const nfs4_stateid *arg_stateid, 1250 const struct pnfs_layout_range *range, 1251 struct list_head *freeme) 1252 { 1253 if (pnfs_layout_is_valid(lo) && 1254 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1255 pnfs_reset_return_info(lo); 1256 else 1257 pnfs_mark_layout_stateid_invalid(lo, freeme); 1258 pnfs_clear_layoutreturn_waitbit(lo); 1259 } 1260 1261 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1262 const nfs4_stateid *arg_stateid, 1263 const struct pnfs_layout_range *range) 1264 { 1265 struct inode *inode = lo->plh_inode; 1266 LIST_HEAD(freeme); 1267 1268 spin_lock(&inode->i_lock); 1269 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1270 spin_unlock(&inode->i_lock); 1271 pnfs_free_lseg_list(&freeme); 1272 } 1273 1274 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1275 const nfs4_stateid *arg_stateid, 1276 const struct pnfs_layout_range *range, 1277 const nfs4_stateid *stateid) 1278 { 1279 struct inode *inode = lo->plh_inode; 1280 LIST_HEAD(freeme); 1281 1282 spin_lock(&inode->i_lock); 1283 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1284 goto out_unlock; 1285 if (stateid && pnfs_layout_is_valid(lo)) { 1286 u32 seq = be32_to_cpu(arg_stateid->seqid); 1287 1288 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1289 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1290 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1291 pnfs_reset_return_info(lo); 1292 } else 1293 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1294 out_unlock: 1295 pnfs_clear_layoutreturn_waitbit(lo); 1296 spin_unlock(&inode->i_lock); 1297 pnfs_free_lseg_list(&freeme); 1298 1299 } 1300 1301 static bool 1302 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1303 nfs4_stateid *stateid, 1304 const struct cred **cred, 1305 enum pnfs_iomode *iomode) 1306 { 1307 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1308 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1309 return false; 1310 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1311 return false; 1312 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1313 pnfs_get_layout_hdr(lo); 1314 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1315 *cred = get_cred(lo->plh_lc_cred); 1316 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1317 if (lo->plh_return_seq != 0) 1318 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1319 if (iomode != NULL) 1320 *iomode = lo->plh_return_iomode; 1321 pnfs_clear_layoutreturn_info(lo); 1322 } else if (iomode != NULL) 1323 *iomode = IOMODE_ANY; 1324 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1325 return true; 1326 } 1327 1328 static void 1329 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1330 struct pnfs_layout_hdr *lo, 1331 const nfs4_stateid *stateid, 1332 enum pnfs_iomode iomode) 1333 { 1334 struct inode *inode = lo->plh_inode; 1335 1336 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1337 args->inode = inode; 1338 args->range.iomode = iomode; 1339 args->range.offset = 0; 1340 args->range.length = NFS4_MAX_UINT64; 1341 args->layout = lo; 1342 nfs4_stateid_copy(&args->stateid, stateid); 1343 } 1344 1345 static int 1346 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1347 const nfs4_stateid *stateid, 1348 const struct cred **pcred, 1349 enum pnfs_iomode iomode, 1350 unsigned int flags) 1351 { 1352 struct inode *ino = lo->plh_inode; 1353 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1354 struct nfs4_layoutreturn *lrp; 1355 const struct cred *cred = *pcred; 1356 int status = 0; 1357 1358 *pcred = NULL; 1359 lrp = kzalloc(sizeof(*lrp), nfs_io_gfp_mask()); 1360 if (unlikely(lrp == NULL)) { 1361 status = -ENOMEM; 1362 spin_lock(&ino->i_lock); 1363 pnfs_clear_layoutreturn_waitbit(lo); 1364 spin_unlock(&ino->i_lock); 1365 put_cred(cred); 1366 pnfs_put_layout_hdr(lo); 1367 goto out; 1368 } 1369 1370 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1371 lrp->args.ld_private = &lrp->ld_private; 1372 lrp->clp = NFS_SERVER(ino)->nfs_client; 1373 lrp->cred = cred; 1374 if (ld->prepare_layoutreturn) 1375 ld->prepare_layoutreturn(&lrp->args); 1376 1377 status = nfs4_proc_layoutreturn(lrp, flags); 1378 out: 1379 dprintk("<-- %s status: %d\n", __func__, status); 1380 return status; 1381 } 1382 1383 /* Return true if layoutreturn is needed */ 1384 static bool 1385 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1386 { 1387 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1388 return false; 1389 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1390 lo->plh_return_iomode, 1391 lo->plh_return_seq) != EBUSY; 1392 } 1393 1394 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1395 { 1396 struct inode *inode= lo->plh_inode; 1397 1398 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1399 return; 1400 spin_lock(&inode->i_lock); 1401 if (pnfs_layout_need_return(lo)) { 1402 const struct cred *cred; 1403 nfs4_stateid stateid; 1404 enum pnfs_iomode iomode; 1405 bool send; 1406 1407 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1408 spin_unlock(&inode->i_lock); 1409 if (send) { 1410 /* Send an async layoutreturn so we dont deadlock */ 1411 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1412 PNFS_FL_LAYOUTRETURN_ASYNC); 1413 } 1414 } else 1415 spin_unlock(&inode->i_lock); 1416 } 1417 1418 /* 1419 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1420 * when the layout segment list is empty. 1421 * 1422 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1423 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1424 * deviceid is marked invalid. 1425 */ 1426 int 1427 _pnfs_return_layout(struct inode *ino) 1428 { 1429 struct pnfs_layout_hdr *lo = NULL; 1430 struct nfs_inode *nfsi = NFS_I(ino); 1431 struct pnfs_layout_range range = { 1432 .iomode = IOMODE_ANY, 1433 .offset = 0, 1434 .length = NFS4_MAX_UINT64, 1435 }; 1436 LIST_HEAD(tmp_list); 1437 const struct cred *cred; 1438 nfs4_stateid stateid; 1439 int status = 0; 1440 bool send, valid_layout; 1441 1442 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1443 1444 spin_lock(&ino->i_lock); 1445 lo = nfsi->layout; 1446 if (!lo) { 1447 spin_unlock(&ino->i_lock); 1448 dprintk("NFS: %s no layout to return\n", __func__); 1449 goto out; 1450 } 1451 /* Reference matched in nfs4_layoutreturn_release */ 1452 pnfs_get_layout_hdr(lo); 1453 /* Is there an outstanding layoutreturn ? */ 1454 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1455 spin_unlock(&ino->i_lock); 1456 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1457 TASK_UNINTERRUPTIBLE)) 1458 goto out_put_layout_hdr; 1459 spin_lock(&ino->i_lock); 1460 } 1461 valid_layout = pnfs_layout_is_valid(lo); 1462 pnfs_clear_layoutcommit(ino, &tmp_list); 1463 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); 1464 1465 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1466 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1467 1468 /* Don't send a LAYOUTRETURN if list was initially empty */ 1469 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1470 !valid_layout) { 1471 spin_unlock(&ino->i_lock); 1472 dprintk("NFS: %s no layout segments to return\n", __func__); 1473 goto out_wait_layoutreturn; 1474 } 1475 1476 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1477 spin_unlock(&ino->i_lock); 1478 if (send) 1479 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1480 0); 1481 out_wait_layoutreturn: 1482 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1483 out_put_layout_hdr: 1484 pnfs_free_lseg_list(&tmp_list); 1485 pnfs_put_layout_hdr(lo); 1486 out: 1487 dprintk("<-- %s status: %d\n", __func__, status); 1488 return status; 1489 } 1490 1491 int 1492 pnfs_commit_and_return_layout(struct inode *inode) 1493 { 1494 struct pnfs_layout_hdr *lo; 1495 int ret; 1496 1497 spin_lock(&inode->i_lock); 1498 lo = NFS_I(inode)->layout; 1499 if (lo == NULL) { 1500 spin_unlock(&inode->i_lock); 1501 return 0; 1502 } 1503 pnfs_get_layout_hdr(lo); 1504 /* Block new layoutgets and read/write to ds */ 1505 lo->plh_block_lgets++; 1506 spin_unlock(&inode->i_lock); 1507 filemap_fdatawait(inode->i_mapping); 1508 ret = pnfs_layoutcommit_inode(inode, true); 1509 if (ret == 0) 1510 ret = _pnfs_return_layout(inode); 1511 spin_lock(&inode->i_lock); 1512 lo->plh_block_lgets--; 1513 spin_unlock(&inode->i_lock); 1514 pnfs_put_layout_hdr(lo); 1515 return ret; 1516 } 1517 1518 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1519 { 1520 struct inode *inode = lo->plh_inode; 1521 const struct cred *cred; 1522 1523 spin_lock(&inode->i_lock); 1524 if (!pnfs_layout_is_valid(lo)) { 1525 spin_unlock(&inode->i_lock); 1526 return 0; 1527 } 1528 cred = get_cred(lo->plh_lc_cred); 1529 pnfs_get_layout_hdr(lo); 1530 spin_unlock(&inode->i_lock); 1531 1532 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1533 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1534 } 1535 1536 bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, 1537 struct nfs4_layoutreturn_res *res, const struct cred *cred, 1538 bool sync) 1539 { 1540 struct nfs_inode *nfsi = NFS_I(ino); 1541 struct nfs_open_context *ctx; 1542 struct nfs4_state *state; 1543 struct pnfs_layout_hdr *lo; 1544 struct pnfs_layout_segment *lseg, *next; 1545 const struct cred *lc_cred; 1546 nfs4_stateid stateid; 1547 enum pnfs_iomode iomode = 0; 1548 bool layoutreturn = false, roc = false; 1549 bool skip_read; 1550 1551 if (!nfs_have_layout(ino)) 1552 return false; 1553 retry: 1554 rcu_read_lock(); 1555 spin_lock(&ino->i_lock); 1556 lo = nfsi->layout; 1557 if (!lo || !pnfs_layout_is_valid(lo) || 1558 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1559 lo = NULL; 1560 goto out_noroc; 1561 } 1562 1563 /* no roc if we hold a delegation */ 1564 skip_read = false; 1565 if (nfs4_check_delegation(ino, FMODE_READ)) { 1566 if (nfs4_check_delegation(ino, FMODE_WRITE)) { 1567 lo = NULL; 1568 goto out_noroc; 1569 } 1570 skip_read = true; 1571 } 1572 1573 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1574 state = ctx->state; 1575 if (state == NULL) 1576 continue; 1577 /* Don't return layout if there is open file state */ 1578 if (state->state & FMODE_WRITE) { 1579 lo = NULL; 1580 goto out_noroc; 1581 } 1582 if (state->state & FMODE_READ) 1583 skip_read = true; 1584 } 1585 1586 if (skip_read) { 1587 bool writes = false; 1588 1589 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1590 if (lseg->pls_range.iomode != IOMODE_READ) { 1591 writes = true; 1592 break; 1593 } 1594 } 1595 if (!writes) { 1596 lo = NULL; 1597 goto out_noroc; 1598 } 1599 } 1600 1601 pnfs_get_layout_hdr(lo); 1602 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1603 if (!sync) { 1604 pnfs_set_plh_return_info( 1605 lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); 1606 goto out_noroc; 1607 } 1608 spin_unlock(&ino->i_lock); 1609 rcu_read_unlock(); 1610 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1611 TASK_UNINTERRUPTIBLE); 1612 pnfs_put_layout_hdr(lo); 1613 goto retry; 1614 } 1615 1616 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1617 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1618 continue; 1619 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1620 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1621 continue; 1622 /* 1623 * Note: mark lseg for return so pnfs_layout_remove_lseg 1624 * doesn't invalidate the layout for us. 1625 */ 1626 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1627 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1628 continue; 1629 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1630 } 1631 1632 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1633 goto out_noroc; 1634 1635 /* ROC in two conditions: 1636 * 1. there are ROC lsegs 1637 * 2. we don't send layoutreturn 1638 */ 1639 /* lo ref dropped in pnfs_roc_release() */ 1640 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1641 /* If the creds don't match, we can't compound the layoutreturn */ 1642 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1643 goto out_noroc; 1644 1645 roc = layoutreturn; 1646 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1647 res->lrs_present = 0; 1648 layoutreturn = false; 1649 put_cred(lc_cred); 1650 1651 out_noroc: 1652 spin_unlock(&ino->i_lock); 1653 rcu_read_unlock(); 1654 pnfs_layoutcommit_inode(ino, sync); 1655 if (roc) { 1656 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1657 if (ld->prepare_layoutreturn) 1658 ld->prepare_layoutreturn(args); 1659 pnfs_put_layout_hdr(lo); 1660 return true; 1661 } 1662 if (layoutreturn) 1663 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1664 pnfs_put_layout_hdr(lo); 1665 return false; 1666 } 1667 1668 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1669 struct nfs4_layoutreturn_res **respp, int *ret) 1670 { 1671 struct nfs4_layoutreturn_args *arg = *argpp; 1672 int retval = -EAGAIN; 1673 1674 if (!arg) 1675 return 0; 1676 /* Handle Layoutreturn errors */ 1677 switch (*ret) { 1678 case 0: 1679 retval = 0; 1680 break; 1681 case -NFS4ERR_NOMATCHING_LAYOUT: 1682 /* Was there an RPC level error? If not, retry */ 1683 if (task->tk_rpc_status == 0) 1684 break; 1685 /* 1686 * Is there a fatal network level error? 1687 * If so release the layout, but flag the error. 1688 */ 1689 if ((task->tk_rpc_status == -ENETDOWN || 1690 task->tk_rpc_status == -ENETUNREACH) && 1691 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1692 *ret = 0; 1693 (*respp)->lrs_present = 0; 1694 retval = -EIO; 1695 break; 1696 } 1697 /* If the call was not sent, let caller handle it */ 1698 if (!RPC_WAS_SENT(task)) 1699 return 0; 1700 /* 1701 * Otherwise, assume the call succeeded and 1702 * that we need to release the layout 1703 */ 1704 *ret = 0; 1705 (*respp)->lrs_present = 0; 1706 retval = 0; 1707 break; 1708 case -NFS4ERR_DELAY: 1709 /* Let the caller handle the retry */ 1710 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1711 return 0; 1712 case -NFS4ERR_OLD_STATEID: 1713 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1714 &arg->range, arg->inode)) 1715 break; 1716 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1717 return -EAGAIN; 1718 } 1719 *argpp = NULL; 1720 *respp = NULL; 1721 return retval; 1722 } 1723 1724 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1725 struct nfs4_layoutreturn_res *res, int ret) 1726 { 1727 struct pnfs_layout_hdr *lo = args->layout; 1728 struct inode *inode = args->inode; 1729 const nfs4_stateid *res_stateid = NULL; 1730 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1731 LIST_HEAD(freeme); 1732 1733 switch (ret) { 1734 case -NFS4ERR_BADSESSION: 1735 case -NFS4ERR_DEADSESSION: 1736 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1737 case -NFS4ERR_NOMATCHING_LAYOUT: 1738 spin_lock(&inode->i_lock); 1739 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1740 &args->range, &freeme); 1741 spin_unlock(&inode->i_lock); 1742 pnfs_free_lseg_list(&freeme); 1743 break; 1744 case 0: 1745 if (res->lrs_present) 1746 res_stateid = &res->stateid; 1747 fallthrough; 1748 default: 1749 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1750 res_stateid); 1751 } 1752 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1753 if (ld_private && ld_private->ops && ld_private->ops->free) 1754 ld_private->ops->free(ld_private); 1755 pnfs_put_layout_hdr(lo); 1756 } 1757 1758 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1759 { 1760 struct nfs_inode *nfsi = NFS_I(ino); 1761 struct pnfs_layout_hdr *lo; 1762 bool sleep = false; 1763 1764 /* we might not have grabbed lo reference. so need to check under 1765 * i_lock */ 1766 spin_lock(&ino->i_lock); 1767 lo = nfsi->layout; 1768 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1769 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1770 sleep = true; 1771 } 1772 spin_unlock(&ino->i_lock); 1773 return sleep; 1774 } 1775 1776 /* 1777 * Compare two layout segments for sorting into layout cache. 1778 * We want to preferentially return RW over RO layouts, so ensure those 1779 * are seen first. 1780 */ 1781 static s64 1782 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1783 const struct pnfs_layout_range *l2) 1784 { 1785 s64 d; 1786 1787 /* high offset > low offset */ 1788 d = l1->offset - l2->offset; 1789 if (d) 1790 return d; 1791 1792 /* short length > long length */ 1793 d = l2->length - l1->length; 1794 if (d) 1795 return d; 1796 1797 /* read > read/write */ 1798 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1799 } 1800 1801 static bool 1802 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1803 const struct pnfs_layout_range *l2) 1804 { 1805 return pnfs_lseg_range_cmp(l1, l2) > 0; 1806 } 1807 1808 static bool 1809 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1810 struct pnfs_layout_segment *old) 1811 { 1812 return false; 1813 } 1814 1815 void 1816 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1817 struct pnfs_layout_segment *lseg, 1818 bool (*is_after)(const struct pnfs_layout_range *, 1819 const struct pnfs_layout_range *), 1820 bool (*do_merge)(struct pnfs_layout_segment *, 1821 struct pnfs_layout_segment *), 1822 struct list_head *free_me) 1823 { 1824 struct pnfs_layout_segment *lp, *tmp; 1825 1826 dprintk("%s:Begin\n", __func__); 1827 1828 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1829 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1830 continue; 1831 if (do_merge(lseg, lp)) { 1832 mark_lseg_invalid(lp, free_me); 1833 continue; 1834 } 1835 if (is_after(&lseg->pls_range, &lp->pls_range)) 1836 continue; 1837 list_add_tail(&lseg->pls_list, &lp->pls_list); 1838 dprintk("%s: inserted lseg %p " 1839 "iomode %d offset %llu length %llu before " 1840 "lp %p iomode %d offset %llu length %llu\n", 1841 __func__, lseg, lseg->pls_range.iomode, 1842 lseg->pls_range.offset, lseg->pls_range.length, 1843 lp, lp->pls_range.iomode, lp->pls_range.offset, 1844 lp->pls_range.length); 1845 goto out; 1846 } 1847 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1848 dprintk("%s: inserted lseg %p " 1849 "iomode %d offset %llu length %llu at tail\n", 1850 __func__, lseg, lseg->pls_range.iomode, 1851 lseg->pls_range.offset, lseg->pls_range.length); 1852 out: 1853 pnfs_get_layout_hdr(lo); 1854 1855 dprintk("%s:Return\n", __func__); 1856 } 1857 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1858 1859 static void 1860 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1861 struct pnfs_layout_segment *lseg, 1862 struct list_head *free_me) 1863 { 1864 struct inode *inode = lo->plh_inode; 1865 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1866 1867 if (ld->add_lseg != NULL) 1868 ld->add_lseg(lo, lseg, free_me); 1869 else 1870 pnfs_generic_layout_insert_lseg(lo, lseg, 1871 pnfs_lseg_range_is_after, 1872 pnfs_lseg_no_merge, 1873 free_me); 1874 } 1875 1876 static struct pnfs_layout_hdr * 1877 alloc_init_layout_hdr(struct inode *ino, 1878 struct nfs_open_context *ctx, 1879 gfp_t gfp_flags) 1880 { 1881 struct pnfs_layout_hdr *lo; 1882 1883 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1884 if (!lo) 1885 return NULL; 1886 refcount_set(&lo->plh_refcount, 1); 1887 INIT_LIST_HEAD(&lo->plh_layouts); 1888 INIT_LIST_HEAD(&lo->plh_segs); 1889 INIT_LIST_HEAD(&lo->plh_return_segs); 1890 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1891 lo->plh_inode = ino; 1892 lo->plh_lc_cred = get_cred(ctx->cred); 1893 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1894 return lo; 1895 } 1896 1897 static struct pnfs_layout_hdr * 1898 pnfs_find_alloc_layout(struct inode *ino, 1899 struct nfs_open_context *ctx, 1900 gfp_t gfp_flags) 1901 __releases(&ino->i_lock) 1902 __acquires(&ino->i_lock) 1903 { 1904 struct nfs_inode *nfsi = NFS_I(ino); 1905 struct pnfs_layout_hdr *new = NULL; 1906 1907 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1908 1909 if (nfsi->layout != NULL) 1910 goto out_existing; 1911 spin_unlock(&ino->i_lock); 1912 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1913 spin_lock(&ino->i_lock); 1914 1915 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1916 nfsi->layout = new; 1917 return new; 1918 } else if (new != NULL) 1919 pnfs_free_layout_hdr(new); 1920 out_existing: 1921 pnfs_get_layout_hdr(nfsi->layout); 1922 return nfsi->layout; 1923 } 1924 1925 /* 1926 * iomode matching rules: 1927 * iomode lseg strict match 1928 * iomode 1929 * ----- ----- ------ ----- 1930 * ANY READ N/A true 1931 * ANY RW N/A true 1932 * RW READ N/A false 1933 * RW RW N/A true 1934 * READ READ N/A true 1935 * READ RW true false 1936 * READ RW false true 1937 */ 1938 static bool 1939 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1940 const struct pnfs_layout_range *range, 1941 bool strict_iomode) 1942 { 1943 struct pnfs_layout_range range1; 1944 1945 if ((range->iomode == IOMODE_RW && 1946 ls_range->iomode != IOMODE_RW) || 1947 (range->iomode != ls_range->iomode && 1948 strict_iomode) || 1949 !pnfs_lseg_range_intersecting(ls_range, range)) 1950 return false; 1951 1952 /* range1 covers only the first byte in the range */ 1953 range1 = *range; 1954 range1.length = 1; 1955 return pnfs_lseg_range_contained(ls_range, &range1); 1956 } 1957 1958 /* 1959 * lookup range in layout 1960 */ 1961 static struct pnfs_layout_segment * 1962 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1963 struct pnfs_layout_range *range, 1964 bool strict_iomode) 1965 { 1966 struct pnfs_layout_segment *lseg, *ret = NULL; 1967 1968 dprintk("%s:Begin\n", __func__); 1969 1970 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1971 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1972 pnfs_lseg_range_match(&lseg->pls_range, range, 1973 strict_iomode)) { 1974 ret = pnfs_get_lseg(lseg); 1975 break; 1976 } 1977 } 1978 1979 dprintk("%s:Return lseg %p ref %d\n", 1980 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 1981 return ret; 1982 } 1983 1984 /* 1985 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1986 * to the MDS or over pNFS 1987 * 1988 * The nfs_inode read_io and write_io fields are cumulative counters reset 1989 * when there are no layout segments. Note that in pnfs_update_layout iomode 1990 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1991 * WRITE request. 1992 * 1993 * A return of true means use MDS I/O. 1994 * 1995 * From rfc 5661: 1996 * If a file's size is smaller than the file size threshold, data accesses 1997 * SHOULD be sent to the metadata server. If an I/O request has a length that 1998 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1999 * server. If both file size and I/O size are provided, the client SHOULD 2000 * reach or exceed both thresholds before sending its read or write 2001 * requests to the data server. 2002 */ 2003 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 2004 struct inode *ino, int iomode) 2005 { 2006 struct nfs4_threshold *t = ctx->mdsthreshold; 2007 struct nfs_inode *nfsi = NFS_I(ino); 2008 loff_t fsize = i_size_read(ino); 2009 bool size = false, size_set = false, io = false, io_set = false, ret = false; 2010 2011 if (t == NULL) 2012 return ret; 2013 2014 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 2015 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 2016 2017 switch (iomode) { 2018 case IOMODE_READ: 2019 if (t->bm & THRESHOLD_RD) { 2020 dprintk("%s fsize %llu\n", __func__, fsize); 2021 size_set = true; 2022 if (fsize < t->rd_sz) 2023 size = true; 2024 } 2025 if (t->bm & THRESHOLD_RD_IO) { 2026 dprintk("%s nfsi->read_io %llu\n", __func__, 2027 nfsi->read_io); 2028 io_set = true; 2029 if (nfsi->read_io < t->rd_io_sz) 2030 io = true; 2031 } 2032 break; 2033 case IOMODE_RW: 2034 if (t->bm & THRESHOLD_WR) { 2035 dprintk("%s fsize %llu\n", __func__, fsize); 2036 size_set = true; 2037 if (fsize < t->wr_sz) 2038 size = true; 2039 } 2040 if (t->bm & THRESHOLD_WR_IO) { 2041 dprintk("%s nfsi->write_io %llu\n", __func__, 2042 nfsi->write_io); 2043 io_set = true; 2044 if (nfsi->write_io < t->wr_io_sz) 2045 io = true; 2046 } 2047 break; 2048 } 2049 if (size_set && io_set) { 2050 if (size && io) 2051 ret = true; 2052 } else if (size || io) 2053 ret = true; 2054 2055 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2056 return ret; 2057 } 2058 2059 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2060 { 2061 /* 2062 * send layoutcommit as it can hold up layoutreturn due to lseg 2063 * reference 2064 */ 2065 pnfs_layoutcommit_inode(lo->plh_inode, false); 2066 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2067 nfs_wait_bit_killable, 2068 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2069 } 2070 2071 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2072 { 2073 atomic_inc(&lo->plh_outstanding); 2074 } 2075 2076 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2077 { 2078 if (atomic_dec_and_test(&lo->plh_outstanding) && 2079 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2080 smp_mb__after_atomic(); 2081 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2082 } 2083 } 2084 2085 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2086 { 2087 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2088 } 2089 2090 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 2091 { 2092 unsigned long *bitlock = &lo->plh_flags; 2093 2094 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 2095 smp_mb__after_atomic(); 2096 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 2097 } 2098 2099 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2100 struct nfs_server *server) 2101 { 2102 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2103 struct nfs_client *clp = server->nfs_client; 2104 2105 /* The lo must be on the clp list if there is any 2106 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2107 */ 2108 spin_lock(&clp->cl_lock); 2109 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2110 spin_unlock(&clp->cl_lock); 2111 } 2112 } 2113 2114 /* 2115 * Layout segment is retreived from the server if not cached. 2116 * The appropriate layout segment is referenced and returned to the caller. 2117 */ 2118 struct pnfs_layout_segment * 2119 pnfs_update_layout(struct inode *ino, 2120 struct nfs_open_context *ctx, 2121 loff_t pos, 2122 u64 count, 2123 enum pnfs_iomode iomode, 2124 bool strict_iomode, 2125 gfp_t gfp_flags) 2126 { 2127 struct pnfs_layout_range arg = { 2128 .iomode = iomode, 2129 .offset = pos, 2130 .length = count, 2131 }; 2132 unsigned pg_offset; 2133 struct nfs_server *server = NFS_SERVER(ino); 2134 struct nfs_client *clp = server->nfs_client; 2135 struct pnfs_layout_hdr *lo = NULL; 2136 struct pnfs_layout_segment *lseg = NULL; 2137 struct nfs4_layoutget *lgp; 2138 nfs4_stateid stateid; 2139 struct nfs4_exception exception = { 2140 .inode = ino, 2141 }; 2142 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2143 bool first; 2144 2145 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2146 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2147 PNFS_UPDATE_LAYOUT_NO_PNFS); 2148 goto out; 2149 } 2150 2151 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2152 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2153 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2154 goto out; 2155 } 2156 2157 lookup_again: 2158 if (!nfs4_valid_open_stateid(ctx->state)) { 2159 trace_pnfs_update_layout(ino, pos, count, 2160 iomode, lo, lseg, 2161 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2162 lseg = ERR_PTR(-EIO); 2163 goto out; 2164 } 2165 2166 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2167 if (IS_ERR(lseg)) 2168 goto out; 2169 first = false; 2170 spin_lock(&ino->i_lock); 2171 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2172 if (lo == NULL) { 2173 spin_unlock(&ino->i_lock); 2174 lseg = ERR_PTR(-ENOMEM); 2175 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2176 PNFS_UPDATE_LAYOUT_NOMEM); 2177 goto out; 2178 } 2179 2180 /* Do we even need to bother with this? */ 2181 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2182 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2183 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2184 dprintk("%s matches recall, use MDS\n", __func__); 2185 goto out_unlock; 2186 } 2187 2188 /* if LAYOUTGET already failed once we don't try again */ 2189 if (pnfs_layout_io_test_failed(lo, iomode)) { 2190 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2191 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2192 goto out_unlock; 2193 } 2194 2195 /* 2196 * If the layout segment list is empty, but there are outstanding 2197 * layoutget calls, then they might be subject to a layoutrecall. 2198 */ 2199 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2200 atomic_read(&lo->plh_outstanding) != 0) { 2201 spin_unlock(&ino->i_lock); 2202 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2203 TASK_KILLABLE)); 2204 if (IS_ERR(lseg)) 2205 goto out_put_layout_hdr; 2206 pnfs_put_layout_hdr(lo); 2207 goto lookup_again; 2208 } 2209 2210 /* 2211 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2212 * for LAYOUTRETURN. 2213 */ 2214 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2215 spin_unlock(&ino->i_lock); 2216 dprintk("%s wait for layoutreturn\n", __func__); 2217 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2218 if (!IS_ERR(lseg)) { 2219 pnfs_put_layout_hdr(lo); 2220 dprintk("%s retrying\n", __func__); 2221 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2222 lseg, 2223 PNFS_UPDATE_LAYOUT_RETRY); 2224 goto lookup_again; 2225 } 2226 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2227 PNFS_UPDATE_LAYOUT_RETURN); 2228 goto out_put_layout_hdr; 2229 } 2230 2231 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2232 if (lseg) { 2233 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2234 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2235 goto out_unlock; 2236 } 2237 2238 /* 2239 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2240 * stateid, or it has been invalidated, then we must use the open 2241 * stateid. 2242 */ 2243 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2244 int status; 2245 2246 /* 2247 * The first layoutget for the file. Need to serialize per 2248 * RFC 5661 Errata 3208. 2249 */ 2250 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2251 &lo->plh_flags)) { 2252 spin_unlock(&ino->i_lock); 2253 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2254 NFS_LAYOUT_FIRST_LAYOUTGET, 2255 TASK_KILLABLE)); 2256 if (IS_ERR(lseg)) 2257 goto out_put_layout_hdr; 2258 pnfs_put_layout_hdr(lo); 2259 dprintk("%s retrying\n", __func__); 2260 goto lookup_again; 2261 } 2262 2263 spin_unlock(&ino->i_lock); 2264 first = true; 2265 status = nfs4_select_rw_stateid(ctx->state, 2266 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2267 NULL, &stateid, NULL); 2268 if (status != 0) { 2269 lseg = ERR_PTR(status); 2270 trace_pnfs_update_layout(ino, pos, count, 2271 iomode, lo, lseg, 2272 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2273 nfs4_schedule_stateid_recovery(server, ctx->state); 2274 pnfs_clear_first_layoutget(lo); 2275 pnfs_put_layout_hdr(lo); 2276 goto lookup_again; 2277 } 2278 spin_lock(&ino->i_lock); 2279 } else { 2280 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2281 } 2282 2283 if (pnfs_layoutgets_blocked(lo)) { 2284 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2285 PNFS_UPDATE_LAYOUT_BLOCKED); 2286 goto out_unlock; 2287 } 2288 nfs_layoutget_begin(lo); 2289 spin_unlock(&ino->i_lock); 2290 2291 _add_to_server_list(lo, server); 2292 2293 pg_offset = arg.offset & ~PAGE_MASK; 2294 if (pg_offset) { 2295 arg.offset -= pg_offset; 2296 arg.length += pg_offset; 2297 } 2298 if (arg.length != NFS4_MAX_UINT64) 2299 arg.length = PAGE_ALIGN(arg.length); 2300 2301 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); 2302 if (!lgp) { 2303 lseg = ERR_PTR(-ENOMEM); 2304 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2305 PNFS_UPDATE_LAYOUT_NOMEM); 2306 nfs_layoutget_end(lo); 2307 goto out_put_layout_hdr; 2308 } 2309 2310 lgp->lo = lo; 2311 pnfs_get_layout_hdr(lo); 2312 2313 lseg = nfs4_proc_layoutget(lgp, &exception); 2314 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2315 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2316 nfs_layoutget_end(lo); 2317 if (IS_ERR(lseg)) { 2318 switch(PTR_ERR(lseg)) { 2319 case -EBUSY: 2320 if (time_after(jiffies, giveup)) 2321 lseg = NULL; 2322 break; 2323 case -ERECALLCONFLICT: 2324 case -EAGAIN: 2325 break; 2326 case -ENODATA: 2327 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2328 pnfs_layout_set_fail_bit( 2329 lo, pnfs_iomode_to_fail_bit(iomode)); 2330 lseg = NULL; 2331 goto out_put_layout_hdr; 2332 default: 2333 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2334 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2335 lseg = NULL; 2336 } 2337 goto out_put_layout_hdr; 2338 } 2339 if (lseg) { 2340 if (!exception.retry) 2341 goto out_put_layout_hdr; 2342 if (first) 2343 pnfs_clear_first_layoutget(lo); 2344 trace_pnfs_update_layout(ino, pos, count, 2345 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2346 pnfs_put_layout_hdr(lo); 2347 goto lookup_again; 2348 } 2349 } else { 2350 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2351 } 2352 2353 out_put_layout_hdr: 2354 if (first) 2355 pnfs_clear_first_layoutget(lo); 2356 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2357 PNFS_UPDATE_LAYOUT_EXIT); 2358 pnfs_put_layout_hdr(lo); 2359 out: 2360 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2361 "(%s, offset: %llu, length: %llu)\n", 2362 __func__, ino->i_sb->s_id, 2363 (unsigned long long)NFS_FILEID(ino), 2364 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2365 iomode==IOMODE_RW ? "read/write" : "read-only", 2366 (unsigned long long)pos, 2367 (unsigned long long)count); 2368 return lseg; 2369 out_unlock: 2370 spin_unlock(&ino->i_lock); 2371 goto out_put_layout_hdr; 2372 } 2373 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2374 2375 static bool 2376 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2377 { 2378 switch (range->iomode) { 2379 case IOMODE_READ: 2380 case IOMODE_RW: 2381 break; 2382 default: 2383 return false; 2384 } 2385 if (range->offset == NFS4_MAX_UINT64) 2386 return false; 2387 if (range->length == 0) 2388 return false; 2389 if (range->length != NFS4_MAX_UINT64 && 2390 range->length > NFS4_MAX_UINT64 - range->offset) 2391 return false; 2392 return true; 2393 } 2394 2395 static struct pnfs_layout_hdr * 2396 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2397 { 2398 struct pnfs_layout_hdr *lo; 2399 2400 spin_lock(&ino->i_lock); 2401 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2402 if (!lo) 2403 goto out_unlock; 2404 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2405 goto out_unlock; 2406 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2407 goto out_unlock; 2408 if (pnfs_layoutgets_blocked(lo)) 2409 goto out_unlock; 2410 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2411 goto out_unlock; 2412 nfs_layoutget_begin(lo); 2413 spin_unlock(&ino->i_lock); 2414 _add_to_server_list(lo, NFS_SERVER(ino)); 2415 return lo; 2416 2417 out_unlock: 2418 spin_unlock(&ino->i_lock); 2419 pnfs_put_layout_hdr(lo); 2420 return NULL; 2421 } 2422 2423 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2424 struct nfs_open_context *ctx) 2425 { 2426 struct inode *ino = data->dentry->d_inode; 2427 struct pnfs_layout_range rng = { 2428 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2429 IOMODE_RW: IOMODE_READ, 2430 .offset = 0, 2431 .length = NFS4_MAX_UINT64, 2432 }; 2433 struct nfs4_layoutget *lgp; 2434 struct pnfs_layout_hdr *lo; 2435 2436 /* Heuristic: don't send layoutget if we have cached data */ 2437 if (rng.iomode == IOMODE_READ && 2438 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2439 return; 2440 2441 lo = _pnfs_grab_empty_layout(ino, ctx); 2442 if (!lo) 2443 return; 2444 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2445 nfs_io_gfp_mask()); 2446 if (!lgp) { 2447 pnfs_clear_first_layoutget(lo); 2448 nfs_layoutget_end(lo); 2449 pnfs_put_layout_hdr(lo); 2450 return; 2451 } 2452 lgp->lo = lo; 2453 data->lgp = lgp; 2454 data->o_arg.lg_args = &lgp->args; 2455 data->o_res.lg_res = &lgp->res; 2456 } 2457 2458 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2459 struct nfs_open_context *ctx) 2460 { 2461 struct inode *ino = data->dentry->d_inode; 2462 struct pnfs_layout_range rng = { 2463 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2464 IOMODE_RW: IOMODE_READ, 2465 .offset = 0, 2466 .length = NFS4_MAX_UINT64, 2467 }; 2468 struct nfs4_layoutget *lgp; 2469 2470 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2471 nfs_io_gfp_mask()); 2472 if (!lgp) 2473 return; 2474 data->lgp = lgp; 2475 data->o_arg.lg_args = &lgp->args; 2476 data->o_res.lg_res = &lgp->res; 2477 } 2478 2479 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2480 struct nfs_open_context *ctx) 2481 { 2482 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2483 2484 if (!(pnfs_enabled_sb(server) && 2485 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2486 return; 2487 /* Could check on max_ops, but currently hardcoded high enough */ 2488 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2489 return; 2490 if (data->lgp) 2491 return; 2492 if (data->state) 2493 _lgopen_prepare_attached(data, ctx); 2494 else 2495 _lgopen_prepare_floating(data, ctx); 2496 } 2497 2498 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2499 struct nfs_open_context *ctx) 2500 { 2501 struct pnfs_layout_hdr *lo; 2502 struct pnfs_layout_segment *lseg; 2503 struct nfs_server *srv = NFS_SERVER(ino); 2504 u32 iomode; 2505 2506 if (!lgp) 2507 return; 2508 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2509 if (lgp->res.status) { 2510 switch (lgp->res.status) { 2511 default: 2512 break; 2513 /* 2514 * Halt lgopen attempts if the server doesn't recognise 2515 * the "current stateid" value, the layout type, or the 2516 * layoutget operation as being valid. 2517 * Also if it complains about too many ops in the compound 2518 * or of the request/reply being too big. 2519 */ 2520 case -NFS4ERR_BAD_STATEID: 2521 case -NFS4ERR_NOTSUPP: 2522 case -NFS4ERR_REP_TOO_BIG: 2523 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2524 case -NFS4ERR_REQ_TOO_BIG: 2525 case -NFS4ERR_TOO_MANY_OPS: 2526 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2527 srv->caps &= ~NFS_CAP_LGOPEN; 2528 } 2529 return; 2530 } 2531 if (!lgp->lo) { 2532 lo = _pnfs_grab_empty_layout(ino, ctx); 2533 if (!lo) 2534 return; 2535 lgp->lo = lo; 2536 } else 2537 lo = lgp->lo; 2538 2539 lseg = pnfs_layout_process(lgp); 2540 if (!IS_ERR(lseg)) { 2541 iomode = lgp->args.range.iomode; 2542 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2543 pnfs_put_lseg(lseg); 2544 } 2545 } 2546 2547 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2548 { 2549 if (lgp != NULL) { 2550 if (lgp->lo) { 2551 pnfs_clear_first_layoutget(lgp->lo); 2552 nfs_layoutget_end(lgp->lo); 2553 } 2554 pnfs_layoutget_free(lgp); 2555 } 2556 } 2557 2558 struct pnfs_layout_segment * 2559 pnfs_layout_process(struct nfs4_layoutget *lgp) 2560 { 2561 struct pnfs_layout_hdr *lo = lgp->lo; 2562 struct nfs4_layoutget_res *res = &lgp->res; 2563 struct pnfs_layout_segment *lseg; 2564 struct inode *ino = lo->plh_inode; 2565 LIST_HEAD(free_me); 2566 2567 if (!pnfs_sanity_check_layout_range(&res->range)) 2568 return ERR_PTR(-EINVAL); 2569 2570 /* Inject layout blob into I/O device driver */ 2571 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2572 if (IS_ERR_OR_NULL(lseg)) { 2573 if (!lseg) 2574 lseg = ERR_PTR(-ENOMEM); 2575 2576 dprintk("%s: Could not allocate layout: error %ld\n", 2577 __func__, PTR_ERR(lseg)); 2578 return lseg; 2579 } 2580 2581 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2582 2583 spin_lock(&ino->i_lock); 2584 if (pnfs_layoutgets_blocked(lo)) { 2585 dprintk("%s forget reply due to state\n", __func__); 2586 goto out_forget; 2587 } 2588 2589 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2590 !pnfs_is_first_layoutget(lo)) 2591 goto out_forget; 2592 2593 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2594 /* existing state ID, make sure the sequence number matches. */ 2595 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2596 if (!pnfs_layout_is_valid(lo)) 2597 lo->plh_barrier = 0; 2598 dprintk("%s forget reply due to sequence\n", __func__); 2599 goto out_forget; 2600 } 2601 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2602 } else if (pnfs_layout_is_valid(lo)) { 2603 /* 2604 * We got an entirely new state ID. Mark all segments for the 2605 * inode invalid, and retry the layoutget 2606 */ 2607 struct pnfs_layout_range range = { 2608 .iomode = IOMODE_ANY, 2609 .length = NFS4_MAX_UINT64, 2610 }; 2611 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); 2612 goto out_forget; 2613 } else { 2614 /* We have a completely new layout */ 2615 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2616 } 2617 2618 pnfs_get_lseg(lseg); 2619 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2620 2621 2622 if (res->return_on_close) 2623 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2624 2625 spin_unlock(&ino->i_lock); 2626 pnfs_free_lseg_list(&free_me); 2627 return lseg; 2628 2629 out_forget: 2630 spin_unlock(&ino->i_lock); 2631 lseg->pls_layout = lo; 2632 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2633 return ERR_PTR(-EAGAIN); 2634 } 2635 2636 /** 2637 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2638 * @lo: pointer to layout header 2639 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2640 * @return_range: describe layout segment ranges to be returned 2641 * @seq: stateid seqid to match 2642 * 2643 * This function is mainly intended for use by layoutrecall. It attempts 2644 * to free the layout segment immediately, or else to mark it for return 2645 * as soon as its reference count drops to zero. 2646 * 2647 * Returns 2648 * - 0: a layoutreturn needs to be scheduled. 2649 * - EBUSY: there are layout segment that are still in use. 2650 * - ENOENT: there are no layout segments that need to be returned. 2651 */ 2652 int 2653 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2654 struct list_head *tmp_list, 2655 const struct pnfs_layout_range *return_range, 2656 u32 seq) 2657 { 2658 struct pnfs_layout_segment *lseg, *next; 2659 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2660 int remaining = 0; 2661 2662 dprintk("%s:Begin lo %p\n", __func__, lo); 2663 2664 assert_spin_locked(&lo->plh_inode->i_lock); 2665 2666 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2667 tmp_list = &lo->plh_return_segs; 2668 2669 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2670 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2671 dprintk("%s: marking lseg %p iomode %d " 2672 "offset %llu length %llu\n", __func__, 2673 lseg, lseg->pls_range.iomode, 2674 lseg->pls_range.offset, 2675 lseg->pls_range.length); 2676 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2677 tmp_list = &lo->plh_return_segs; 2678 if (mark_lseg_invalid(lseg, tmp_list)) 2679 continue; 2680 remaining++; 2681 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2682 pnfs_lseg_cancel_io(server, lseg); 2683 } 2684 2685 if (remaining) { 2686 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2687 return -EBUSY; 2688 } 2689 2690 if (!list_empty(&lo->plh_return_segs)) { 2691 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2692 return 0; 2693 } 2694 2695 return -ENOENT; 2696 } 2697 2698 static void 2699 pnfs_mark_layout_for_return(struct inode *inode, 2700 const struct pnfs_layout_range *range) 2701 { 2702 struct pnfs_layout_hdr *lo; 2703 bool return_now = false; 2704 2705 spin_lock(&inode->i_lock); 2706 lo = NFS_I(inode)->layout; 2707 if (!pnfs_layout_is_valid(lo)) { 2708 spin_unlock(&inode->i_lock); 2709 return; 2710 } 2711 pnfs_set_plh_return_info(lo, range->iomode, 0); 2712 /* 2713 * mark all matching lsegs so that we are sure to have no live 2714 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2715 * for how it works. 2716 */ 2717 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { 2718 const struct cred *cred; 2719 nfs4_stateid stateid; 2720 enum pnfs_iomode iomode; 2721 2722 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2723 spin_unlock(&inode->i_lock); 2724 if (return_now) 2725 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2726 PNFS_FL_LAYOUTRETURN_ASYNC); 2727 } else { 2728 spin_unlock(&inode->i_lock); 2729 nfs_commit_inode(inode, 0); 2730 } 2731 } 2732 2733 void pnfs_error_mark_layout_for_return(struct inode *inode, 2734 struct pnfs_layout_segment *lseg) 2735 { 2736 struct pnfs_layout_range range = { 2737 .iomode = lseg->pls_range.iomode, 2738 .offset = 0, 2739 .length = NFS4_MAX_UINT64, 2740 }; 2741 2742 pnfs_mark_layout_for_return(inode, &range); 2743 } 2744 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2745 2746 static bool 2747 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2748 { 2749 return pnfs_layout_is_valid(lo) && 2750 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2751 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2752 } 2753 2754 static struct pnfs_layout_segment * 2755 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2756 const struct pnfs_layout_range *range, 2757 enum pnfs_iomode iomode) 2758 { 2759 struct pnfs_layout_segment *lseg; 2760 2761 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2762 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2763 continue; 2764 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2765 continue; 2766 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2767 continue; 2768 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2769 return lseg; 2770 } 2771 return NULL; 2772 } 2773 2774 /* Find open file states whose mode matches that of the range */ 2775 static bool 2776 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2777 const struct pnfs_layout_range *range) 2778 { 2779 struct list_head *head; 2780 struct nfs_open_context *ctx; 2781 fmode_t mode = 0; 2782 2783 if (!pnfs_layout_can_be_returned(lo) || 2784 !pnfs_find_first_lseg(lo, range, range->iomode)) 2785 return false; 2786 2787 head = &NFS_I(lo->plh_inode)->open_files; 2788 list_for_each_entry_rcu(ctx, head, list) { 2789 if (ctx->state) 2790 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2791 } 2792 2793 switch (range->iomode) { 2794 default: 2795 break; 2796 case IOMODE_READ: 2797 mode &= ~FMODE_WRITE; 2798 break; 2799 case IOMODE_RW: 2800 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2801 mode &= ~FMODE_READ; 2802 } 2803 return mode == 0; 2804 } 2805 2806 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2807 void *data) 2808 { 2809 const struct pnfs_layout_range *range = data; 2810 const struct cred *cred; 2811 struct pnfs_layout_hdr *lo; 2812 struct inode *inode; 2813 nfs4_stateid stateid; 2814 enum pnfs_iomode iomode; 2815 2816 restart: 2817 rcu_read_lock(); 2818 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2819 inode = lo->plh_inode; 2820 if (!inode || !pnfs_layout_can_be_returned(lo) || 2821 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2822 continue; 2823 spin_lock(&inode->i_lock); 2824 if (!lo->plh_inode || 2825 !pnfs_should_return_unused_layout(lo, range)) { 2826 spin_unlock(&inode->i_lock); 2827 continue; 2828 } 2829 pnfs_get_layout_hdr(lo); 2830 pnfs_set_plh_return_info(lo, range->iomode, 0); 2831 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2832 range, 0) != 0 || 2833 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2834 spin_unlock(&inode->i_lock); 2835 rcu_read_unlock(); 2836 pnfs_put_layout_hdr(lo); 2837 cond_resched(); 2838 goto restart; 2839 } 2840 spin_unlock(&inode->i_lock); 2841 rcu_read_unlock(); 2842 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2843 PNFS_FL_LAYOUTRETURN_ASYNC); 2844 pnfs_put_layout_hdr(lo); 2845 cond_resched(); 2846 goto restart; 2847 } 2848 rcu_read_unlock(); 2849 return 0; 2850 } 2851 2852 void 2853 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2854 enum pnfs_iomode iomode) 2855 { 2856 struct pnfs_layout_range range = { 2857 .iomode = iomode, 2858 .offset = 0, 2859 .length = NFS4_MAX_UINT64, 2860 }; 2861 2862 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2863 &range); 2864 } 2865 2866 /* Check if we have we have a valid layout but if there isn't an intersection 2867 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 2868 */ 2869 void 2870 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 2871 struct nfs_page *req) 2872 { 2873 if (pgio->pg_lseg == NULL || 2874 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 2875 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 2876 return; 2877 pnfs_put_lseg(pgio->pg_lseg); 2878 pgio->pg_lseg = NULL; 2879 } 2880 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2881 2882 void 2883 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2884 { 2885 u64 rd_size; 2886 2887 pnfs_generic_pg_check_layout(pgio, req); 2888 if (pgio->pg_lseg == NULL) { 2889 if (pgio->pg_dreq == NULL) 2890 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2891 else 2892 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 2893 req_offset(req)); 2894 2895 pgio->pg_lseg = 2896 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2897 req_offset(req), rd_size, 2898 IOMODE_READ, false, 2899 nfs_io_gfp_mask()); 2900 if (IS_ERR(pgio->pg_lseg)) { 2901 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2902 pgio->pg_lseg = NULL; 2903 return; 2904 } 2905 } 2906 /* If no lseg, fall back to read through mds */ 2907 if (pgio->pg_lseg == NULL) 2908 nfs_pageio_reset_read_mds(pgio); 2909 2910 } 2911 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2912 2913 void 2914 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2915 struct nfs_page *req, u64 wb_size) 2916 { 2917 pnfs_generic_pg_check_layout(pgio, req); 2918 if (pgio->pg_lseg == NULL) { 2919 pgio->pg_lseg = 2920 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2921 req_offset(req), wb_size, IOMODE_RW, 2922 false, nfs_io_gfp_mask()); 2923 if (IS_ERR(pgio->pg_lseg)) { 2924 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2925 pgio->pg_lseg = NULL; 2926 return; 2927 } 2928 } 2929 /* If no lseg, fall back to write through mds */ 2930 if (pgio->pg_lseg == NULL) 2931 nfs_pageio_reset_write_mds(pgio); 2932 } 2933 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2934 2935 void 2936 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2937 { 2938 if (desc->pg_lseg) { 2939 pnfs_put_lseg(desc->pg_lseg); 2940 desc->pg_lseg = NULL; 2941 } 2942 } 2943 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2944 2945 /* 2946 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2947 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2948 */ 2949 size_t 2950 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2951 struct nfs_page *prev, struct nfs_page *req) 2952 { 2953 unsigned int size; 2954 u64 seg_end, req_start, seg_left; 2955 2956 size = nfs_generic_pg_test(pgio, prev, req); 2957 if (!size) 2958 return 0; 2959 2960 /* 2961 * 'size' contains the number of bytes left in the current page (up 2962 * to the original size asked for in @req->wb_bytes). 2963 * 2964 * Calculate how many bytes are left in the layout segment 2965 * and if there are less bytes than 'size', return that instead. 2966 * 2967 * Please also note that 'end_offset' is actually the offset of the 2968 * first byte that lies outside the pnfs_layout_range. FIXME? 2969 * 2970 */ 2971 if (pgio->pg_lseg) { 2972 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2973 pgio->pg_lseg->pls_range.length); 2974 req_start = req_offset(req); 2975 2976 /* start of request is past the last byte of this segment */ 2977 if (req_start >= seg_end) 2978 return 0; 2979 2980 /* adjust 'size' iff there are fewer bytes left in the 2981 * segment than what nfs_generic_pg_test returned */ 2982 seg_left = seg_end - req_start; 2983 if (seg_left < size) 2984 size = (unsigned int)seg_left; 2985 } 2986 2987 return size; 2988 } 2989 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2990 2991 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2992 { 2993 struct nfs_pageio_descriptor pgio; 2994 2995 /* Resend all requests through the MDS */ 2996 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2997 hdr->completion_ops); 2998 return nfs_pageio_resend(&pgio, hdr); 2999 } 3000 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 3001 3002 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 3003 { 3004 3005 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 3006 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3007 PNFS_LAYOUTRET_ON_ERROR) { 3008 pnfs_return_layout(hdr->inode); 3009 } 3010 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3011 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 3012 } 3013 3014 /* 3015 * Called by non rpc-based layout drivers 3016 */ 3017 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 3018 { 3019 if (likely(!hdr->pnfs_error)) { 3020 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 3021 hdr->mds_offset + hdr->res.count); 3022 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3023 } 3024 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3025 if (unlikely(hdr->pnfs_error)) 3026 pnfs_ld_handle_write_error(hdr); 3027 hdr->mds_ops->rpc_release(hdr); 3028 } 3029 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3030 3031 static void 3032 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3033 struct nfs_pgio_header *hdr) 3034 { 3035 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3036 3037 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3038 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3039 nfs_pageio_reset_write_mds(desc); 3040 mirror->pg_recoalesce = 1; 3041 } 3042 hdr->completion_ops->completion(hdr); 3043 } 3044 3045 static enum pnfs_try_status 3046 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3047 const struct rpc_call_ops *call_ops, 3048 struct pnfs_layout_segment *lseg, 3049 int how) 3050 { 3051 struct inode *inode = hdr->inode; 3052 enum pnfs_try_status trypnfs; 3053 struct nfs_server *nfss = NFS_SERVER(inode); 3054 3055 hdr->mds_ops = call_ops; 3056 3057 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 3058 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3059 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3060 if (trypnfs != PNFS_NOT_ATTEMPTED) 3061 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3062 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3063 return trypnfs; 3064 } 3065 3066 static void 3067 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3068 struct nfs_pgio_header *hdr, int how) 3069 { 3070 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3071 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3072 enum pnfs_try_status trypnfs; 3073 3074 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3075 switch (trypnfs) { 3076 case PNFS_NOT_ATTEMPTED: 3077 pnfs_write_through_mds(desc, hdr); 3078 break; 3079 case PNFS_ATTEMPTED: 3080 break; 3081 case PNFS_TRY_AGAIN: 3082 /* cleanup hdr and prepare to redo pnfs */ 3083 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3084 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3085 list_splice_init(&hdr->pages, &mirror->pg_list); 3086 mirror->pg_recoalesce = 1; 3087 } 3088 hdr->mds_ops->rpc_release(hdr); 3089 } 3090 } 3091 3092 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3093 { 3094 pnfs_put_lseg(hdr->lseg); 3095 nfs_pgio_header_free(hdr); 3096 } 3097 3098 int 3099 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3100 { 3101 struct nfs_pgio_header *hdr; 3102 int ret; 3103 3104 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3105 if (!hdr) { 3106 desc->pg_error = -ENOMEM; 3107 return desc->pg_error; 3108 } 3109 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3110 3111 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3112 ret = nfs_generic_pgio(desc, hdr); 3113 if (!ret) 3114 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3115 3116 return ret; 3117 } 3118 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3119 3120 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3121 { 3122 struct nfs_pageio_descriptor pgio; 3123 3124 /* Resend all requests through the MDS */ 3125 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3126 return nfs_pageio_resend(&pgio, hdr); 3127 } 3128 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3129 3130 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3131 { 3132 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3133 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3134 PNFS_LAYOUTRET_ON_ERROR) { 3135 pnfs_return_layout(hdr->inode); 3136 } 3137 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3138 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3139 } 3140 3141 /* 3142 * Called by non rpc-based layout drivers 3143 */ 3144 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3145 { 3146 if (likely(!hdr->pnfs_error)) 3147 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3148 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3149 if (unlikely(hdr->pnfs_error)) 3150 pnfs_ld_handle_read_error(hdr); 3151 hdr->mds_ops->rpc_release(hdr); 3152 } 3153 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3154 3155 static void 3156 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3157 struct nfs_pgio_header *hdr) 3158 { 3159 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3160 3161 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3162 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3163 nfs_pageio_reset_read_mds(desc); 3164 mirror->pg_recoalesce = 1; 3165 } 3166 hdr->completion_ops->completion(hdr); 3167 } 3168 3169 /* 3170 * Call the appropriate parallel I/O subsystem read function. 3171 */ 3172 static enum pnfs_try_status 3173 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3174 const struct rpc_call_ops *call_ops, 3175 struct pnfs_layout_segment *lseg) 3176 { 3177 struct inode *inode = hdr->inode; 3178 struct nfs_server *nfss = NFS_SERVER(inode); 3179 enum pnfs_try_status trypnfs; 3180 3181 hdr->mds_ops = call_ops; 3182 3183 dprintk("%s: Reading ino:%lu %u@%llu\n", 3184 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3185 3186 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3187 if (trypnfs != PNFS_NOT_ATTEMPTED) 3188 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3189 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3190 return trypnfs; 3191 } 3192 3193 /* Resend all requests through pnfs. */ 3194 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3195 unsigned int mirror_idx) 3196 { 3197 struct nfs_pageio_descriptor pgio; 3198 3199 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3200 /* Prevent deadlocks with layoutreturn! */ 3201 pnfs_put_lseg(hdr->lseg); 3202 hdr->lseg = NULL; 3203 3204 nfs_pageio_init_read(&pgio, hdr->inode, false, 3205 hdr->completion_ops); 3206 pgio.pg_mirror_idx = mirror_idx; 3207 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3208 } 3209 } 3210 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3211 3212 static void 3213 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3214 { 3215 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3216 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3217 enum pnfs_try_status trypnfs; 3218 3219 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3220 switch (trypnfs) { 3221 case PNFS_NOT_ATTEMPTED: 3222 pnfs_read_through_mds(desc, hdr); 3223 break; 3224 case PNFS_ATTEMPTED: 3225 break; 3226 case PNFS_TRY_AGAIN: 3227 /* cleanup hdr and prepare to redo pnfs */ 3228 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3229 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3230 list_splice_init(&hdr->pages, &mirror->pg_list); 3231 mirror->pg_recoalesce = 1; 3232 } 3233 hdr->mds_ops->rpc_release(hdr); 3234 } 3235 } 3236 3237 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3238 { 3239 pnfs_put_lseg(hdr->lseg); 3240 nfs_pgio_header_free(hdr); 3241 } 3242 3243 int 3244 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3245 { 3246 struct nfs_pgio_header *hdr; 3247 int ret; 3248 3249 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3250 if (!hdr) { 3251 desc->pg_error = -ENOMEM; 3252 return desc->pg_error; 3253 } 3254 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3255 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3256 ret = nfs_generic_pgio(desc, hdr); 3257 if (!ret) 3258 pnfs_do_read(desc, hdr); 3259 return ret; 3260 } 3261 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3262 3263 static void pnfs_clear_layoutcommitting(struct inode *inode) 3264 { 3265 unsigned long *bitlock = &NFS_I(inode)->flags; 3266 3267 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 3268 smp_mb__after_atomic(); 3269 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 3270 } 3271 3272 /* 3273 * There can be multiple RW segments. 3274 */ 3275 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3276 { 3277 struct pnfs_layout_segment *lseg; 3278 3279 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3280 if (lseg->pls_range.iomode == IOMODE_RW && 3281 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3282 list_add(&lseg->pls_lc_list, listp); 3283 } 3284 } 3285 3286 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3287 { 3288 struct pnfs_layout_segment *lseg, *tmp; 3289 3290 /* Matched by references in pnfs_set_layoutcommit */ 3291 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3292 list_del_init(&lseg->pls_lc_list); 3293 pnfs_put_lseg(lseg); 3294 } 3295 3296 pnfs_clear_layoutcommitting(inode); 3297 } 3298 3299 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3300 { 3301 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3302 } 3303 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3304 3305 void 3306 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3307 loff_t end_pos) 3308 { 3309 struct nfs_inode *nfsi = NFS_I(inode); 3310 bool mark_as_dirty = false; 3311 3312 spin_lock(&inode->i_lock); 3313 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3314 nfsi->layout->plh_lwb = end_pos; 3315 mark_as_dirty = true; 3316 dprintk("%s: Set layoutcommit for inode %lu ", 3317 __func__, inode->i_ino); 3318 } else if (end_pos > nfsi->layout->plh_lwb) 3319 nfsi->layout->plh_lwb = end_pos; 3320 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3321 /* references matched in nfs4_layoutcommit_release */ 3322 pnfs_get_lseg(lseg); 3323 } 3324 spin_unlock(&inode->i_lock); 3325 dprintk("%s: lseg %p end_pos %llu\n", 3326 __func__, lseg, nfsi->layout->plh_lwb); 3327 3328 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3329 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3330 if (mark_as_dirty) 3331 mark_inode_dirty_sync(inode); 3332 } 3333 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3334 3335 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3336 { 3337 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3338 3339 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3340 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3341 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3342 } 3343 3344 /* 3345 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3346 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3347 * data to disk to allow the server to recover the data if it crashes. 3348 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3349 * is off, and a COMMIT is sent to a data server, or 3350 * if WRITEs to a data server return NFS_DATA_SYNC. 3351 */ 3352 int 3353 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3354 { 3355 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3356 struct nfs4_layoutcommit_data *data; 3357 struct nfs_inode *nfsi = NFS_I(inode); 3358 loff_t end_pos; 3359 int status; 3360 bool mark_as_dirty = false; 3361 3362 if (!pnfs_layoutcommit_outstanding(inode)) 3363 return 0; 3364 3365 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 3366 3367 status = -EAGAIN; 3368 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3369 if (!sync) 3370 goto out; 3371 status = wait_on_bit_lock_action(&nfsi->flags, 3372 NFS_INO_LAYOUTCOMMITTING, 3373 nfs_wait_bit_killable, 3374 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3375 if (status) 3376 goto out; 3377 } 3378 3379 status = -ENOMEM; 3380 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3381 data = kzalloc(sizeof(*data), nfs_io_gfp_mask()); 3382 if (!data) 3383 goto clear_layoutcommitting; 3384 3385 status = 0; 3386 spin_lock(&inode->i_lock); 3387 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3388 goto out_unlock; 3389 3390 INIT_LIST_HEAD(&data->lseg_list); 3391 pnfs_list_write_lseg(inode, &data->lseg_list); 3392 3393 end_pos = nfsi->layout->plh_lwb; 3394 3395 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3396 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3397 spin_unlock(&inode->i_lock); 3398 3399 data->args.inode = inode; 3400 nfs_fattr_init(&data->fattr); 3401 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3402 data->res.fattr = &data->fattr; 3403 if (end_pos != 0) 3404 data->args.lastbytewritten = end_pos - 1; 3405 else 3406 data->args.lastbytewritten = U64_MAX; 3407 data->res.server = NFS_SERVER(inode); 3408 3409 if (ld->prepare_layoutcommit) { 3410 status = ld->prepare_layoutcommit(&data->args); 3411 if (status) { 3412 if (status != -ENOSPC) 3413 put_cred(data->cred); 3414 spin_lock(&inode->i_lock); 3415 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3416 if (end_pos > nfsi->layout->plh_lwb) 3417 nfsi->layout->plh_lwb = end_pos; 3418 if (status != -ENOSPC) 3419 goto out_unlock; 3420 spin_unlock(&inode->i_lock); 3421 mark_as_dirty = true; 3422 } 3423 } 3424 3425 3426 status = nfs4_proc_layoutcommit(data, sync); 3427 out: 3428 if (status || mark_as_dirty) 3429 mark_inode_dirty_sync(inode); 3430 dprintk("<-- %s status %d\n", __func__, status); 3431 return status; 3432 out_unlock: 3433 spin_unlock(&inode->i_lock); 3434 kfree(data); 3435 clear_layoutcommitting: 3436 pnfs_clear_layoutcommitting(inode); 3437 goto out; 3438 } 3439 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3440 3441 int 3442 pnfs_generic_sync(struct inode *inode, bool datasync) 3443 { 3444 return pnfs_layoutcommit_inode(inode, true); 3445 } 3446 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3447 3448 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3449 { 3450 struct nfs4_threshold *thp; 3451 3452 thp = kzalloc(sizeof(*thp), nfs_io_gfp_mask()); 3453 if (!thp) { 3454 dprintk("%s mdsthreshold allocation failed\n", __func__); 3455 return NULL; 3456 } 3457 return thp; 3458 } 3459 3460 #if IS_ENABLED(CONFIG_NFS_V4_2) 3461 int 3462 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3463 { 3464 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3465 struct nfs_server *server = NFS_SERVER(inode); 3466 struct nfs_inode *nfsi = NFS_I(inode); 3467 struct nfs42_layoutstat_data *data; 3468 struct pnfs_layout_hdr *hdr; 3469 int status = 0; 3470 3471 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3472 goto out; 3473 3474 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3475 goto out; 3476 3477 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3478 goto out; 3479 3480 spin_lock(&inode->i_lock); 3481 if (!NFS_I(inode)->layout) { 3482 spin_unlock(&inode->i_lock); 3483 goto out_clear_layoutstats; 3484 } 3485 hdr = NFS_I(inode)->layout; 3486 pnfs_get_layout_hdr(hdr); 3487 spin_unlock(&inode->i_lock); 3488 3489 data = kzalloc(sizeof(*data), gfp_flags); 3490 if (!data) { 3491 status = -ENOMEM; 3492 goto out_put; 3493 } 3494 3495 data->args.fh = NFS_FH(inode); 3496 data->args.inode = inode; 3497 status = ld->prepare_layoutstats(&data->args); 3498 if (status) 3499 goto out_free; 3500 3501 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3502 3503 out: 3504 dprintk("%s returns %d\n", __func__, status); 3505 return status; 3506 3507 out_free: 3508 kfree(data); 3509 out_put: 3510 pnfs_put_layout_hdr(hdr); 3511 out_clear_layoutstats: 3512 smp_mb__before_atomic(); 3513 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3514 smp_mb__after_atomic(); 3515 goto out; 3516 } 3517 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3518 #endif 3519 3520 unsigned int layoutstats_timer; 3521 module_param(layoutstats_timer, uint, 0644); 3522 EXPORT_SYMBOL_GPL(layoutstats_timer); 3523