1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 436 if (err != -EBUSY) { 437 dst->seqid = lo->plh_stateid.seqid; 438 *dst_range = range; 439 ret = true; 440 } 441 } 442 out: 443 spin_unlock(&inode->i_lock); 444 pnfs_free_lseg_list(&head); 445 return ret; 446 } 447 448 /* 449 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 450 * 451 * In order to continue using the pnfs_layout_hdr, a full recovery 452 * is required. 453 * Note that caller must hold inode->i_lock. 454 */ 455 int 456 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 457 struct list_head *lseg_list) 458 { 459 struct pnfs_layout_range range = { 460 .iomode = IOMODE_ANY, 461 .offset = 0, 462 .length = NFS4_MAX_UINT64, 463 }; 464 struct pnfs_layout_segment *lseg, *next; 465 466 if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 467 return !list_empty(&lo->plh_segs); 468 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); 469 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 470 pnfs_clear_lseg_state(lseg, lseg_list); 471 pnfs_clear_layoutreturn_info(lo); 472 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 473 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 474 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 475 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 476 pnfs_clear_layoutreturn_waitbit(lo); 477 return !list_empty(&lo->plh_segs); 478 } 479 480 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 481 struct list_head *lseg_list, 482 enum pnfs_iomode iomode, u32 seq) 483 { 484 struct pnfs_layout_range range = { 485 .iomode = iomode, 486 .length = NFS4_MAX_UINT64, 487 }; 488 489 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); 490 } 491 492 static int 493 pnfs_iomode_to_fail_bit(u32 iomode) 494 { 495 return iomode == IOMODE_RW ? 496 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 497 } 498 499 static void 500 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 501 { 502 lo->plh_retry_timestamp = jiffies; 503 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 504 refcount_inc(&lo->plh_refcount); 505 } 506 507 static void 508 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 509 { 510 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 511 refcount_dec(&lo->plh_refcount); 512 } 513 514 static void 515 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 516 { 517 struct inode *inode = lo->plh_inode; 518 struct pnfs_layout_range range = { 519 .iomode = iomode, 520 .offset = 0, 521 .length = NFS4_MAX_UINT64, 522 }; 523 LIST_HEAD(head); 524 525 spin_lock(&inode->i_lock); 526 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 527 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 528 spin_unlock(&inode->i_lock); 529 pnfs_free_lseg_list(&head); 530 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 531 iomode == IOMODE_RW ? "RW" : "READ"); 532 } 533 534 static bool 535 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 536 { 537 unsigned long start, end; 538 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 539 540 if (test_bit(fail_bit, &lo->plh_flags) == 0) 541 return false; 542 end = jiffies; 543 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 544 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 545 /* It is time to retry the failed layoutgets */ 546 pnfs_layout_clear_fail_bit(lo, fail_bit); 547 return false; 548 } 549 return true; 550 } 551 552 static void 553 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 554 const struct pnfs_layout_range *range, 555 const nfs4_stateid *stateid) 556 { 557 INIT_LIST_HEAD(&lseg->pls_list); 558 INIT_LIST_HEAD(&lseg->pls_lc_list); 559 INIT_LIST_HEAD(&lseg->pls_commits); 560 refcount_set(&lseg->pls_refcount, 1); 561 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 562 lseg->pls_layout = lo; 563 lseg->pls_range = *range; 564 lseg->pls_seq = be32_to_cpu(stateid->seqid); 565 } 566 567 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 568 { 569 if (lseg != NULL) { 570 struct inode *inode = lseg->pls_layout->plh_inode; 571 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 572 } 573 } 574 575 static void 576 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 577 struct pnfs_layout_segment *lseg) 578 { 579 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 580 list_del_init(&lseg->pls_list); 581 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 582 refcount_dec(&lo->plh_refcount); 583 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 584 return; 585 if (list_empty(&lo->plh_segs) && 586 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 587 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 588 if (atomic_read(&lo->plh_outstanding) == 0) 589 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 590 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 591 } 592 } 593 594 static bool 595 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 596 struct pnfs_layout_segment *lseg) 597 { 598 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 599 pnfs_layout_is_valid(lo)) { 600 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 601 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 602 return true; 603 } 604 return false; 605 } 606 607 void 608 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 609 { 610 struct pnfs_layout_hdr *lo; 611 struct inode *inode; 612 613 if (!lseg) 614 return; 615 616 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 617 refcount_read(&lseg->pls_refcount), 618 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 619 620 lo = lseg->pls_layout; 621 inode = lo->plh_inode; 622 623 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 624 pnfs_get_layout_hdr(lo); 625 pnfs_layout_remove_lseg(lo, lseg); 626 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 627 lseg = NULL; 628 spin_unlock(&inode->i_lock); 629 pnfs_free_lseg(lseg); 630 pnfs_put_layout_hdr(lo); 631 } 632 } 633 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 634 635 /* 636 * is l2 fully contained in l1? 637 * start1 end1 638 * [----------------------------------) 639 * start2 end2 640 * [----------------) 641 */ 642 static bool 643 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 644 const struct pnfs_layout_range *l2) 645 { 646 u64 start1 = l1->offset; 647 u64 end1 = pnfs_end_offset(start1, l1->length); 648 u64 start2 = l2->offset; 649 u64 end2 = pnfs_end_offset(start2, l2->length); 650 651 return (start1 <= start2) && (end1 >= end2); 652 } 653 654 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 655 struct list_head *tmp_list) 656 { 657 if (!refcount_dec_and_test(&lseg->pls_refcount)) 658 return false; 659 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 660 list_add(&lseg->pls_list, tmp_list); 661 return true; 662 } 663 664 /* Returns 1 if lseg is removed from list, 0 otherwise */ 665 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 666 struct list_head *tmp_list) 667 { 668 int rv = 0; 669 670 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 671 /* Remove the reference keeping the lseg in the 672 * list. It will now be removed when all 673 * outstanding io is finished. 674 */ 675 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 676 refcount_read(&lseg->pls_refcount)); 677 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 678 rv = 1; 679 } 680 return rv; 681 } 682 683 static bool 684 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 685 const struct pnfs_layout_range *recall_range) 686 { 687 return (recall_range->iomode == IOMODE_ANY || 688 lseg_range->iomode == recall_range->iomode) && 689 pnfs_lseg_range_intersecting(lseg_range, recall_range); 690 } 691 692 static bool 693 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 694 const struct pnfs_layout_range *recall_range, 695 u32 seq) 696 { 697 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 698 return false; 699 if (recall_range == NULL) 700 return true; 701 return pnfs_should_free_range(&lseg->pls_range, recall_range); 702 } 703 704 /** 705 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 706 * @lo: layout header containing the lsegs 707 * @tmp_list: list head where doomed lsegs should go 708 * @recall_range: optional recall range argument to match (may be NULL) 709 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 710 * 711 * Walk the list of lsegs in the layout header, and tear down any that should 712 * be destroyed. If "recall_range" is specified then the segment must match 713 * that range. If "seq" is non-zero, then only match segments that were handed 714 * out at or before that sequence. 715 * 716 * Returns number of matching invalid lsegs remaining in list after scanning 717 * it and purging them. 718 */ 719 int 720 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 721 struct list_head *tmp_list, 722 const struct pnfs_layout_range *recall_range, 723 u32 seq) 724 { 725 struct pnfs_layout_segment *lseg, *next; 726 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 727 int remaining = 0; 728 729 dprintk("%s:Begin lo %p\n", __func__, lo); 730 731 if (list_empty(&lo->plh_segs)) 732 return 0; 733 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 734 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 735 dprintk("%s: freeing lseg %p iomode %d seq %u " 736 "offset %llu length %llu\n", __func__, 737 lseg, lseg->pls_range.iomode, lseg->pls_seq, 738 lseg->pls_range.offset, lseg->pls_range.length); 739 if (mark_lseg_invalid(lseg, tmp_list)) 740 continue; 741 remaining++; 742 pnfs_lseg_cancel_io(server, lseg); 743 } 744 dprintk("%s:Return %i\n", __func__, remaining); 745 return remaining; 746 } 747 748 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 749 { 750 struct pnfs_layout_segment *lseg; 751 752 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 753 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 754 } 755 756 static void 757 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 758 struct list_head *free_me, 759 const struct pnfs_layout_range *range, 760 u32 seq) 761 { 762 struct pnfs_layout_segment *lseg, *next; 763 764 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 765 if (pnfs_match_lseg_recall(lseg, range, seq)) 766 list_move_tail(&lseg->pls_list, free_me); 767 } 768 } 769 770 /* note free_me must contain lsegs from a single layout_hdr */ 771 void 772 pnfs_free_lseg_list(struct list_head *free_me) 773 { 774 struct pnfs_layout_segment *lseg, *tmp; 775 776 if (list_empty(free_me)) 777 return; 778 779 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 780 list_del(&lseg->pls_list); 781 pnfs_free_lseg(lseg); 782 } 783 } 784 785 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 786 { 787 struct pnfs_layout_hdr *lo; 788 LIST_HEAD(tmp_list); 789 790 spin_lock(&nfsi->vfs_inode.i_lock); 791 lo = nfsi->layout; 792 if (lo) { 793 pnfs_get_layout_hdr(lo); 794 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 795 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 796 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 797 spin_unlock(&nfsi->vfs_inode.i_lock); 798 pnfs_free_lseg_list(&tmp_list); 799 nfs_commit_inode(&nfsi->vfs_inode, 0); 800 pnfs_put_layout_hdr(lo); 801 } else 802 spin_unlock(&nfsi->vfs_inode.i_lock); 803 return lo; 804 } 805 806 void pnfs_destroy_layout(struct nfs_inode *nfsi) 807 { 808 __pnfs_destroy_layout(nfsi); 809 } 810 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 811 812 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 813 { 814 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 815 struct inode *inode = &nfsi->vfs_inode; 816 817 if (lo) { 818 spin_lock(&inode->i_lock); 819 wait_var_event_spinlock(lo, nfsi->layout != lo, 820 &inode->i_lock); 821 spin_unlock(&inode->i_lock); 822 } 823 } 824 825 static bool 826 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 827 struct list_head *layout_list) 828 { 829 struct pnfs_layout_hdr *lo; 830 bool ret = false; 831 832 spin_lock(&inode->i_lock); 833 lo = NFS_I(inode)->layout; 834 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 835 pnfs_get_layout_hdr(lo); 836 list_add(&lo->plh_bulk_destroy, layout_list); 837 ret = true; 838 } 839 spin_unlock(&inode->i_lock); 840 return ret; 841 } 842 843 /* Caller must hold rcu_read_lock and clp->cl_lock */ 844 static int 845 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 846 struct nfs_server *server, 847 struct list_head *layout_list) 848 __must_hold(&clp->cl_lock) 849 __must_hold(RCU) 850 { 851 struct pnfs_layout_hdr *lo, *next; 852 struct inode *inode; 853 854 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 855 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 856 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 857 !list_empty(&lo->plh_bulk_destroy)) 858 continue; 859 /* If the sb is being destroyed, just bail */ 860 if (!nfs_sb_active(server->super)) 861 break; 862 inode = pnfs_grab_inode_layout_hdr(lo); 863 if (inode != NULL) { 864 if (pnfs_layout_add_bulk_destroy_list(inode, 865 layout_list)) 866 continue; 867 rcu_read_unlock(); 868 spin_unlock(&clp->cl_lock); 869 iput(inode); 870 } else { 871 rcu_read_unlock(); 872 spin_unlock(&clp->cl_lock); 873 } 874 nfs_sb_deactive(server->super); 875 spin_lock(&clp->cl_lock); 876 rcu_read_lock(); 877 return -EAGAIN; 878 } 879 return 0; 880 } 881 882 static int 883 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 884 enum pnfs_layout_destroy_mode mode) 885 { 886 struct pnfs_layout_hdr *lo; 887 struct inode *inode; 888 LIST_HEAD(lseg_list); 889 int ret = 0; 890 891 while (!list_empty(layout_list)) { 892 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 893 plh_bulk_destroy); 894 dprintk("%s freeing layout for inode %lu\n", __func__, 895 lo->plh_inode->i_ino); 896 inode = lo->plh_inode; 897 898 pnfs_layoutcommit_inode(inode, false); 899 900 spin_lock(&inode->i_lock); 901 list_del_init(&lo->plh_bulk_destroy); 902 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 903 pnfs_mark_layout_stateid_return(lo, &lseg_list, 904 IOMODE_ANY, 0); 905 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 906 if (mode == PNFS_LAYOUT_BULK_RETURN) 907 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 908 ret = -EAGAIN; 909 } 910 spin_unlock(&inode->i_lock); 911 pnfs_free_lseg_list(&lseg_list); 912 /* Free all lsegs that are attached to commit buckets */ 913 nfs_commit_inode(inode, 0); 914 pnfs_put_layout_hdr(lo); 915 nfs_iput_and_deactive(inode); 916 } 917 return ret; 918 } 919 920 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 921 enum pnfs_layout_destroy_mode mode) 922 { 923 struct nfs_server *server; 924 LIST_HEAD(layout_list); 925 926 spin_lock(&clp->cl_lock); 927 rcu_read_lock(); 928 restart: 929 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 930 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 931 continue; 932 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 933 server, 934 &layout_list) != 0) 935 goto restart; 936 } 937 rcu_read_unlock(); 938 spin_unlock(&clp->cl_lock); 939 940 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 941 } 942 943 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 944 struct list_head *list) 945 { 946 struct nfs_server *server; 947 948 spin_lock(&clp->cl_lock); 949 rcu_read_lock(); 950 restart: 951 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 952 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 953 list) != 0) 954 goto restart; 955 } 956 rcu_read_unlock(); 957 spin_unlock(&clp->cl_lock); 958 } 959 960 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 961 struct list_head *list, 962 enum pnfs_layout_destroy_mode mode) 963 { 964 pnfs_layout_build_destroy_list_byclient(clp, list); 965 return pnfs_layout_free_bulk_destroy_list(list, mode); 966 } 967 968 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 969 enum pnfs_layout_destroy_mode mode) 970 { 971 LIST_HEAD(layout_list); 972 973 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 974 } 975 976 /* 977 * Called by the state manager to remove all layouts established under an 978 * expired lease. 979 */ 980 void 981 pnfs_destroy_all_layouts(struct nfs_client *clp) 982 { 983 nfs4_deviceid_mark_client_invalid(clp); 984 nfs4_deviceid_purge_client(clp); 985 986 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 987 } 988 989 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 990 struct list_head *list) 991 { 992 struct nfs_server *server; 993 994 spin_lock(&clp->cl_lock); 995 rcu_read_lock(); 996 restart: 997 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 998 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 999 continue; 1000 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 1001 list) != 0) 1002 goto restart; 1003 } 1004 rcu_read_unlock(); 1005 spin_unlock(&clp->cl_lock); 1006 } 1007 1008 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1009 { 1010 struct pnfs_layout_hdr *lo; 1011 struct nfs_server *server; 1012 int ret; 1013 1014 list_for_each_entry(lo, list, plh_bulk_destroy) { 1015 server = NFS_SERVER(lo->plh_inode); 1016 ret = pnfs_layout_return_on_reboot(lo); 1017 switch (ret) { 1018 case 0: 1019 continue; 1020 case -NFS4ERR_BAD_STATEID: 1021 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1022 break; 1023 case -NFS4ERR_NO_GRACE: 1024 break; 1025 default: 1026 goto err; 1027 } 1028 break; 1029 } 1030 return 0; 1031 err: 1032 return ret; 1033 } 1034 1035 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1036 { 1037 LIST_HEAD(list); 1038 int ret = 0, ret2; 1039 1040 pnfs_layout_build_recover_list_byclient(clp, &list); 1041 if (!list_empty(&list)) 1042 ret = pnfs_layout_bulk_list_reboot(&list); 1043 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1044 PNFS_LAYOUT_INVALIDATE); 1045 if (!ret) 1046 ret = ret2; 1047 return (ret == 0) ? 0 : -EAGAIN; 1048 } 1049 1050 static void 1051 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1052 { 1053 const struct cred *old; 1054 1055 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1056 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1057 put_cred(old); 1058 } 1059 } 1060 1061 /* update lo->plh_stateid with new if is more recent */ 1062 void 1063 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1064 const struct cred *cred, bool update_barrier) 1065 { 1066 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1067 u32 newseq = be32_to_cpu(new->seqid); 1068 1069 if (!pnfs_layout_is_valid(lo)) { 1070 pnfs_set_layout_cred(lo, cred); 1071 nfs4_stateid_copy(&lo->plh_stateid, new); 1072 lo->plh_barrier = newseq; 1073 pnfs_clear_layoutreturn_info(lo); 1074 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1075 return; 1076 } 1077 1078 if (pnfs_seqid_is_newer(newseq, oldseq)) 1079 nfs4_stateid_copy(&lo->plh_stateid, new); 1080 1081 if (update_barrier) { 1082 pnfs_barrier_update(lo, newseq); 1083 return; 1084 } 1085 /* 1086 * Because of wraparound, we want to keep the barrier 1087 * "close" to the current seqids. We really only want to 1088 * get here from a layoutget call. 1089 */ 1090 if (atomic_read(&lo->plh_outstanding) == 1) 1091 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1092 } 1093 1094 static bool 1095 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1096 const nfs4_stateid *stateid) 1097 { 1098 u32 seqid = be32_to_cpu(stateid->seqid); 1099 1100 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1101 } 1102 1103 /* lget is set to 1 if called from inside send_layoutget call chain */ 1104 static bool 1105 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1106 { 1107 return lo->plh_block_lgets || 1108 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1109 } 1110 1111 static struct nfs_server * 1112 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1113 { 1114 struct nfs_server *server; 1115 1116 if (inode) { 1117 server = NFS_SERVER(inode); 1118 } else { 1119 struct dentry *parent_dir = dget_parent(ctx->dentry); 1120 server = NFS_SERVER(parent_dir->d_inode); 1121 dput(parent_dir); 1122 } 1123 return server; 1124 } 1125 1126 static void nfs4_free_pages(struct page **pages, size_t size) 1127 { 1128 int i; 1129 1130 if (!pages) 1131 return; 1132 1133 for (i = 0; i < size; i++) { 1134 if (!pages[i]) 1135 break; 1136 __free_page(pages[i]); 1137 } 1138 kfree(pages); 1139 } 1140 1141 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1142 { 1143 struct page **pages; 1144 int i; 1145 1146 pages = kmalloc_array(size, sizeof(struct page *), gfp_flags); 1147 if (!pages) { 1148 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1149 return NULL; 1150 } 1151 1152 for (i = 0; i < size; i++) { 1153 pages[i] = alloc_page(gfp_flags); 1154 if (!pages[i]) { 1155 dprintk("%s: failed to allocate page\n", __func__); 1156 nfs4_free_pages(pages, i); 1157 return NULL; 1158 } 1159 } 1160 1161 return pages; 1162 } 1163 1164 static struct nfs4_layoutget * 1165 pnfs_alloc_init_layoutget_args(struct inode *ino, 1166 struct nfs_open_context *ctx, 1167 const nfs4_stateid *stateid, 1168 const struct pnfs_layout_range *range, 1169 gfp_t gfp_flags) 1170 { 1171 struct nfs_server *server = pnfs_find_server(ino, ctx); 1172 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1173 size_t max_pages = max_response_pages(server); 1174 struct nfs4_layoutget *lgp; 1175 1176 dprintk("--> %s\n", __func__); 1177 1178 lgp = kzalloc(sizeof(*lgp), gfp_flags); 1179 if (lgp == NULL) 1180 return NULL; 1181 1182 if (max_reply_sz) { 1183 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1184 if (npages < max_pages) 1185 max_pages = npages; 1186 } 1187 1188 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1189 if (!lgp->args.layout.pages) { 1190 kfree(lgp); 1191 return NULL; 1192 } 1193 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1194 lgp->res.layoutp = &lgp->args.layout; 1195 1196 /* Don't confuse uninitialised result and success */ 1197 lgp->res.status = -NFS4ERR_DELAY; 1198 1199 lgp->args.minlength = PAGE_SIZE; 1200 if (lgp->args.minlength > range->length) 1201 lgp->args.minlength = range->length; 1202 if (ino) { 1203 loff_t i_size = i_size_read(ino); 1204 1205 if (range->iomode == IOMODE_READ) { 1206 if (range->offset >= i_size) 1207 lgp->args.minlength = 0; 1208 else if (i_size - range->offset < lgp->args.minlength) 1209 lgp->args.minlength = i_size - range->offset; 1210 } 1211 } 1212 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 1213 pnfs_copy_range(&lgp->args.range, range); 1214 lgp->args.type = server->pnfs_curr_ld->id; 1215 lgp->args.inode = ino; 1216 lgp->args.ctx = get_nfs_open_context(ctx); 1217 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1218 lgp->gfp_flags = gfp_flags; 1219 lgp->cred = ctx->cred; 1220 return lgp; 1221 } 1222 1223 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1224 { 1225 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1226 1227 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1228 pnfs_put_layout_hdr(lgp->lo); 1229 put_nfs_open_context(lgp->args.ctx); 1230 kfree(lgp); 1231 } 1232 1233 static void pnfs_clear_layoutcommit(struct inode *inode, 1234 struct list_head *head) 1235 { 1236 struct nfs_inode *nfsi = NFS_I(inode); 1237 struct pnfs_layout_segment *lseg, *tmp; 1238 1239 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1240 return; 1241 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1242 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1243 continue; 1244 pnfs_lseg_dec_and_remove_zero(lseg, head); 1245 } 1246 } 1247 1248 static void 1249 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1250 const nfs4_stateid *arg_stateid, 1251 const struct pnfs_layout_range *range, 1252 struct list_head *freeme) 1253 { 1254 if (pnfs_layout_is_valid(lo) && 1255 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1256 pnfs_reset_return_info(lo); 1257 else 1258 pnfs_mark_layout_stateid_invalid(lo, freeme); 1259 pnfs_clear_layoutreturn_waitbit(lo); 1260 } 1261 1262 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1263 const nfs4_stateid *arg_stateid, 1264 const struct pnfs_layout_range *range) 1265 { 1266 struct inode *inode = lo->plh_inode; 1267 LIST_HEAD(freeme); 1268 1269 spin_lock(&inode->i_lock); 1270 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1271 spin_unlock(&inode->i_lock); 1272 pnfs_free_lseg_list(&freeme); 1273 } 1274 1275 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1276 const nfs4_stateid *arg_stateid, 1277 const struct pnfs_layout_range *range, 1278 const nfs4_stateid *stateid) 1279 { 1280 struct inode *inode = lo->plh_inode; 1281 LIST_HEAD(freeme); 1282 1283 spin_lock(&inode->i_lock); 1284 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1285 goto out_unlock; 1286 if (stateid && pnfs_layout_is_valid(lo)) { 1287 u32 seq = be32_to_cpu(arg_stateid->seqid); 1288 1289 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1290 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1291 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1292 pnfs_reset_return_info(lo); 1293 } else 1294 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1295 out_unlock: 1296 pnfs_clear_layoutreturn_waitbit(lo); 1297 spin_unlock(&inode->i_lock); 1298 pnfs_free_lseg_list(&freeme); 1299 1300 } 1301 1302 static bool 1303 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1304 nfs4_stateid *stateid, 1305 const struct cred **cred, 1306 enum pnfs_iomode *iomode) 1307 { 1308 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1309 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1310 return false; 1311 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1312 return false; 1313 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1314 pnfs_get_layout_hdr(lo); 1315 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1316 *cred = get_cred(lo->plh_lc_cred); 1317 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1318 if (lo->plh_return_seq != 0) 1319 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1320 if (iomode != NULL) 1321 *iomode = lo->plh_return_iomode; 1322 pnfs_clear_layoutreturn_info(lo); 1323 } else if (iomode != NULL) 1324 *iomode = IOMODE_ANY; 1325 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1326 return true; 1327 } 1328 1329 static void 1330 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1331 struct pnfs_layout_hdr *lo, 1332 const nfs4_stateid *stateid, 1333 enum pnfs_iomode iomode) 1334 { 1335 struct inode *inode = lo->plh_inode; 1336 1337 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1338 args->inode = inode; 1339 args->range.iomode = iomode; 1340 args->range.offset = 0; 1341 args->range.length = NFS4_MAX_UINT64; 1342 args->layout = lo; 1343 nfs4_stateid_copy(&args->stateid, stateid); 1344 } 1345 1346 static int 1347 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1348 const nfs4_stateid *stateid, 1349 const struct cred **pcred, 1350 enum pnfs_iomode iomode, 1351 unsigned int flags) 1352 { 1353 struct inode *ino = lo->plh_inode; 1354 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1355 struct nfs4_layoutreturn *lrp; 1356 const struct cred *cred = *pcred; 1357 int status = 0; 1358 1359 *pcred = NULL; 1360 lrp = kzalloc(sizeof(*lrp), nfs_io_gfp_mask()); 1361 if (unlikely(lrp == NULL)) { 1362 status = -ENOMEM; 1363 spin_lock(&ino->i_lock); 1364 pnfs_clear_layoutreturn_waitbit(lo); 1365 spin_unlock(&ino->i_lock); 1366 put_cred(cred); 1367 pnfs_put_layout_hdr(lo); 1368 goto out; 1369 } 1370 1371 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1372 lrp->args.ld_private = &lrp->ld_private; 1373 lrp->clp = NFS_SERVER(ino)->nfs_client; 1374 lrp->cred = cred; 1375 if (ld->prepare_layoutreturn) 1376 ld->prepare_layoutreturn(&lrp->args); 1377 1378 status = nfs4_proc_layoutreturn(lrp, flags); 1379 out: 1380 dprintk("<-- %s status: %d\n", __func__, status); 1381 return status; 1382 } 1383 1384 /* Return true if layoutreturn is needed */ 1385 static bool 1386 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1387 { 1388 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1389 return false; 1390 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1391 lo->plh_return_iomode, 1392 lo->plh_return_seq) != EBUSY; 1393 } 1394 1395 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1396 { 1397 struct inode *inode= lo->plh_inode; 1398 1399 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1400 return; 1401 spin_lock(&inode->i_lock); 1402 if (pnfs_layout_need_return(lo)) { 1403 const struct cred *cred; 1404 nfs4_stateid stateid; 1405 enum pnfs_iomode iomode; 1406 bool send; 1407 1408 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1409 spin_unlock(&inode->i_lock); 1410 if (send) { 1411 /* Send an async layoutreturn so we dont deadlock */ 1412 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1413 PNFS_FL_LAYOUTRETURN_ASYNC); 1414 } 1415 } else 1416 spin_unlock(&inode->i_lock); 1417 } 1418 1419 /* 1420 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1421 * when the layout segment list is empty. 1422 * 1423 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1424 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1425 * deviceid is marked invalid. 1426 */ 1427 int 1428 _pnfs_return_layout(struct inode *ino) 1429 { 1430 struct pnfs_layout_hdr *lo = NULL; 1431 struct nfs_inode *nfsi = NFS_I(ino); 1432 struct pnfs_layout_range range = { 1433 .iomode = IOMODE_ANY, 1434 .offset = 0, 1435 .length = NFS4_MAX_UINT64, 1436 }; 1437 LIST_HEAD(tmp_list); 1438 const struct cred *cred; 1439 nfs4_stateid stateid; 1440 int status = 0; 1441 bool send, valid_layout; 1442 1443 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1444 1445 spin_lock(&ino->i_lock); 1446 lo = nfsi->layout; 1447 if (!lo) { 1448 spin_unlock(&ino->i_lock); 1449 dprintk("NFS: %s no layout to return\n", __func__); 1450 goto out; 1451 } 1452 /* Reference matched in nfs4_layoutreturn_release */ 1453 pnfs_get_layout_hdr(lo); 1454 /* Is there an outstanding layoutreturn ? */ 1455 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1456 spin_unlock(&ino->i_lock); 1457 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1458 TASK_UNINTERRUPTIBLE)) 1459 goto out_put_layout_hdr; 1460 spin_lock(&ino->i_lock); 1461 } 1462 valid_layout = pnfs_layout_is_valid(lo); 1463 pnfs_clear_layoutcommit(ino, &tmp_list); 1464 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); 1465 1466 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1467 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1468 1469 /* Don't send a LAYOUTRETURN if list was initially empty */ 1470 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1471 !valid_layout) { 1472 spin_unlock(&ino->i_lock); 1473 dprintk("NFS: %s no layout segments to return\n", __func__); 1474 goto out_wait_layoutreturn; 1475 } 1476 1477 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1478 spin_unlock(&ino->i_lock); 1479 if (send) 1480 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1481 0); 1482 out_wait_layoutreturn: 1483 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1484 out_put_layout_hdr: 1485 pnfs_free_lseg_list(&tmp_list); 1486 pnfs_put_layout_hdr(lo); 1487 out: 1488 dprintk("<-- %s status: %d\n", __func__, status); 1489 return status; 1490 } 1491 1492 int 1493 pnfs_commit_and_return_layout(struct inode *inode) 1494 { 1495 struct pnfs_layout_hdr *lo; 1496 int ret; 1497 1498 spin_lock(&inode->i_lock); 1499 lo = NFS_I(inode)->layout; 1500 if (lo == NULL) { 1501 spin_unlock(&inode->i_lock); 1502 return 0; 1503 } 1504 pnfs_get_layout_hdr(lo); 1505 /* Block new layoutgets and read/write to ds */ 1506 lo->plh_block_lgets++; 1507 spin_unlock(&inode->i_lock); 1508 filemap_fdatawait(inode->i_mapping); 1509 ret = pnfs_layoutcommit_inode(inode, true); 1510 if (ret == 0) 1511 ret = _pnfs_return_layout(inode); 1512 spin_lock(&inode->i_lock); 1513 lo->plh_block_lgets--; 1514 spin_unlock(&inode->i_lock); 1515 pnfs_put_layout_hdr(lo); 1516 return ret; 1517 } 1518 1519 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1520 { 1521 struct inode *inode = lo->plh_inode; 1522 const struct cred *cred; 1523 1524 spin_lock(&inode->i_lock); 1525 if (!pnfs_layout_is_valid(lo)) { 1526 spin_unlock(&inode->i_lock); 1527 return 0; 1528 } 1529 cred = get_cred(lo->plh_lc_cred); 1530 pnfs_get_layout_hdr(lo); 1531 spin_unlock(&inode->i_lock); 1532 1533 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1534 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1535 } 1536 1537 bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, 1538 struct nfs4_layoutreturn_res *res, const struct cred *cred, 1539 bool sync) 1540 { 1541 struct nfs_inode *nfsi = NFS_I(ino); 1542 struct nfs_open_context *ctx; 1543 struct nfs4_state *state; 1544 struct pnfs_layout_hdr *lo; 1545 struct pnfs_layout_segment *lseg, *next; 1546 const struct cred *lc_cred; 1547 nfs4_stateid stateid; 1548 enum pnfs_iomode iomode = 0; 1549 bool layoutreturn = false, roc = false; 1550 bool skip_read; 1551 1552 if (!nfs_have_layout(ino)) 1553 return false; 1554 retry: 1555 rcu_read_lock(); 1556 spin_lock(&ino->i_lock); 1557 lo = nfsi->layout; 1558 if (!lo || !pnfs_layout_is_valid(lo) || 1559 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1560 lo = NULL; 1561 goto out_noroc; 1562 } 1563 1564 /* no roc if we hold a delegation */ 1565 skip_read = false; 1566 if (nfs4_check_delegation(ino, FMODE_READ)) { 1567 if (nfs4_check_delegation(ino, FMODE_WRITE)) { 1568 lo = NULL; 1569 goto out_noroc; 1570 } 1571 skip_read = true; 1572 } 1573 1574 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1575 state = ctx->state; 1576 if (state == NULL) 1577 continue; 1578 /* Don't return layout if there is open file state */ 1579 if (state->state & FMODE_WRITE) { 1580 lo = NULL; 1581 goto out_noroc; 1582 } 1583 if (state->state & FMODE_READ) 1584 skip_read = true; 1585 } 1586 1587 if (skip_read) { 1588 bool writes = false; 1589 1590 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1591 if (lseg->pls_range.iomode != IOMODE_READ) { 1592 writes = true; 1593 break; 1594 } 1595 } 1596 if (!writes) { 1597 lo = NULL; 1598 goto out_noroc; 1599 } 1600 } 1601 1602 pnfs_get_layout_hdr(lo); 1603 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1604 if (!sync) { 1605 pnfs_set_plh_return_info( 1606 lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); 1607 goto out_noroc; 1608 } 1609 spin_unlock(&ino->i_lock); 1610 rcu_read_unlock(); 1611 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1612 TASK_UNINTERRUPTIBLE); 1613 pnfs_put_layout_hdr(lo); 1614 goto retry; 1615 } 1616 1617 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1618 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1619 continue; 1620 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1621 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1622 continue; 1623 /* 1624 * Note: mark lseg for return so pnfs_layout_remove_lseg 1625 * doesn't invalidate the layout for us. 1626 */ 1627 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1628 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1629 continue; 1630 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1631 } 1632 1633 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1634 goto out_noroc; 1635 1636 /* ROC in two conditions: 1637 * 1. there are ROC lsegs 1638 * 2. we don't send layoutreturn 1639 */ 1640 /* lo ref dropped in pnfs_roc_release() */ 1641 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1642 /* If the creds don't match, we can't compound the layoutreturn */ 1643 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1644 goto out_noroc; 1645 1646 roc = layoutreturn; 1647 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1648 res->lrs_present = 0; 1649 layoutreturn = false; 1650 put_cred(lc_cred); 1651 1652 out_noroc: 1653 spin_unlock(&ino->i_lock); 1654 rcu_read_unlock(); 1655 pnfs_layoutcommit_inode(ino, sync); 1656 if (roc) { 1657 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1658 if (ld->prepare_layoutreturn) 1659 ld->prepare_layoutreturn(args); 1660 pnfs_put_layout_hdr(lo); 1661 return true; 1662 } 1663 if (layoutreturn) 1664 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1665 pnfs_put_layout_hdr(lo); 1666 return false; 1667 } 1668 1669 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1670 struct nfs4_layoutreturn_res **respp, int *ret) 1671 { 1672 struct nfs4_layoutreturn_args *arg = *argpp; 1673 int retval = -EAGAIN; 1674 1675 if (!arg) 1676 return 0; 1677 /* Handle Layoutreturn errors */ 1678 switch (*ret) { 1679 case 0: 1680 retval = 0; 1681 break; 1682 case -NFS4ERR_NOMATCHING_LAYOUT: 1683 /* Was there an RPC level error? If not, retry */ 1684 if (task->tk_rpc_status == 0) 1685 break; 1686 /* 1687 * Is there a fatal network level error? 1688 * If so release the layout, but flag the error. 1689 */ 1690 if ((task->tk_rpc_status == -ENETDOWN || 1691 task->tk_rpc_status == -ENETUNREACH) && 1692 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1693 *ret = 0; 1694 (*respp)->lrs_present = 0; 1695 retval = -EIO; 1696 break; 1697 } 1698 /* If the call was not sent, let caller handle it */ 1699 if (!RPC_WAS_SENT(task)) 1700 return 0; 1701 /* 1702 * Otherwise, assume the call succeeded and 1703 * that we need to release the layout 1704 */ 1705 *ret = 0; 1706 (*respp)->lrs_present = 0; 1707 retval = 0; 1708 break; 1709 case -NFS4ERR_DELAY: 1710 /* Let the caller handle the retry */ 1711 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1712 return 0; 1713 case -NFS4ERR_OLD_STATEID: 1714 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1715 &arg->range, arg->inode)) 1716 break; 1717 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1718 return -EAGAIN; 1719 } 1720 *argpp = NULL; 1721 *respp = NULL; 1722 return retval; 1723 } 1724 1725 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1726 struct nfs4_layoutreturn_res *res, int ret) 1727 { 1728 struct pnfs_layout_hdr *lo = args->layout; 1729 struct inode *inode = args->inode; 1730 const nfs4_stateid *res_stateid = NULL; 1731 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1732 LIST_HEAD(freeme); 1733 1734 switch (ret) { 1735 case -NFS4ERR_BADSESSION: 1736 case -NFS4ERR_DEADSESSION: 1737 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1738 case -NFS4ERR_NOMATCHING_LAYOUT: 1739 spin_lock(&inode->i_lock); 1740 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1741 &args->range, &freeme); 1742 spin_unlock(&inode->i_lock); 1743 pnfs_free_lseg_list(&freeme); 1744 break; 1745 case 0: 1746 if (res->lrs_present) 1747 res_stateid = &res->stateid; 1748 fallthrough; 1749 default: 1750 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1751 res_stateid); 1752 } 1753 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1754 if (ld_private && ld_private->ops && ld_private->ops->free) 1755 ld_private->ops->free(ld_private); 1756 pnfs_put_layout_hdr(lo); 1757 } 1758 1759 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1760 { 1761 struct nfs_inode *nfsi = NFS_I(ino); 1762 struct pnfs_layout_hdr *lo; 1763 bool sleep = false; 1764 1765 /* we might not have grabbed lo reference. so need to check under 1766 * i_lock */ 1767 spin_lock(&ino->i_lock); 1768 lo = nfsi->layout; 1769 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1770 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1771 sleep = true; 1772 } 1773 spin_unlock(&ino->i_lock); 1774 return sleep; 1775 } 1776 1777 /* 1778 * Compare two layout segments for sorting into layout cache. 1779 * We want to preferentially return RW over RO layouts, so ensure those 1780 * are seen first. 1781 */ 1782 static s64 1783 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1784 const struct pnfs_layout_range *l2) 1785 { 1786 s64 d; 1787 1788 /* high offset > low offset */ 1789 d = l1->offset - l2->offset; 1790 if (d) 1791 return d; 1792 1793 /* short length > long length */ 1794 d = l2->length - l1->length; 1795 if (d) 1796 return d; 1797 1798 /* read > read/write */ 1799 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1800 } 1801 1802 static bool 1803 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1804 const struct pnfs_layout_range *l2) 1805 { 1806 return pnfs_lseg_range_cmp(l1, l2) > 0; 1807 } 1808 1809 static bool 1810 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1811 struct pnfs_layout_segment *old) 1812 { 1813 return false; 1814 } 1815 1816 void 1817 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1818 struct pnfs_layout_segment *lseg, 1819 bool (*is_after)(const struct pnfs_layout_range *, 1820 const struct pnfs_layout_range *), 1821 bool (*do_merge)(struct pnfs_layout_segment *, 1822 struct pnfs_layout_segment *), 1823 struct list_head *free_me) 1824 { 1825 struct pnfs_layout_segment *lp, *tmp; 1826 1827 dprintk("%s:Begin\n", __func__); 1828 1829 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1830 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1831 continue; 1832 if (do_merge(lseg, lp)) { 1833 mark_lseg_invalid(lp, free_me); 1834 continue; 1835 } 1836 if (is_after(&lseg->pls_range, &lp->pls_range)) 1837 continue; 1838 list_add_tail(&lseg->pls_list, &lp->pls_list); 1839 dprintk("%s: inserted lseg %p " 1840 "iomode %d offset %llu length %llu before " 1841 "lp %p iomode %d offset %llu length %llu\n", 1842 __func__, lseg, lseg->pls_range.iomode, 1843 lseg->pls_range.offset, lseg->pls_range.length, 1844 lp, lp->pls_range.iomode, lp->pls_range.offset, 1845 lp->pls_range.length); 1846 goto out; 1847 } 1848 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1849 dprintk("%s: inserted lseg %p " 1850 "iomode %d offset %llu length %llu at tail\n", 1851 __func__, lseg, lseg->pls_range.iomode, 1852 lseg->pls_range.offset, lseg->pls_range.length); 1853 out: 1854 pnfs_get_layout_hdr(lo); 1855 1856 dprintk("%s:Return\n", __func__); 1857 } 1858 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1859 1860 static void 1861 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1862 struct pnfs_layout_segment *lseg, 1863 struct list_head *free_me) 1864 { 1865 struct inode *inode = lo->plh_inode; 1866 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1867 1868 if (ld->add_lseg != NULL) 1869 ld->add_lseg(lo, lseg, free_me); 1870 else 1871 pnfs_generic_layout_insert_lseg(lo, lseg, 1872 pnfs_lseg_range_is_after, 1873 pnfs_lseg_no_merge, 1874 free_me); 1875 } 1876 1877 static struct pnfs_layout_hdr * 1878 alloc_init_layout_hdr(struct inode *ino, 1879 struct nfs_open_context *ctx, 1880 gfp_t gfp_flags) 1881 { 1882 struct pnfs_layout_hdr *lo; 1883 1884 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1885 if (!lo) 1886 return NULL; 1887 refcount_set(&lo->plh_refcount, 1); 1888 INIT_LIST_HEAD(&lo->plh_layouts); 1889 INIT_LIST_HEAD(&lo->plh_segs); 1890 INIT_LIST_HEAD(&lo->plh_return_segs); 1891 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1892 lo->plh_inode = ino; 1893 lo->plh_lc_cred = get_cred(ctx->cred); 1894 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1895 return lo; 1896 } 1897 1898 static struct pnfs_layout_hdr * 1899 pnfs_find_alloc_layout(struct inode *ino, 1900 struct nfs_open_context *ctx, 1901 gfp_t gfp_flags) 1902 __releases(&ino->i_lock) 1903 __acquires(&ino->i_lock) 1904 { 1905 struct nfs_inode *nfsi = NFS_I(ino); 1906 struct pnfs_layout_hdr *new = NULL; 1907 1908 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1909 1910 if (nfsi->layout != NULL) 1911 goto out_existing; 1912 spin_unlock(&ino->i_lock); 1913 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1914 spin_lock(&ino->i_lock); 1915 1916 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1917 nfsi->layout = new; 1918 return new; 1919 } else if (new != NULL) 1920 pnfs_free_layout_hdr(new); 1921 out_existing: 1922 pnfs_get_layout_hdr(nfsi->layout); 1923 return nfsi->layout; 1924 } 1925 1926 /* 1927 * iomode matching rules: 1928 * iomode lseg strict match 1929 * iomode 1930 * ----- ----- ------ ----- 1931 * ANY READ N/A true 1932 * ANY RW N/A true 1933 * RW READ N/A false 1934 * RW RW N/A true 1935 * READ READ N/A true 1936 * READ RW true false 1937 * READ RW false true 1938 */ 1939 static bool 1940 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1941 const struct pnfs_layout_range *range, 1942 bool strict_iomode) 1943 { 1944 struct pnfs_layout_range range1; 1945 1946 if ((range->iomode == IOMODE_RW && 1947 ls_range->iomode != IOMODE_RW) || 1948 (range->iomode != ls_range->iomode && 1949 strict_iomode) || 1950 !pnfs_lseg_range_intersecting(ls_range, range)) 1951 return false; 1952 1953 /* range1 covers only the first byte in the range */ 1954 range1 = *range; 1955 range1.length = 1; 1956 return pnfs_lseg_range_contained(ls_range, &range1); 1957 } 1958 1959 /* 1960 * lookup range in layout 1961 */ 1962 static struct pnfs_layout_segment * 1963 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1964 struct pnfs_layout_range *range, 1965 bool strict_iomode) 1966 { 1967 struct pnfs_layout_segment *lseg, *ret = NULL; 1968 1969 dprintk("%s:Begin\n", __func__); 1970 1971 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1972 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1973 pnfs_lseg_range_match(&lseg->pls_range, range, 1974 strict_iomode)) { 1975 ret = pnfs_get_lseg(lseg); 1976 break; 1977 } 1978 } 1979 1980 dprintk("%s:Return lseg %p ref %d\n", 1981 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 1982 return ret; 1983 } 1984 1985 /* 1986 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1987 * to the MDS or over pNFS 1988 * 1989 * The nfs_inode read_io and write_io fields are cumulative counters reset 1990 * when there are no layout segments. Note that in pnfs_update_layout iomode 1991 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1992 * WRITE request. 1993 * 1994 * A return of true means use MDS I/O. 1995 * 1996 * From rfc 5661: 1997 * If a file's size is smaller than the file size threshold, data accesses 1998 * SHOULD be sent to the metadata server. If an I/O request has a length that 1999 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 2000 * server. If both file size and I/O size are provided, the client SHOULD 2001 * reach or exceed both thresholds before sending its read or write 2002 * requests to the data server. 2003 */ 2004 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 2005 struct inode *ino, int iomode) 2006 { 2007 struct nfs4_threshold *t = ctx->mdsthreshold; 2008 struct nfs_inode *nfsi = NFS_I(ino); 2009 loff_t fsize = i_size_read(ino); 2010 bool size = false, size_set = false, io = false, io_set = false, ret = false; 2011 2012 if (t == NULL) 2013 return ret; 2014 2015 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 2016 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 2017 2018 switch (iomode) { 2019 case IOMODE_READ: 2020 if (t->bm & THRESHOLD_RD) { 2021 dprintk("%s fsize %llu\n", __func__, fsize); 2022 size_set = true; 2023 if (fsize < t->rd_sz) 2024 size = true; 2025 } 2026 if (t->bm & THRESHOLD_RD_IO) { 2027 dprintk("%s nfsi->read_io %llu\n", __func__, 2028 nfsi->read_io); 2029 io_set = true; 2030 if (nfsi->read_io < t->rd_io_sz) 2031 io = true; 2032 } 2033 break; 2034 case IOMODE_RW: 2035 if (t->bm & THRESHOLD_WR) { 2036 dprintk("%s fsize %llu\n", __func__, fsize); 2037 size_set = true; 2038 if (fsize < t->wr_sz) 2039 size = true; 2040 } 2041 if (t->bm & THRESHOLD_WR_IO) { 2042 dprintk("%s nfsi->write_io %llu\n", __func__, 2043 nfsi->write_io); 2044 io_set = true; 2045 if (nfsi->write_io < t->wr_io_sz) 2046 io = true; 2047 } 2048 break; 2049 } 2050 if (size_set && io_set) { 2051 if (size && io) 2052 ret = true; 2053 } else if (size || io) 2054 ret = true; 2055 2056 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2057 return ret; 2058 } 2059 2060 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2061 { 2062 /* 2063 * send layoutcommit as it can hold up layoutreturn due to lseg 2064 * reference 2065 */ 2066 pnfs_layoutcommit_inode(lo->plh_inode, false); 2067 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2068 nfs_wait_bit_killable, 2069 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2070 } 2071 2072 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2073 { 2074 atomic_inc(&lo->plh_outstanding); 2075 } 2076 2077 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2078 { 2079 if (atomic_dec_and_test(&lo->plh_outstanding) && 2080 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2081 smp_mb__after_atomic(); 2082 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2083 } 2084 } 2085 2086 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2087 { 2088 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2089 } 2090 2091 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 2092 { 2093 unsigned long *bitlock = &lo->plh_flags; 2094 2095 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 2096 smp_mb__after_atomic(); 2097 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 2098 } 2099 2100 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2101 struct nfs_server *server) 2102 { 2103 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2104 struct nfs_client *clp = server->nfs_client; 2105 2106 /* The lo must be on the clp list if there is any 2107 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2108 */ 2109 spin_lock(&clp->cl_lock); 2110 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2111 spin_unlock(&clp->cl_lock); 2112 } 2113 } 2114 2115 /* 2116 * Layout segment is retreived from the server if not cached. 2117 * The appropriate layout segment is referenced and returned to the caller. 2118 */ 2119 struct pnfs_layout_segment * 2120 pnfs_update_layout(struct inode *ino, 2121 struct nfs_open_context *ctx, 2122 loff_t pos, 2123 u64 count, 2124 enum pnfs_iomode iomode, 2125 bool strict_iomode, 2126 gfp_t gfp_flags) 2127 { 2128 struct pnfs_layout_range arg = { 2129 .iomode = iomode, 2130 .offset = pos, 2131 .length = count, 2132 }; 2133 unsigned pg_offset; 2134 struct nfs_server *server = NFS_SERVER(ino); 2135 struct nfs_client *clp = server->nfs_client; 2136 struct pnfs_layout_hdr *lo = NULL; 2137 struct pnfs_layout_segment *lseg = NULL; 2138 struct nfs4_layoutget *lgp; 2139 nfs4_stateid stateid; 2140 struct nfs4_exception exception = { 2141 .inode = ino, 2142 }; 2143 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2144 bool first; 2145 2146 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2147 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2148 PNFS_UPDATE_LAYOUT_NO_PNFS); 2149 goto out; 2150 } 2151 2152 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2153 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2154 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2155 goto out; 2156 } 2157 2158 lookup_again: 2159 if (!nfs4_valid_open_stateid(ctx->state)) { 2160 trace_pnfs_update_layout(ino, pos, count, 2161 iomode, lo, lseg, 2162 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2163 lseg = ERR_PTR(-EIO); 2164 goto out; 2165 } 2166 2167 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2168 if (IS_ERR(lseg)) 2169 goto out; 2170 first = false; 2171 spin_lock(&ino->i_lock); 2172 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2173 if (lo == NULL) { 2174 spin_unlock(&ino->i_lock); 2175 lseg = ERR_PTR(-ENOMEM); 2176 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2177 PNFS_UPDATE_LAYOUT_NOMEM); 2178 goto out; 2179 } 2180 2181 /* Do we even need to bother with this? */ 2182 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2183 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2184 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2185 dprintk("%s matches recall, use MDS\n", __func__); 2186 goto out_unlock; 2187 } 2188 2189 /* if LAYOUTGET already failed once we don't try again */ 2190 if (pnfs_layout_io_test_failed(lo, iomode)) { 2191 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2192 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2193 goto out_unlock; 2194 } 2195 2196 /* 2197 * If the layout segment list is empty, but there are outstanding 2198 * layoutget calls, then they might be subject to a layoutrecall. 2199 */ 2200 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2201 atomic_read(&lo->plh_outstanding) != 0) { 2202 spin_unlock(&ino->i_lock); 2203 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2204 TASK_KILLABLE)); 2205 if (IS_ERR(lseg)) 2206 goto out_put_layout_hdr; 2207 pnfs_put_layout_hdr(lo); 2208 goto lookup_again; 2209 } 2210 2211 /* 2212 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2213 * for LAYOUTRETURN. 2214 */ 2215 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2216 spin_unlock(&ino->i_lock); 2217 dprintk("%s wait for layoutreturn\n", __func__); 2218 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2219 if (!IS_ERR(lseg)) { 2220 pnfs_put_layout_hdr(lo); 2221 dprintk("%s retrying\n", __func__); 2222 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2223 lseg, 2224 PNFS_UPDATE_LAYOUT_RETRY); 2225 goto lookup_again; 2226 } 2227 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2228 PNFS_UPDATE_LAYOUT_RETURN); 2229 goto out_put_layout_hdr; 2230 } 2231 2232 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2233 if (lseg) { 2234 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2235 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2236 goto out_unlock; 2237 } 2238 2239 /* 2240 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2241 * stateid, or it has been invalidated, then we must use the open 2242 * stateid. 2243 */ 2244 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2245 int status; 2246 2247 /* 2248 * The first layoutget for the file. Need to serialize per 2249 * RFC 5661 Errata 3208. 2250 */ 2251 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2252 &lo->plh_flags)) { 2253 spin_unlock(&ino->i_lock); 2254 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2255 NFS_LAYOUT_FIRST_LAYOUTGET, 2256 TASK_KILLABLE)); 2257 if (IS_ERR(lseg)) 2258 goto out_put_layout_hdr; 2259 pnfs_put_layout_hdr(lo); 2260 dprintk("%s retrying\n", __func__); 2261 goto lookup_again; 2262 } 2263 2264 spin_unlock(&ino->i_lock); 2265 first = true; 2266 status = nfs4_select_rw_stateid(ctx->state, 2267 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2268 NULL, &stateid, NULL); 2269 if (status != 0) { 2270 lseg = ERR_PTR(status); 2271 trace_pnfs_update_layout(ino, pos, count, 2272 iomode, lo, lseg, 2273 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2274 nfs4_schedule_stateid_recovery(server, ctx->state); 2275 pnfs_clear_first_layoutget(lo); 2276 pnfs_put_layout_hdr(lo); 2277 goto lookup_again; 2278 } 2279 spin_lock(&ino->i_lock); 2280 } else { 2281 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2282 } 2283 2284 if (pnfs_layoutgets_blocked(lo)) { 2285 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2286 PNFS_UPDATE_LAYOUT_BLOCKED); 2287 goto out_unlock; 2288 } 2289 nfs_layoutget_begin(lo); 2290 spin_unlock(&ino->i_lock); 2291 2292 _add_to_server_list(lo, server); 2293 2294 pg_offset = arg.offset & ~PAGE_MASK; 2295 if (pg_offset) { 2296 arg.offset -= pg_offset; 2297 arg.length += pg_offset; 2298 } 2299 if (arg.length != NFS4_MAX_UINT64) 2300 arg.length = PAGE_ALIGN(arg.length); 2301 2302 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); 2303 if (!lgp) { 2304 lseg = ERR_PTR(-ENOMEM); 2305 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2306 PNFS_UPDATE_LAYOUT_NOMEM); 2307 nfs_layoutget_end(lo); 2308 goto out_put_layout_hdr; 2309 } 2310 2311 lgp->lo = lo; 2312 pnfs_get_layout_hdr(lo); 2313 2314 lseg = nfs4_proc_layoutget(lgp, &exception); 2315 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2316 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2317 nfs_layoutget_end(lo); 2318 if (IS_ERR(lseg)) { 2319 switch(PTR_ERR(lseg)) { 2320 case -EBUSY: 2321 if (time_after(jiffies, giveup)) 2322 lseg = NULL; 2323 break; 2324 case -ERECALLCONFLICT: 2325 case -EAGAIN: 2326 break; 2327 case -ENODATA: 2328 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2329 pnfs_layout_set_fail_bit( 2330 lo, pnfs_iomode_to_fail_bit(iomode)); 2331 lseg = NULL; 2332 goto out_put_layout_hdr; 2333 default: 2334 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2335 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2336 lseg = NULL; 2337 } 2338 goto out_put_layout_hdr; 2339 } 2340 if (lseg) { 2341 if (!exception.retry) 2342 goto out_put_layout_hdr; 2343 if (first) 2344 pnfs_clear_first_layoutget(lo); 2345 trace_pnfs_update_layout(ino, pos, count, 2346 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2347 pnfs_put_layout_hdr(lo); 2348 goto lookup_again; 2349 } 2350 } else { 2351 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2352 } 2353 2354 out_put_layout_hdr: 2355 if (first) 2356 pnfs_clear_first_layoutget(lo); 2357 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2358 PNFS_UPDATE_LAYOUT_EXIT); 2359 pnfs_put_layout_hdr(lo); 2360 out: 2361 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2362 "(%s, offset: %llu, length: %llu)\n", 2363 __func__, ino->i_sb->s_id, 2364 (unsigned long long)NFS_FILEID(ino), 2365 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2366 iomode==IOMODE_RW ? "read/write" : "read-only", 2367 (unsigned long long)pos, 2368 (unsigned long long)count); 2369 return lseg; 2370 out_unlock: 2371 spin_unlock(&ino->i_lock); 2372 goto out_put_layout_hdr; 2373 } 2374 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2375 2376 static bool 2377 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2378 { 2379 switch (range->iomode) { 2380 case IOMODE_READ: 2381 case IOMODE_RW: 2382 break; 2383 default: 2384 return false; 2385 } 2386 if (range->offset == NFS4_MAX_UINT64) 2387 return false; 2388 if (range->length == 0) 2389 return false; 2390 if (range->length != NFS4_MAX_UINT64 && 2391 range->length > NFS4_MAX_UINT64 - range->offset) 2392 return false; 2393 return true; 2394 } 2395 2396 static struct pnfs_layout_hdr * 2397 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2398 { 2399 struct pnfs_layout_hdr *lo; 2400 2401 spin_lock(&ino->i_lock); 2402 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2403 if (!lo) 2404 goto out_unlock; 2405 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2406 goto out_unlock; 2407 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2408 goto out_unlock; 2409 if (pnfs_layoutgets_blocked(lo)) 2410 goto out_unlock; 2411 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2412 goto out_unlock; 2413 nfs_layoutget_begin(lo); 2414 spin_unlock(&ino->i_lock); 2415 _add_to_server_list(lo, NFS_SERVER(ino)); 2416 return lo; 2417 2418 out_unlock: 2419 spin_unlock(&ino->i_lock); 2420 pnfs_put_layout_hdr(lo); 2421 return NULL; 2422 } 2423 2424 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2425 struct nfs_open_context *ctx) 2426 { 2427 struct inode *ino = data->dentry->d_inode; 2428 struct pnfs_layout_range rng = { 2429 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2430 IOMODE_RW: IOMODE_READ, 2431 .offset = 0, 2432 .length = NFS4_MAX_UINT64, 2433 }; 2434 struct nfs4_layoutget *lgp; 2435 struct pnfs_layout_hdr *lo; 2436 2437 /* Heuristic: don't send layoutget if we have cached data */ 2438 if (rng.iomode == IOMODE_READ && 2439 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2440 return; 2441 2442 lo = _pnfs_grab_empty_layout(ino, ctx); 2443 if (!lo) 2444 return; 2445 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2446 nfs_io_gfp_mask()); 2447 if (!lgp) { 2448 pnfs_clear_first_layoutget(lo); 2449 nfs_layoutget_end(lo); 2450 pnfs_put_layout_hdr(lo); 2451 return; 2452 } 2453 lgp->lo = lo; 2454 data->lgp = lgp; 2455 data->o_arg.lg_args = &lgp->args; 2456 data->o_res.lg_res = &lgp->res; 2457 } 2458 2459 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2460 struct nfs_open_context *ctx) 2461 { 2462 struct inode *ino = data->dentry->d_inode; 2463 struct pnfs_layout_range rng = { 2464 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2465 IOMODE_RW: IOMODE_READ, 2466 .offset = 0, 2467 .length = NFS4_MAX_UINT64, 2468 }; 2469 struct nfs4_layoutget *lgp; 2470 2471 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2472 nfs_io_gfp_mask()); 2473 if (!lgp) 2474 return; 2475 data->lgp = lgp; 2476 data->o_arg.lg_args = &lgp->args; 2477 data->o_res.lg_res = &lgp->res; 2478 } 2479 2480 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2481 struct nfs_open_context *ctx) 2482 { 2483 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2484 2485 if (!(pnfs_enabled_sb(server) && 2486 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2487 return; 2488 /* Could check on max_ops, but currently hardcoded high enough */ 2489 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2490 return; 2491 if (data->lgp) 2492 return; 2493 if (data->state) 2494 _lgopen_prepare_attached(data, ctx); 2495 else 2496 _lgopen_prepare_floating(data, ctx); 2497 } 2498 2499 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2500 struct nfs_open_context *ctx) 2501 { 2502 struct pnfs_layout_hdr *lo; 2503 struct pnfs_layout_segment *lseg; 2504 struct nfs_server *srv = NFS_SERVER(ino); 2505 u32 iomode; 2506 2507 if (!lgp) 2508 return; 2509 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2510 if (lgp->res.status) { 2511 switch (lgp->res.status) { 2512 default: 2513 break; 2514 /* 2515 * Halt lgopen attempts if the server doesn't recognise 2516 * the "current stateid" value, the layout type, or the 2517 * layoutget operation as being valid. 2518 * Also if it complains about too many ops in the compound 2519 * or of the request/reply being too big. 2520 */ 2521 case -NFS4ERR_BAD_STATEID: 2522 case -NFS4ERR_NOTSUPP: 2523 case -NFS4ERR_REP_TOO_BIG: 2524 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2525 case -NFS4ERR_REQ_TOO_BIG: 2526 case -NFS4ERR_TOO_MANY_OPS: 2527 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2528 srv->caps &= ~NFS_CAP_LGOPEN; 2529 } 2530 return; 2531 } 2532 if (!lgp->lo) { 2533 lo = _pnfs_grab_empty_layout(ino, ctx); 2534 if (!lo) 2535 return; 2536 lgp->lo = lo; 2537 } else 2538 lo = lgp->lo; 2539 2540 lseg = pnfs_layout_process(lgp); 2541 if (!IS_ERR(lseg)) { 2542 iomode = lgp->args.range.iomode; 2543 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2544 pnfs_put_lseg(lseg); 2545 } 2546 } 2547 2548 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2549 { 2550 if (lgp != NULL) { 2551 if (lgp->lo) { 2552 pnfs_clear_first_layoutget(lgp->lo); 2553 nfs_layoutget_end(lgp->lo); 2554 } 2555 pnfs_layoutget_free(lgp); 2556 } 2557 } 2558 2559 struct pnfs_layout_segment * 2560 pnfs_layout_process(struct nfs4_layoutget *lgp) 2561 { 2562 struct pnfs_layout_hdr *lo = lgp->lo; 2563 struct nfs4_layoutget_res *res = &lgp->res; 2564 struct pnfs_layout_segment *lseg; 2565 struct inode *ino = lo->plh_inode; 2566 LIST_HEAD(free_me); 2567 2568 if (!pnfs_sanity_check_layout_range(&res->range)) 2569 return ERR_PTR(-EINVAL); 2570 2571 /* Inject layout blob into I/O device driver */ 2572 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2573 if (IS_ERR_OR_NULL(lseg)) { 2574 if (!lseg) 2575 lseg = ERR_PTR(-ENOMEM); 2576 2577 dprintk("%s: Could not allocate layout: error %ld\n", 2578 __func__, PTR_ERR(lseg)); 2579 return lseg; 2580 } 2581 2582 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2583 2584 spin_lock(&ino->i_lock); 2585 if (pnfs_layoutgets_blocked(lo)) { 2586 dprintk("%s forget reply due to state\n", __func__); 2587 goto out_forget; 2588 } 2589 2590 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2591 !pnfs_is_first_layoutget(lo)) 2592 goto out_forget; 2593 2594 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2595 /* existing state ID, make sure the sequence number matches. */ 2596 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2597 if (!pnfs_layout_is_valid(lo)) 2598 lo->plh_barrier = 0; 2599 dprintk("%s forget reply due to sequence\n", __func__); 2600 goto out_forget; 2601 } 2602 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2603 } else if (pnfs_layout_is_valid(lo)) { 2604 /* 2605 * We got an entirely new state ID. Mark all segments for the 2606 * inode invalid, and retry the layoutget 2607 */ 2608 struct pnfs_layout_range range = { 2609 .iomode = IOMODE_ANY, 2610 .length = NFS4_MAX_UINT64, 2611 }; 2612 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); 2613 goto out_forget; 2614 } else { 2615 /* We have a completely new layout */ 2616 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2617 } 2618 2619 pnfs_get_lseg(lseg); 2620 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2621 2622 2623 if (res->return_on_close) 2624 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2625 2626 spin_unlock(&ino->i_lock); 2627 pnfs_free_lseg_list(&free_me); 2628 return lseg; 2629 2630 out_forget: 2631 spin_unlock(&ino->i_lock); 2632 lseg->pls_layout = lo; 2633 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2634 return ERR_PTR(-EAGAIN); 2635 } 2636 2637 /** 2638 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2639 * @lo: pointer to layout header 2640 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2641 * @return_range: describe layout segment ranges to be returned 2642 * @seq: stateid seqid to match 2643 * 2644 * This function is mainly intended for use by layoutrecall. It attempts 2645 * to free the layout segment immediately, or else to mark it for return 2646 * as soon as its reference count drops to zero. 2647 * 2648 * Returns 2649 * - 0: a layoutreturn needs to be scheduled. 2650 * - EBUSY: there are layout segment that are still in use. 2651 * - ENOENT: there are no layout segments that need to be returned. 2652 */ 2653 int 2654 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2655 struct list_head *tmp_list, 2656 const struct pnfs_layout_range *return_range, 2657 u32 seq) 2658 { 2659 struct pnfs_layout_segment *lseg, *next; 2660 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2661 int remaining = 0; 2662 2663 dprintk("%s:Begin lo %p\n", __func__, lo); 2664 2665 assert_spin_locked(&lo->plh_inode->i_lock); 2666 2667 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2668 tmp_list = &lo->plh_return_segs; 2669 2670 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2671 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2672 dprintk("%s: marking lseg %p iomode %d " 2673 "offset %llu length %llu\n", __func__, 2674 lseg, lseg->pls_range.iomode, 2675 lseg->pls_range.offset, 2676 lseg->pls_range.length); 2677 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2678 tmp_list = &lo->plh_return_segs; 2679 if (mark_lseg_invalid(lseg, tmp_list)) 2680 continue; 2681 remaining++; 2682 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2683 pnfs_lseg_cancel_io(server, lseg); 2684 } 2685 2686 if (remaining) { 2687 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2688 return -EBUSY; 2689 } 2690 2691 if (!list_empty(&lo->plh_return_segs)) { 2692 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2693 return 0; 2694 } 2695 2696 return -ENOENT; 2697 } 2698 2699 static void 2700 pnfs_mark_layout_for_return(struct inode *inode, 2701 const struct pnfs_layout_range *range) 2702 { 2703 struct pnfs_layout_hdr *lo; 2704 bool return_now = false; 2705 2706 spin_lock(&inode->i_lock); 2707 lo = NFS_I(inode)->layout; 2708 if (!pnfs_layout_is_valid(lo)) { 2709 spin_unlock(&inode->i_lock); 2710 return; 2711 } 2712 pnfs_set_plh_return_info(lo, range->iomode, 0); 2713 /* 2714 * mark all matching lsegs so that we are sure to have no live 2715 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2716 * for how it works. 2717 */ 2718 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { 2719 const struct cred *cred; 2720 nfs4_stateid stateid; 2721 enum pnfs_iomode iomode; 2722 2723 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2724 spin_unlock(&inode->i_lock); 2725 if (return_now) 2726 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2727 PNFS_FL_LAYOUTRETURN_ASYNC); 2728 } else { 2729 spin_unlock(&inode->i_lock); 2730 nfs_commit_inode(inode, 0); 2731 } 2732 } 2733 2734 void pnfs_error_mark_layout_for_return(struct inode *inode, 2735 struct pnfs_layout_segment *lseg) 2736 { 2737 struct pnfs_layout_range range = { 2738 .iomode = lseg->pls_range.iomode, 2739 .offset = 0, 2740 .length = NFS4_MAX_UINT64, 2741 }; 2742 2743 pnfs_mark_layout_for_return(inode, &range); 2744 } 2745 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2746 2747 static bool 2748 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2749 { 2750 return pnfs_layout_is_valid(lo) && 2751 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2752 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2753 } 2754 2755 static struct pnfs_layout_segment * 2756 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2757 const struct pnfs_layout_range *range, 2758 enum pnfs_iomode iomode) 2759 { 2760 struct pnfs_layout_segment *lseg; 2761 2762 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2763 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2764 continue; 2765 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2766 continue; 2767 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2768 continue; 2769 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2770 return lseg; 2771 } 2772 return NULL; 2773 } 2774 2775 /* Find open file states whose mode matches that of the range */ 2776 static bool 2777 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2778 const struct pnfs_layout_range *range) 2779 { 2780 struct list_head *head; 2781 struct nfs_open_context *ctx; 2782 fmode_t mode = 0; 2783 2784 if (!pnfs_layout_can_be_returned(lo) || 2785 !pnfs_find_first_lseg(lo, range, range->iomode)) 2786 return false; 2787 2788 head = &NFS_I(lo->plh_inode)->open_files; 2789 list_for_each_entry_rcu(ctx, head, list) { 2790 if (ctx->state) 2791 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2792 } 2793 2794 switch (range->iomode) { 2795 default: 2796 break; 2797 case IOMODE_READ: 2798 mode &= ~FMODE_WRITE; 2799 break; 2800 case IOMODE_RW: 2801 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2802 mode &= ~FMODE_READ; 2803 } 2804 return mode == 0; 2805 } 2806 2807 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2808 void *data) 2809 { 2810 const struct pnfs_layout_range *range = data; 2811 const struct cred *cred; 2812 struct pnfs_layout_hdr *lo; 2813 struct inode *inode; 2814 nfs4_stateid stateid; 2815 enum pnfs_iomode iomode; 2816 2817 restart: 2818 rcu_read_lock(); 2819 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2820 inode = lo->plh_inode; 2821 if (!inode || !pnfs_layout_can_be_returned(lo) || 2822 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2823 continue; 2824 spin_lock(&inode->i_lock); 2825 if (!lo->plh_inode || 2826 !pnfs_should_return_unused_layout(lo, range)) { 2827 spin_unlock(&inode->i_lock); 2828 continue; 2829 } 2830 pnfs_get_layout_hdr(lo); 2831 pnfs_set_plh_return_info(lo, range->iomode, 0); 2832 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2833 range, 0) != 0 || 2834 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2835 spin_unlock(&inode->i_lock); 2836 rcu_read_unlock(); 2837 pnfs_put_layout_hdr(lo); 2838 cond_resched(); 2839 goto restart; 2840 } 2841 spin_unlock(&inode->i_lock); 2842 rcu_read_unlock(); 2843 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2844 PNFS_FL_LAYOUTRETURN_ASYNC); 2845 pnfs_put_layout_hdr(lo); 2846 cond_resched(); 2847 goto restart; 2848 } 2849 rcu_read_unlock(); 2850 return 0; 2851 } 2852 2853 void 2854 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2855 enum pnfs_iomode iomode) 2856 { 2857 struct pnfs_layout_range range = { 2858 .iomode = iomode, 2859 .offset = 0, 2860 .length = NFS4_MAX_UINT64, 2861 }; 2862 2863 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2864 &range); 2865 } 2866 2867 /* Check if we have we have a valid layout but if there isn't an intersection 2868 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 2869 */ 2870 void 2871 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 2872 struct nfs_page *req) 2873 { 2874 if (pgio->pg_lseg == NULL || 2875 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 2876 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 2877 return; 2878 pnfs_put_lseg(pgio->pg_lseg); 2879 pgio->pg_lseg = NULL; 2880 } 2881 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2882 2883 void 2884 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2885 { 2886 u64 rd_size; 2887 2888 pnfs_generic_pg_check_layout(pgio, req); 2889 if (pgio->pg_lseg == NULL) { 2890 if (pgio->pg_dreq == NULL) 2891 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2892 else 2893 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 2894 req_offset(req)); 2895 2896 pgio->pg_lseg = 2897 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2898 req_offset(req), rd_size, 2899 IOMODE_READ, false, 2900 nfs_io_gfp_mask()); 2901 if (IS_ERR(pgio->pg_lseg)) { 2902 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2903 pgio->pg_lseg = NULL; 2904 return; 2905 } 2906 } 2907 /* If no lseg, fall back to read through mds */ 2908 if (pgio->pg_lseg == NULL) 2909 nfs_pageio_reset_read_mds(pgio); 2910 2911 } 2912 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2913 2914 void 2915 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2916 struct nfs_page *req, u64 wb_size) 2917 { 2918 pnfs_generic_pg_check_layout(pgio, req); 2919 if (pgio->pg_lseg == NULL) { 2920 pgio->pg_lseg = 2921 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2922 req_offset(req), wb_size, IOMODE_RW, 2923 false, nfs_io_gfp_mask()); 2924 if (IS_ERR(pgio->pg_lseg)) { 2925 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2926 pgio->pg_lseg = NULL; 2927 return; 2928 } 2929 } 2930 /* If no lseg, fall back to write through mds */ 2931 if (pgio->pg_lseg == NULL) 2932 nfs_pageio_reset_write_mds(pgio); 2933 } 2934 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2935 2936 void 2937 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2938 { 2939 if (desc->pg_lseg) { 2940 pnfs_put_lseg(desc->pg_lseg); 2941 desc->pg_lseg = NULL; 2942 } 2943 } 2944 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2945 2946 /* 2947 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2948 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2949 */ 2950 size_t 2951 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2952 struct nfs_page *prev, struct nfs_page *req) 2953 { 2954 unsigned int size; 2955 u64 seg_end, req_start, seg_left; 2956 2957 size = nfs_generic_pg_test(pgio, prev, req); 2958 if (!size) 2959 return 0; 2960 2961 /* 2962 * 'size' contains the number of bytes left in the current page (up 2963 * to the original size asked for in @req->wb_bytes). 2964 * 2965 * Calculate how many bytes are left in the layout segment 2966 * and if there are less bytes than 'size', return that instead. 2967 * 2968 * Please also note that 'end_offset' is actually the offset of the 2969 * first byte that lies outside the pnfs_layout_range. FIXME? 2970 * 2971 */ 2972 if (pgio->pg_lseg) { 2973 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2974 pgio->pg_lseg->pls_range.length); 2975 req_start = req_offset(req); 2976 2977 /* start of request is past the last byte of this segment */ 2978 if (req_start >= seg_end) 2979 return 0; 2980 2981 /* adjust 'size' iff there are fewer bytes left in the 2982 * segment than what nfs_generic_pg_test returned */ 2983 seg_left = seg_end - req_start; 2984 if (seg_left < size) 2985 size = (unsigned int)seg_left; 2986 } 2987 2988 return size; 2989 } 2990 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2991 2992 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2993 { 2994 struct nfs_pageio_descriptor pgio; 2995 2996 /* Resend all requests through the MDS */ 2997 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2998 hdr->completion_ops); 2999 return nfs_pageio_resend(&pgio, hdr); 3000 } 3001 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 3002 3003 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 3004 { 3005 3006 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 3007 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3008 PNFS_LAYOUTRET_ON_ERROR) { 3009 pnfs_return_layout(hdr->inode); 3010 } 3011 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3012 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 3013 } 3014 3015 /* 3016 * Called by non rpc-based layout drivers 3017 */ 3018 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 3019 { 3020 if (likely(!hdr->pnfs_error)) { 3021 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 3022 hdr->mds_offset + hdr->res.count); 3023 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3024 } 3025 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3026 if (unlikely(hdr->pnfs_error)) 3027 pnfs_ld_handle_write_error(hdr); 3028 hdr->mds_ops->rpc_release(hdr); 3029 } 3030 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3031 3032 static void 3033 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3034 struct nfs_pgio_header *hdr) 3035 { 3036 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3037 3038 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3039 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3040 nfs_pageio_reset_write_mds(desc); 3041 mirror->pg_recoalesce = 1; 3042 } 3043 hdr->completion_ops->completion(hdr); 3044 } 3045 3046 static enum pnfs_try_status 3047 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3048 const struct rpc_call_ops *call_ops, 3049 struct pnfs_layout_segment *lseg, 3050 int how) 3051 { 3052 struct inode *inode = hdr->inode; 3053 enum pnfs_try_status trypnfs; 3054 struct nfs_server *nfss = NFS_SERVER(inode); 3055 3056 hdr->mds_ops = call_ops; 3057 3058 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 3059 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3060 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3061 if (trypnfs != PNFS_NOT_ATTEMPTED) 3062 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3063 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3064 return trypnfs; 3065 } 3066 3067 static void 3068 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3069 struct nfs_pgio_header *hdr, int how) 3070 { 3071 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3072 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3073 enum pnfs_try_status trypnfs; 3074 3075 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3076 switch (trypnfs) { 3077 case PNFS_NOT_ATTEMPTED: 3078 pnfs_write_through_mds(desc, hdr); 3079 break; 3080 case PNFS_ATTEMPTED: 3081 break; 3082 case PNFS_TRY_AGAIN: 3083 /* cleanup hdr and prepare to redo pnfs */ 3084 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3085 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3086 list_splice_init(&hdr->pages, &mirror->pg_list); 3087 mirror->pg_recoalesce = 1; 3088 } 3089 hdr->mds_ops->rpc_release(hdr); 3090 } 3091 } 3092 3093 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3094 { 3095 pnfs_put_lseg(hdr->lseg); 3096 nfs_pgio_header_free(hdr); 3097 } 3098 3099 int 3100 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3101 { 3102 struct nfs_pgio_header *hdr; 3103 int ret; 3104 3105 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3106 if (!hdr) { 3107 desc->pg_error = -ENOMEM; 3108 return desc->pg_error; 3109 } 3110 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3111 3112 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3113 ret = nfs_generic_pgio(desc, hdr); 3114 if (!ret) 3115 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3116 3117 return ret; 3118 } 3119 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3120 3121 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3122 { 3123 struct nfs_pageio_descriptor pgio; 3124 3125 /* Resend all requests through the MDS */ 3126 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3127 return nfs_pageio_resend(&pgio, hdr); 3128 } 3129 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3130 3131 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3132 { 3133 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3134 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3135 PNFS_LAYOUTRET_ON_ERROR) { 3136 pnfs_return_layout(hdr->inode); 3137 } 3138 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3139 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3140 } 3141 3142 /* 3143 * Called by non rpc-based layout drivers 3144 */ 3145 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3146 { 3147 if (likely(!hdr->pnfs_error)) 3148 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3149 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3150 if (unlikely(hdr->pnfs_error)) 3151 pnfs_ld_handle_read_error(hdr); 3152 hdr->mds_ops->rpc_release(hdr); 3153 } 3154 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3155 3156 static void 3157 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3158 struct nfs_pgio_header *hdr) 3159 { 3160 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3161 3162 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3163 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3164 nfs_pageio_reset_read_mds(desc); 3165 mirror->pg_recoalesce = 1; 3166 } 3167 hdr->completion_ops->completion(hdr); 3168 } 3169 3170 /* 3171 * Call the appropriate parallel I/O subsystem read function. 3172 */ 3173 static enum pnfs_try_status 3174 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3175 const struct rpc_call_ops *call_ops, 3176 struct pnfs_layout_segment *lseg) 3177 { 3178 struct inode *inode = hdr->inode; 3179 struct nfs_server *nfss = NFS_SERVER(inode); 3180 enum pnfs_try_status trypnfs; 3181 3182 hdr->mds_ops = call_ops; 3183 3184 dprintk("%s: Reading ino:%lu %u@%llu\n", 3185 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3186 3187 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3188 if (trypnfs != PNFS_NOT_ATTEMPTED) 3189 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3190 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3191 return trypnfs; 3192 } 3193 3194 /* Resend all requests through pnfs. */ 3195 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3196 unsigned int mirror_idx) 3197 { 3198 struct nfs_pageio_descriptor pgio; 3199 3200 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3201 /* Prevent deadlocks with layoutreturn! */ 3202 pnfs_put_lseg(hdr->lseg); 3203 hdr->lseg = NULL; 3204 3205 nfs_pageio_init_read(&pgio, hdr->inode, false, 3206 hdr->completion_ops); 3207 pgio.pg_mirror_idx = mirror_idx; 3208 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3209 } 3210 } 3211 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3212 3213 static void 3214 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3215 { 3216 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3217 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3218 enum pnfs_try_status trypnfs; 3219 3220 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3221 switch (trypnfs) { 3222 case PNFS_NOT_ATTEMPTED: 3223 pnfs_read_through_mds(desc, hdr); 3224 break; 3225 case PNFS_ATTEMPTED: 3226 break; 3227 case PNFS_TRY_AGAIN: 3228 /* cleanup hdr and prepare to redo pnfs */ 3229 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3230 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3231 list_splice_init(&hdr->pages, &mirror->pg_list); 3232 mirror->pg_recoalesce = 1; 3233 } 3234 hdr->mds_ops->rpc_release(hdr); 3235 } 3236 } 3237 3238 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3239 { 3240 pnfs_put_lseg(hdr->lseg); 3241 nfs_pgio_header_free(hdr); 3242 } 3243 3244 int 3245 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3246 { 3247 struct nfs_pgio_header *hdr; 3248 int ret; 3249 3250 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3251 if (!hdr) { 3252 desc->pg_error = -ENOMEM; 3253 return desc->pg_error; 3254 } 3255 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3256 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3257 ret = nfs_generic_pgio(desc, hdr); 3258 if (!ret) 3259 pnfs_do_read(desc, hdr); 3260 return ret; 3261 } 3262 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3263 3264 static void pnfs_clear_layoutcommitting(struct inode *inode) 3265 { 3266 unsigned long *bitlock = &NFS_I(inode)->flags; 3267 3268 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 3269 smp_mb__after_atomic(); 3270 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 3271 } 3272 3273 /* 3274 * There can be multiple RW segments. 3275 */ 3276 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3277 { 3278 struct pnfs_layout_segment *lseg; 3279 3280 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3281 if (lseg->pls_range.iomode == IOMODE_RW && 3282 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3283 list_add(&lseg->pls_lc_list, listp); 3284 } 3285 } 3286 3287 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3288 { 3289 struct pnfs_layout_segment *lseg, *tmp; 3290 3291 /* Matched by references in pnfs_set_layoutcommit */ 3292 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3293 list_del_init(&lseg->pls_lc_list); 3294 pnfs_put_lseg(lseg); 3295 } 3296 3297 pnfs_clear_layoutcommitting(inode); 3298 } 3299 3300 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3301 { 3302 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3303 } 3304 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3305 3306 void 3307 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3308 loff_t end_pos) 3309 { 3310 struct nfs_inode *nfsi = NFS_I(inode); 3311 bool mark_as_dirty = false; 3312 3313 spin_lock(&inode->i_lock); 3314 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3315 nfsi->layout->plh_lwb = end_pos; 3316 mark_as_dirty = true; 3317 dprintk("%s: Set layoutcommit for inode %lu ", 3318 __func__, inode->i_ino); 3319 } else if (end_pos > nfsi->layout->plh_lwb) 3320 nfsi->layout->plh_lwb = end_pos; 3321 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3322 /* references matched in nfs4_layoutcommit_release */ 3323 pnfs_get_lseg(lseg); 3324 } 3325 spin_unlock(&inode->i_lock); 3326 dprintk("%s: lseg %p end_pos %llu\n", 3327 __func__, lseg, nfsi->layout->plh_lwb); 3328 3329 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3330 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3331 if (mark_as_dirty) 3332 mark_inode_dirty_sync(inode); 3333 } 3334 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3335 3336 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3337 { 3338 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3339 3340 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3341 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3342 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3343 } 3344 3345 /* 3346 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3347 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3348 * data to disk to allow the server to recover the data if it crashes. 3349 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3350 * is off, and a COMMIT is sent to a data server, or 3351 * if WRITEs to a data server return NFS_DATA_SYNC. 3352 */ 3353 int 3354 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3355 { 3356 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3357 struct nfs4_layoutcommit_data *data; 3358 struct nfs_inode *nfsi = NFS_I(inode); 3359 loff_t end_pos; 3360 int status; 3361 bool mark_as_dirty = false; 3362 3363 if (!pnfs_layoutcommit_outstanding(inode)) 3364 return 0; 3365 3366 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 3367 3368 status = -EAGAIN; 3369 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3370 if (!sync) 3371 goto out; 3372 status = wait_on_bit_lock_action(&nfsi->flags, 3373 NFS_INO_LAYOUTCOMMITTING, 3374 nfs_wait_bit_killable, 3375 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3376 if (status) 3377 goto out; 3378 } 3379 3380 status = -ENOMEM; 3381 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3382 data = kzalloc(sizeof(*data), nfs_io_gfp_mask()); 3383 if (!data) 3384 goto clear_layoutcommitting; 3385 3386 status = 0; 3387 spin_lock(&inode->i_lock); 3388 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3389 goto out_unlock; 3390 3391 INIT_LIST_HEAD(&data->lseg_list); 3392 pnfs_list_write_lseg(inode, &data->lseg_list); 3393 3394 end_pos = nfsi->layout->plh_lwb; 3395 3396 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3397 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3398 spin_unlock(&inode->i_lock); 3399 3400 data->args.inode = inode; 3401 nfs_fattr_init(&data->fattr); 3402 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3403 data->res.fattr = &data->fattr; 3404 if (end_pos != 0) 3405 data->args.lastbytewritten = end_pos - 1; 3406 else 3407 data->args.lastbytewritten = U64_MAX; 3408 data->res.server = NFS_SERVER(inode); 3409 3410 if (ld->prepare_layoutcommit) { 3411 status = ld->prepare_layoutcommit(&data->args); 3412 if (status) { 3413 if (status != -ENOSPC) 3414 put_cred(data->cred); 3415 spin_lock(&inode->i_lock); 3416 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3417 if (end_pos > nfsi->layout->plh_lwb) 3418 nfsi->layout->plh_lwb = end_pos; 3419 if (status != -ENOSPC) 3420 goto out_unlock; 3421 spin_unlock(&inode->i_lock); 3422 mark_as_dirty = true; 3423 } 3424 } 3425 3426 3427 status = nfs4_proc_layoutcommit(data, sync); 3428 out: 3429 if (status || mark_as_dirty) 3430 mark_inode_dirty_sync(inode); 3431 dprintk("<-- %s status %d\n", __func__, status); 3432 return status; 3433 out_unlock: 3434 spin_unlock(&inode->i_lock); 3435 kfree(data); 3436 clear_layoutcommitting: 3437 pnfs_clear_layoutcommitting(inode); 3438 goto out; 3439 } 3440 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3441 3442 int 3443 pnfs_generic_sync(struct inode *inode, bool datasync) 3444 { 3445 return pnfs_layoutcommit_inode(inode, true); 3446 } 3447 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3448 3449 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3450 { 3451 struct nfs4_threshold *thp; 3452 3453 thp = kzalloc(sizeof(*thp), nfs_io_gfp_mask()); 3454 if (!thp) { 3455 dprintk("%s mdsthreshold allocation failed\n", __func__); 3456 return NULL; 3457 } 3458 return thp; 3459 } 3460 3461 #if IS_ENABLED(CONFIG_NFS_V4_2) 3462 int 3463 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3464 { 3465 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3466 struct nfs_server *server = NFS_SERVER(inode); 3467 struct nfs_inode *nfsi = NFS_I(inode); 3468 struct nfs42_layoutstat_data *data; 3469 struct pnfs_layout_hdr *hdr; 3470 int status = 0; 3471 3472 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3473 goto out; 3474 3475 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3476 goto out; 3477 3478 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3479 goto out; 3480 3481 spin_lock(&inode->i_lock); 3482 if (!NFS_I(inode)->layout) { 3483 spin_unlock(&inode->i_lock); 3484 goto out_clear_layoutstats; 3485 } 3486 hdr = NFS_I(inode)->layout; 3487 pnfs_get_layout_hdr(hdr); 3488 spin_unlock(&inode->i_lock); 3489 3490 data = kzalloc(sizeof(*data), gfp_flags); 3491 if (!data) { 3492 status = -ENOMEM; 3493 goto out_put; 3494 } 3495 3496 data->args.fh = NFS_FH(inode); 3497 data->args.inode = inode; 3498 status = ld->prepare_layoutstats(&data->args); 3499 if (status) 3500 goto out_free; 3501 3502 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3503 3504 out: 3505 dprintk("%s returns %d\n", __func__, status); 3506 return status; 3507 3508 out_free: 3509 kfree(data); 3510 out_put: 3511 pnfs_put_layout_hdr(hdr); 3512 out_clear_layoutstats: 3513 smp_mb__before_atomic(); 3514 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3515 smp_mb__after_atomic(); 3516 goto out; 3517 } 3518 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3519 #endif 3520 3521 unsigned int layoutstats_timer; 3522 module_param(layoutstats_timer, uint, 0644); 3523 EXPORT_SYMBOL_GPL(layoutstats_timer); 3524