1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 436 if (err != -EBUSY) { 437 dst->seqid = lo->plh_stateid.seqid; 438 *dst_range = range; 439 ret = true; 440 } 441 } 442 out: 443 spin_unlock(&inode->i_lock); 444 pnfs_free_lseg_list(&head); 445 return ret; 446 } 447 448 /* 449 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 450 * 451 * In order to continue using the pnfs_layout_hdr, a full recovery 452 * is required. 453 * Note that caller must hold inode->i_lock. 454 */ 455 int 456 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 457 struct list_head *lseg_list) 458 { 459 struct pnfs_layout_range range = { 460 .iomode = IOMODE_ANY, 461 .offset = 0, 462 .length = NFS4_MAX_UINT64, 463 }; 464 struct pnfs_layout_segment *lseg, *next; 465 466 if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 467 return !list_empty(&lo->plh_segs); 468 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); 469 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 470 pnfs_clear_lseg_state(lseg, lseg_list); 471 pnfs_clear_layoutreturn_info(lo); 472 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 473 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 474 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 475 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 476 pnfs_clear_layoutreturn_waitbit(lo); 477 return !list_empty(&lo->plh_segs); 478 } 479 480 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 481 struct list_head *lseg_list, 482 enum pnfs_iomode iomode, u32 seq) 483 { 484 struct pnfs_layout_range range = { 485 .iomode = iomode, 486 .length = NFS4_MAX_UINT64, 487 }; 488 489 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); 490 } 491 492 static int 493 pnfs_iomode_to_fail_bit(u32 iomode) 494 { 495 return iomode == IOMODE_RW ? 496 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 497 } 498 499 static void 500 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 501 { 502 lo->plh_retry_timestamp = jiffies; 503 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 504 refcount_inc(&lo->plh_refcount); 505 } 506 507 static void 508 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 509 { 510 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 511 refcount_dec(&lo->plh_refcount); 512 } 513 514 static void 515 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 516 { 517 struct inode *inode = lo->plh_inode; 518 struct pnfs_layout_range range = { 519 .iomode = iomode, 520 .offset = 0, 521 .length = NFS4_MAX_UINT64, 522 }; 523 LIST_HEAD(head); 524 525 spin_lock(&inode->i_lock); 526 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 527 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); 528 spin_unlock(&inode->i_lock); 529 pnfs_free_lseg_list(&head); 530 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 531 iomode == IOMODE_RW ? "RW" : "READ"); 532 } 533 534 static bool 535 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 536 { 537 unsigned long start, end; 538 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 539 540 if (test_bit(fail_bit, &lo->plh_flags) == 0) 541 return false; 542 end = jiffies; 543 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 544 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 545 /* It is time to retry the failed layoutgets */ 546 pnfs_layout_clear_fail_bit(lo, fail_bit); 547 return false; 548 } 549 return true; 550 } 551 552 static void 553 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 554 const struct pnfs_layout_range *range, 555 const nfs4_stateid *stateid) 556 { 557 INIT_LIST_HEAD(&lseg->pls_list); 558 INIT_LIST_HEAD(&lseg->pls_lc_list); 559 INIT_LIST_HEAD(&lseg->pls_commits); 560 refcount_set(&lseg->pls_refcount, 1); 561 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 562 lseg->pls_layout = lo; 563 lseg->pls_range = *range; 564 lseg->pls_seq = be32_to_cpu(stateid->seqid); 565 } 566 567 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 568 { 569 if (lseg != NULL) { 570 struct inode *inode = lseg->pls_layout->plh_inode; 571 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 572 } 573 } 574 575 static void 576 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 577 struct pnfs_layout_segment *lseg) 578 { 579 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 580 list_del_init(&lseg->pls_list); 581 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 582 refcount_dec(&lo->plh_refcount); 583 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 584 return; 585 if (list_empty(&lo->plh_segs) && 586 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 587 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 588 if (atomic_read(&lo->plh_outstanding) == 0) 589 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 590 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 591 } 592 } 593 594 static bool 595 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 596 struct pnfs_layout_segment *lseg) 597 { 598 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 599 pnfs_layout_is_valid(lo)) { 600 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 601 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 602 return true; 603 } 604 return false; 605 } 606 607 void 608 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 609 { 610 struct pnfs_layout_hdr *lo; 611 struct inode *inode; 612 613 if (!lseg) 614 return; 615 616 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 617 refcount_read(&lseg->pls_refcount), 618 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 619 620 lo = lseg->pls_layout; 621 inode = lo->plh_inode; 622 623 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 624 pnfs_get_layout_hdr(lo); 625 pnfs_layout_remove_lseg(lo, lseg); 626 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 627 lseg = NULL; 628 spin_unlock(&inode->i_lock); 629 pnfs_free_lseg(lseg); 630 pnfs_put_layout_hdr(lo); 631 } 632 } 633 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 634 635 /* 636 * is l2 fully contained in l1? 637 * start1 end1 638 * [----------------------------------) 639 * start2 end2 640 * [----------------) 641 */ 642 static bool 643 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 644 const struct pnfs_layout_range *l2) 645 { 646 u64 start1 = l1->offset; 647 u64 end1 = pnfs_end_offset(start1, l1->length); 648 u64 start2 = l2->offset; 649 u64 end2 = pnfs_end_offset(start2, l2->length); 650 651 return (start1 <= start2) && (end1 >= end2); 652 } 653 654 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 655 struct list_head *tmp_list) 656 { 657 if (!refcount_dec_and_test(&lseg->pls_refcount)) 658 return false; 659 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 660 list_add(&lseg->pls_list, tmp_list); 661 return true; 662 } 663 664 /* Returns 1 if lseg is removed from list, 0 otherwise */ 665 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 666 struct list_head *tmp_list) 667 { 668 int rv = 0; 669 670 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 671 /* Remove the reference keeping the lseg in the 672 * list. It will now be removed when all 673 * outstanding io is finished. 674 */ 675 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 676 refcount_read(&lseg->pls_refcount)); 677 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 678 rv = 1; 679 } 680 return rv; 681 } 682 683 static bool 684 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 685 const struct pnfs_layout_range *recall_range) 686 { 687 return (recall_range->iomode == IOMODE_ANY || 688 lseg_range->iomode == recall_range->iomode) && 689 pnfs_lseg_range_intersecting(lseg_range, recall_range); 690 } 691 692 static bool 693 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 694 const struct pnfs_layout_range *recall_range, 695 u32 seq) 696 { 697 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 698 return false; 699 if (recall_range == NULL) 700 return true; 701 return pnfs_should_free_range(&lseg->pls_range, recall_range); 702 } 703 704 /** 705 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 706 * @lo: layout header containing the lsegs 707 * @tmp_list: list head where doomed lsegs should go 708 * @recall_range: optional recall range argument to match (may be NULL) 709 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 710 * 711 * Walk the list of lsegs in the layout header, and tear down any that should 712 * be destroyed. If "recall_range" is specified then the segment must match 713 * that range. If "seq" is non-zero, then only match segments that were handed 714 * out at or before that sequence. 715 * 716 * Returns number of matching invalid lsegs remaining in list after scanning 717 * it and purging them. 718 */ 719 int 720 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 721 struct list_head *tmp_list, 722 const struct pnfs_layout_range *recall_range, 723 u32 seq) 724 { 725 struct pnfs_layout_segment *lseg, *next; 726 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 727 int remaining = 0; 728 729 dprintk("%s:Begin lo %p\n", __func__, lo); 730 731 if (list_empty(&lo->plh_segs)) 732 return 0; 733 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 734 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 735 dprintk("%s: freeing lseg %p iomode %d seq %u " 736 "offset %llu length %llu\n", __func__, 737 lseg, lseg->pls_range.iomode, lseg->pls_seq, 738 lseg->pls_range.offset, lseg->pls_range.length); 739 if (mark_lseg_invalid(lseg, tmp_list)) 740 continue; 741 remaining++; 742 pnfs_lseg_cancel_io(server, lseg); 743 } 744 dprintk("%s:Return %i\n", __func__, remaining); 745 return remaining; 746 } 747 748 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 749 { 750 struct pnfs_layout_segment *lseg; 751 752 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 753 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 754 } 755 756 static void 757 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 758 struct list_head *free_me, 759 const struct pnfs_layout_range *range, 760 u32 seq) 761 { 762 struct pnfs_layout_segment *lseg, *next; 763 764 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 765 if (pnfs_match_lseg_recall(lseg, range, seq)) 766 list_move_tail(&lseg->pls_list, free_me); 767 } 768 } 769 770 /* note free_me must contain lsegs from a single layout_hdr */ 771 void 772 pnfs_free_lseg_list(struct list_head *free_me) 773 { 774 struct pnfs_layout_segment *lseg, *tmp; 775 776 if (list_empty(free_me)) 777 return; 778 779 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 780 list_del(&lseg->pls_list); 781 pnfs_free_lseg(lseg); 782 } 783 } 784 785 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 786 { 787 struct pnfs_layout_hdr *lo; 788 LIST_HEAD(tmp_list); 789 790 spin_lock(&nfsi->vfs_inode.i_lock); 791 lo = nfsi->layout; 792 if (lo) { 793 pnfs_get_layout_hdr(lo); 794 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 795 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 796 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 797 spin_unlock(&nfsi->vfs_inode.i_lock); 798 pnfs_free_lseg_list(&tmp_list); 799 nfs_commit_inode(&nfsi->vfs_inode, 0); 800 pnfs_put_layout_hdr(lo); 801 } else 802 spin_unlock(&nfsi->vfs_inode.i_lock); 803 return lo; 804 } 805 806 void pnfs_destroy_layout(struct nfs_inode *nfsi) 807 { 808 __pnfs_destroy_layout(nfsi); 809 } 810 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 811 812 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 813 { 814 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 815 struct inode *inode = &nfsi->vfs_inode; 816 817 if (lo) { 818 spin_lock(&inode->i_lock); 819 wait_var_event_spinlock(lo, nfsi->layout != lo, 820 &inode->i_lock); 821 spin_unlock(&inode->i_lock); 822 } 823 } 824 825 static bool 826 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 827 struct list_head *layout_list) 828 { 829 struct pnfs_layout_hdr *lo; 830 bool ret = false; 831 832 spin_lock(&inode->i_lock); 833 lo = NFS_I(inode)->layout; 834 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 835 pnfs_get_layout_hdr(lo); 836 list_add(&lo->plh_bulk_destroy, layout_list); 837 ret = true; 838 } 839 spin_unlock(&inode->i_lock); 840 return ret; 841 } 842 843 /* Caller must hold rcu_read_lock and clp->cl_lock */ 844 static int 845 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 846 struct nfs_server *server, 847 struct list_head *layout_list) 848 __must_hold(&clp->cl_lock) 849 __must_hold(RCU) 850 { 851 struct pnfs_layout_hdr *lo, *next; 852 struct inode *inode; 853 854 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 855 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 856 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 857 !list_empty(&lo->plh_bulk_destroy)) 858 continue; 859 /* If the sb is being destroyed, just bail */ 860 if (!nfs_sb_active(server->super)) 861 break; 862 inode = pnfs_grab_inode_layout_hdr(lo); 863 if (inode != NULL) { 864 if (pnfs_layout_add_bulk_destroy_list(inode, 865 layout_list)) 866 continue; 867 rcu_read_unlock(); 868 spin_unlock(&clp->cl_lock); 869 iput(inode); 870 } else { 871 rcu_read_unlock(); 872 spin_unlock(&clp->cl_lock); 873 } 874 nfs_sb_deactive(server->super); 875 spin_lock(&clp->cl_lock); 876 rcu_read_lock(); 877 return -EAGAIN; 878 } 879 return 0; 880 } 881 882 static int 883 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 884 enum pnfs_layout_destroy_mode mode) 885 { 886 struct pnfs_layout_hdr *lo; 887 struct inode *inode; 888 LIST_HEAD(lseg_list); 889 int ret = 0; 890 891 while (!list_empty(layout_list)) { 892 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 893 plh_bulk_destroy); 894 dprintk("%s freeing layout for inode %llu\n", __func__, 895 lo->plh_inode->i_ino); 896 inode = lo->plh_inode; 897 898 pnfs_layoutcommit_inode(inode, false); 899 900 spin_lock(&inode->i_lock); 901 list_del_init(&lo->plh_bulk_destroy); 902 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 903 pnfs_mark_layout_stateid_return(lo, &lseg_list, 904 IOMODE_ANY, 0); 905 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 906 if (mode == PNFS_LAYOUT_BULK_RETURN) 907 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 908 ret = -EAGAIN; 909 } 910 spin_unlock(&inode->i_lock); 911 pnfs_free_lseg_list(&lseg_list); 912 /* Free all lsegs that are attached to commit buckets */ 913 nfs_commit_inode(inode, 0); 914 pnfs_put_layout_hdr(lo); 915 nfs_iput_and_deactive(inode); 916 } 917 return ret; 918 } 919 920 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 921 enum pnfs_layout_destroy_mode mode) 922 { 923 struct nfs_server *server; 924 LIST_HEAD(layout_list); 925 926 spin_lock(&clp->cl_lock); 927 rcu_read_lock(); 928 restart: 929 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 930 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 931 continue; 932 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 933 server, 934 &layout_list) != 0) 935 goto restart; 936 } 937 rcu_read_unlock(); 938 spin_unlock(&clp->cl_lock); 939 940 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 941 } 942 943 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 944 struct list_head *list) 945 { 946 struct nfs_server *server; 947 948 spin_lock(&clp->cl_lock); 949 rcu_read_lock(); 950 restart: 951 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 952 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 953 list) != 0) 954 goto restart; 955 } 956 rcu_read_unlock(); 957 spin_unlock(&clp->cl_lock); 958 } 959 960 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 961 struct list_head *list, 962 enum pnfs_layout_destroy_mode mode) 963 { 964 pnfs_layout_build_destroy_list_byclient(clp, list); 965 return pnfs_layout_free_bulk_destroy_list(list, mode); 966 } 967 968 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 969 enum pnfs_layout_destroy_mode mode) 970 { 971 LIST_HEAD(layout_list); 972 973 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 974 } 975 976 /* 977 * Called by the state manager to remove all layouts established under an 978 * expired lease. 979 */ 980 void 981 pnfs_destroy_all_layouts(struct nfs_client *clp) 982 { 983 nfs4_deviceid_mark_client_invalid(clp); 984 nfs4_deviceid_purge_client(clp); 985 986 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 987 } 988 989 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 990 struct list_head *list) 991 { 992 struct nfs_server *server; 993 994 spin_lock(&clp->cl_lock); 995 rcu_read_lock(); 996 restart: 997 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 998 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 999 continue; 1000 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 1001 list) != 0) 1002 goto restart; 1003 } 1004 rcu_read_unlock(); 1005 spin_unlock(&clp->cl_lock); 1006 } 1007 1008 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1009 { 1010 struct pnfs_layout_hdr *lo; 1011 struct nfs_server *server; 1012 int ret; 1013 1014 list_for_each_entry(lo, list, plh_bulk_destroy) { 1015 server = NFS_SERVER(lo->plh_inode); 1016 ret = pnfs_layout_return_on_reboot(lo); 1017 switch (ret) { 1018 case 0: 1019 continue; 1020 case -NFS4ERR_BAD_STATEID: 1021 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1022 break; 1023 case -NFS4ERR_NO_GRACE: 1024 break; 1025 default: 1026 goto err; 1027 } 1028 break; 1029 } 1030 return 0; 1031 err: 1032 return ret; 1033 } 1034 1035 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1036 { 1037 LIST_HEAD(list); 1038 int ret = 0, ret2; 1039 1040 pnfs_layout_build_recover_list_byclient(clp, &list); 1041 if (!list_empty(&list)) 1042 ret = pnfs_layout_bulk_list_reboot(&list); 1043 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1044 PNFS_LAYOUT_INVALIDATE); 1045 if (!ret) 1046 ret = ret2; 1047 return (ret == 0) ? 0 : -EAGAIN; 1048 } 1049 1050 static void 1051 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1052 { 1053 const struct cred *old; 1054 1055 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1056 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1057 put_cred(old); 1058 } 1059 } 1060 1061 /* update lo->plh_stateid with new if is more recent */ 1062 void 1063 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1064 const struct cred *cred, bool update_barrier) 1065 { 1066 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1067 u32 newseq = be32_to_cpu(new->seqid); 1068 1069 if (!pnfs_layout_is_valid(lo)) { 1070 pnfs_set_layout_cred(lo, cred); 1071 nfs4_stateid_copy(&lo->plh_stateid, new); 1072 lo->plh_barrier = newseq; 1073 pnfs_clear_layoutreturn_info(lo); 1074 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1075 return; 1076 } 1077 1078 if (pnfs_seqid_is_newer(newseq, oldseq)) 1079 nfs4_stateid_copy(&lo->plh_stateid, new); 1080 1081 if (update_barrier) { 1082 pnfs_barrier_update(lo, newseq); 1083 return; 1084 } 1085 /* 1086 * Because of wraparound, we want to keep the barrier 1087 * "close" to the current seqids. We really only want to 1088 * get here from a layoutget call. 1089 */ 1090 if (atomic_read(&lo->plh_outstanding) == 1) 1091 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1092 } 1093 1094 static bool 1095 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1096 const nfs4_stateid *stateid) 1097 { 1098 u32 seqid = be32_to_cpu(stateid->seqid); 1099 1100 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1101 } 1102 1103 /* lget is set to 1 if called from inside send_layoutget call chain */ 1104 static bool 1105 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1106 { 1107 return lo->plh_block_lgets || 1108 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1109 } 1110 1111 static struct nfs_server * 1112 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1113 { 1114 struct nfs_server *server; 1115 1116 if (inode) { 1117 server = NFS_SERVER(inode); 1118 } else { 1119 struct dentry *parent_dir = dget_parent(ctx->dentry); 1120 server = NFS_SERVER(parent_dir->d_inode); 1121 dput(parent_dir); 1122 } 1123 return server; 1124 } 1125 1126 static void nfs4_free_pages(struct page **pages, size_t size) 1127 { 1128 int i; 1129 1130 if (!pages) 1131 return; 1132 1133 for (i = 0; i < size; i++) { 1134 if (!pages[i]) 1135 break; 1136 __free_page(pages[i]); 1137 } 1138 kfree(pages); 1139 } 1140 1141 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1142 { 1143 struct page **pages; 1144 int i; 1145 1146 pages = kmalloc_objs(struct page *, size, gfp_flags); 1147 if (!pages) { 1148 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1149 return NULL; 1150 } 1151 1152 for (i = 0; i < size; i++) { 1153 pages[i] = alloc_page(gfp_flags); 1154 if (!pages[i]) { 1155 dprintk("%s: failed to allocate page\n", __func__); 1156 nfs4_free_pages(pages, i); 1157 return NULL; 1158 } 1159 } 1160 1161 return pages; 1162 } 1163 1164 static struct nfs4_layoutget * 1165 pnfs_alloc_init_layoutget_args(struct inode *ino, 1166 struct nfs_open_context *ctx, 1167 const nfs4_stateid *stateid, 1168 const struct pnfs_layout_range *range, 1169 gfp_t gfp_flags) 1170 { 1171 struct nfs_server *server = pnfs_find_server(ino, ctx); 1172 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1173 size_t max_pages = max_response_pages(server); 1174 struct nfs4_layoutget *lgp; 1175 1176 dprintk("--> %s\n", __func__); 1177 1178 lgp = kzalloc_obj(*lgp, gfp_flags); 1179 if (lgp == NULL) 1180 return NULL; 1181 1182 if (max_reply_sz) { 1183 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1184 if (npages < max_pages) 1185 max_pages = npages; 1186 } 1187 1188 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1189 if (!lgp->args.layout.pages) { 1190 kfree(lgp); 1191 return NULL; 1192 } 1193 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1194 lgp->res.layoutp = &lgp->args.layout; 1195 1196 /* Don't confuse uninitialised result and success */ 1197 lgp->res.status = -NFS4ERR_DELAY; 1198 1199 lgp->args.minlength = PAGE_SIZE; 1200 if (lgp->args.minlength > range->length) 1201 lgp->args.minlength = range->length; 1202 if (ino) { 1203 loff_t i_size = i_size_read(ino); 1204 1205 if (range->iomode == IOMODE_READ) { 1206 if (range->offset >= i_size) 1207 lgp->args.minlength = 0; 1208 else if (i_size - range->offset < lgp->args.minlength) 1209 lgp->args.minlength = i_size - range->offset; 1210 } 1211 } 1212 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 1213 pnfs_copy_range(&lgp->args.range, range); 1214 lgp->args.type = server->pnfs_curr_ld->id; 1215 lgp->args.inode = ino; 1216 lgp->args.ctx = get_nfs_open_context(ctx); 1217 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1218 lgp->gfp_flags = gfp_flags; 1219 lgp->cred = ctx->cred; 1220 return lgp; 1221 } 1222 1223 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1224 { 1225 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1226 1227 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1228 pnfs_put_layout_hdr(lgp->lo); 1229 put_nfs_open_context(lgp->args.ctx); 1230 kfree(lgp); 1231 } 1232 1233 static void pnfs_clear_layoutcommit(struct inode *inode, 1234 struct list_head *head) 1235 { 1236 struct nfs_inode *nfsi = NFS_I(inode); 1237 struct pnfs_layout_segment *lseg, *tmp; 1238 1239 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1240 return; 1241 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1242 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1243 continue; 1244 pnfs_lseg_dec_and_remove_zero(lseg, head); 1245 } 1246 } 1247 1248 static void 1249 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1250 const nfs4_stateid *arg_stateid, 1251 const struct pnfs_layout_range *range, 1252 struct list_head *freeme) 1253 { 1254 if (pnfs_layout_is_valid(lo) && 1255 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1256 pnfs_reset_return_info(lo); 1257 else 1258 pnfs_mark_layout_stateid_invalid(lo, freeme); 1259 pnfs_clear_layoutreturn_waitbit(lo); 1260 } 1261 1262 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1263 const nfs4_stateid *arg_stateid, 1264 const struct pnfs_layout_range *range) 1265 { 1266 struct inode *inode = lo->plh_inode; 1267 LIST_HEAD(freeme); 1268 1269 spin_lock(&inode->i_lock); 1270 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1271 spin_unlock(&inode->i_lock); 1272 pnfs_free_lseg_list(&freeme); 1273 } 1274 1275 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1276 const nfs4_stateid *arg_stateid, 1277 const struct pnfs_layout_range *range, 1278 const nfs4_stateid *stateid) 1279 { 1280 struct inode *inode = lo->plh_inode; 1281 LIST_HEAD(freeme); 1282 1283 spin_lock(&inode->i_lock); 1284 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1285 goto out_unlock; 1286 if (stateid && pnfs_layout_is_valid(lo)) { 1287 u32 seq = be32_to_cpu(arg_stateid->seqid); 1288 1289 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1290 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1291 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1292 pnfs_reset_return_info(lo); 1293 } else 1294 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1295 out_unlock: 1296 pnfs_clear_layoutreturn_waitbit(lo); 1297 spin_unlock(&inode->i_lock); 1298 pnfs_free_lseg_list(&freeme); 1299 1300 } 1301 1302 static bool 1303 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1304 nfs4_stateid *stateid, 1305 const struct cred **cred, 1306 enum pnfs_iomode *iomode) 1307 { 1308 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1309 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1310 return false; 1311 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1312 return false; 1313 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1314 pnfs_get_layout_hdr(lo); 1315 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1316 *cred = get_cred(lo->plh_lc_cred); 1317 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1318 if (lo->plh_return_seq != 0) 1319 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1320 if (iomode != NULL) 1321 *iomode = lo->plh_return_iomode; 1322 pnfs_clear_layoutreturn_info(lo); 1323 } else if (iomode != NULL) 1324 *iomode = IOMODE_ANY; 1325 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1326 return true; 1327 } 1328 1329 static void 1330 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1331 struct pnfs_layout_hdr *lo, 1332 const nfs4_stateid *stateid, 1333 enum pnfs_iomode iomode) 1334 { 1335 struct inode *inode = lo->plh_inode; 1336 1337 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1338 args->inode = inode; 1339 args->range.iomode = iomode; 1340 args->range.offset = 0; 1341 args->range.length = NFS4_MAX_UINT64; 1342 args->layout = lo; 1343 nfs4_stateid_copy(&args->stateid, stateid); 1344 } 1345 1346 static int 1347 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1348 const nfs4_stateid *stateid, 1349 const struct cred **pcred, 1350 enum pnfs_iomode iomode, 1351 unsigned int flags) 1352 { 1353 struct inode *ino = lo->plh_inode; 1354 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1355 struct nfs4_layoutreturn *lrp; 1356 const struct cred *cred = *pcred; 1357 int status = 0; 1358 1359 *pcred = NULL; 1360 lrp = kzalloc_obj(*lrp, nfs_io_gfp_mask()); 1361 if (unlikely(lrp == NULL)) { 1362 status = -ENOMEM; 1363 spin_lock(&ino->i_lock); 1364 pnfs_clear_layoutreturn_waitbit(lo); 1365 spin_unlock(&ino->i_lock); 1366 put_cred(cred); 1367 pnfs_put_layout_hdr(lo); 1368 goto out; 1369 } 1370 1371 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1372 lrp->args.ld_private = &lrp->ld_private; 1373 lrp->clp = NFS_SERVER(ino)->nfs_client; 1374 lrp->cred = cred; 1375 if (ld->prepare_layoutreturn) 1376 ld->prepare_layoutreturn(&lrp->args); 1377 1378 status = nfs4_proc_layoutreturn(lrp, flags); 1379 out: 1380 dprintk("<-- %s status: %d\n", __func__, status); 1381 return status; 1382 } 1383 1384 /* Return true if layoutreturn is needed */ 1385 static bool 1386 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1387 { 1388 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1389 return false; 1390 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1391 lo->plh_return_iomode, 1392 lo->plh_return_seq) != EBUSY; 1393 } 1394 1395 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1396 { 1397 struct inode *inode= lo->plh_inode; 1398 1399 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1400 return; 1401 spin_lock(&inode->i_lock); 1402 if (pnfs_layout_need_return(lo)) { 1403 const struct cred *cred; 1404 nfs4_stateid stateid; 1405 enum pnfs_iomode iomode; 1406 bool send; 1407 1408 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1409 spin_unlock(&inode->i_lock); 1410 if (send) { 1411 /* Send an async layoutreturn so we dont deadlock */ 1412 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1413 PNFS_FL_LAYOUTRETURN_ASYNC); 1414 } 1415 } else 1416 spin_unlock(&inode->i_lock); 1417 } 1418 1419 /* 1420 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1421 * when the layout segment list is empty. 1422 * 1423 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1424 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1425 * deviceid is marked invalid. 1426 */ 1427 int 1428 _pnfs_return_layout(struct inode *ino) 1429 { 1430 struct pnfs_layout_hdr *lo = NULL; 1431 struct nfs_inode *nfsi = NFS_I(ino); 1432 struct pnfs_layout_range range = { 1433 .iomode = IOMODE_ANY, 1434 .offset = 0, 1435 .length = NFS4_MAX_UINT64, 1436 }; 1437 LIST_HEAD(tmp_list); 1438 const struct cred *cred; 1439 nfs4_stateid stateid; 1440 int status = 0; 1441 bool send, valid_layout; 1442 1443 dprintk("NFS: %s for inode %llu\n", __func__, ino->i_ino); 1444 1445 spin_lock(&ino->i_lock); 1446 lo = nfsi->layout; 1447 if (!lo) { 1448 spin_unlock(&ino->i_lock); 1449 dprintk("NFS: %s no layout to return\n", __func__); 1450 goto out; 1451 } 1452 /* Reference matched in nfs4_layoutreturn_release */ 1453 pnfs_get_layout_hdr(lo); 1454 /* Is there an outstanding layoutreturn ? */ 1455 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1456 spin_unlock(&ino->i_lock); 1457 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1458 TASK_UNINTERRUPTIBLE)) 1459 goto out_put_layout_hdr; 1460 spin_lock(&ino->i_lock); 1461 } 1462 valid_layout = pnfs_layout_is_valid(lo); 1463 pnfs_clear_layoutcommit(ino, &tmp_list); 1464 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); 1465 1466 1467 /* Don't send a LAYOUTRETURN if list was initially empty */ 1468 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1469 !valid_layout) { 1470 spin_unlock(&ino->i_lock); 1471 dprintk("NFS: %s no layout segments to return\n", __func__); 1472 goto out_wait_layoutreturn; 1473 } 1474 1475 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1476 spin_unlock(&ino->i_lock); 1477 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1478 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1479 if (send) 1480 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1481 0); 1482 out_wait_layoutreturn: 1483 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1484 out_put_layout_hdr: 1485 pnfs_free_lseg_list(&tmp_list); 1486 pnfs_put_layout_hdr(lo); 1487 out: 1488 dprintk("<-- %s status: %d\n", __func__, status); 1489 return status; 1490 } 1491 1492 int 1493 pnfs_commit_and_return_layout(struct inode *inode) 1494 { 1495 struct pnfs_layout_hdr *lo; 1496 int ret; 1497 1498 spin_lock(&inode->i_lock); 1499 lo = NFS_I(inode)->layout; 1500 if (lo == NULL) { 1501 spin_unlock(&inode->i_lock); 1502 return 0; 1503 } 1504 pnfs_get_layout_hdr(lo); 1505 /* Block new layoutgets and read/write to ds */ 1506 lo->plh_block_lgets++; 1507 spin_unlock(&inode->i_lock); 1508 filemap_fdatawait(inode->i_mapping); 1509 ret = pnfs_layoutcommit_inode(inode, true); 1510 if (ret == 0) 1511 ret = _pnfs_return_layout(inode); 1512 spin_lock(&inode->i_lock); 1513 lo->plh_block_lgets--; 1514 spin_unlock(&inode->i_lock); 1515 pnfs_put_layout_hdr(lo); 1516 return ret; 1517 } 1518 1519 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1520 { 1521 struct inode *inode = lo->plh_inode; 1522 const struct cred *cred; 1523 1524 spin_lock(&inode->i_lock); 1525 if (!pnfs_layout_is_valid(lo)) { 1526 spin_unlock(&inode->i_lock); 1527 return 0; 1528 } 1529 cred = get_cred(lo->plh_lc_cred); 1530 pnfs_get_layout_hdr(lo); 1531 spin_unlock(&inode->i_lock); 1532 1533 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1534 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1535 } 1536 1537 bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, 1538 struct nfs4_layoutreturn_res *res, const struct cred *cred, 1539 bool sync) 1540 { 1541 struct nfs_inode *nfsi = NFS_I(ino); 1542 struct nfs_open_context *ctx; 1543 struct nfs4_state *state; 1544 struct pnfs_layout_hdr *lo; 1545 struct pnfs_layout_segment *lseg, *next; 1546 const struct cred *lc_cred; 1547 nfs4_stateid stateid; 1548 enum pnfs_iomode iomode = 0; 1549 bool layoutreturn = false, roc = false; 1550 bool skip_read; 1551 1552 if (!nfs_have_layout(ino)) 1553 return false; 1554 retry: 1555 rcu_read_lock(); 1556 spin_lock(&ino->i_lock); 1557 lo = nfsi->layout; 1558 if (!lo || !pnfs_layout_is_valid(lo) || 1559 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1560 lo = NULL; 1561 goto out_noroc; 1562 } 1563 1564 /* no roc if we hold a delegation */ 1565 skip_read = false; 1566 if (nfs4_check_delegation(ino, FMODE_READ)) { 1567 if (nfs4_check_delegation(ino, FMODE_WRITE)) { 1568 lo = NULL; 1569 goto out_noroc; 1570 } 1571 skip_read = true; 1572 } 1573 1574 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1575 state = ctx->state; 1576 if (state == NULL) 1577 continue; 1578 /* Don't return layout if there is open file state */ 1579 if (state->state & FMODE_WRITE) { 1580 lo = NULL; 1581 goto out_noroc; 1582 } 1583 if (state->state & FMODE_READ) 1584 skip_read = true; 1585 } 1586 1587 if (skip_read) { 1588 bool writes = false; 1589 1590 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1591 if (lseg->pls_range.iomode != IOMODE_READ) { 1592 writes = true; 1593 break; 1594 } 1595 } 1596 if (!writes) { 1597 lo = NULL; 1598 goto out_noroc; 1599 } 1600 } 1601 1602 pnfs_get_layout_hdr(lo); 1603 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1604 if (!sync) { 1605 pnfs_set_plh_return_info( 1606 lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); 1607 goto out_noroc; 1608 } 1609 spin_unlock(&ino->i_lock); 1610 rcu_read_unlock(); 1611 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1612 TASK_UNINTERRUPTIBLE); 1613 pnfs_put_layout_hdr(lo); 1614 goto retry; 1615 } 1616 1617 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1618 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1619 continue; 1620 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1621 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1622 continue; 1623 /* 1624 * Note: mark lseg for return so pnfs_layout_remove_lseg 1625 * doesn't invalidate the layout for us. 1626 */ 1627 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1628 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1629 continue; 1630 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1631 } 1632 1633 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1634 goto out_noroc; 1635 1636 /* ROC in two conditions: 1637 * 1. there are ROC lsegs 1638 * 2. we don't send layoutreturn 1639 */ 1640 /* lo ref dropped in pnfs_roc_release() */ 1641 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1642 /* If the creds don't match, we can't compound the layoutreturn */ 1643 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1644 goto out_noroc; 1645 1646 roc = layoutreturn; 1647 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1648 res->lrs_present = 0; 1649 layoutreturn = false; 1650 put_cred(lc_cred); 1651 1652 out_noroc: 1653 spin_unlock(&ino->i_lock); 1654 rcu_read_unlock(); 1655 pnfs_layoutcommit_inode(ino, sync); 1656 if (roc) { 1657 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1658 if (ld->prepare_layoutreturn) 1659 ld->prepare_layoutreturn(args); 1660 pnfs_put_layout_hdr(lo); 1661 return true; 1662 } 1663 if (layoutreturn) 1664 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1665 pnfs_put_layout_hdr(lo); 1666 return false; 1667 } 1668 1669 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1670 struct nfs4_layoutreturn_res **respp, int *ret) 1671 { 1672 struct nfs4_layoutreturn_args *arg = *argpp; 1673 int retval = -EAGAIN; 1674 1675 if (!arg) 1676 return 0; 1677 /* Handle Layoutreturn errors */ 1678 switch (*ret) { 1679 case 0: 1680 retval = 0; 1681 break; 1682 case -NFS4ERR_NOMATCHING_LAYOUT: 1683 /* Was there an RPC level error? If not, retry */ 1684 if (task->tk_rpc_status == 0) 1685 break; 1686 /* 1687 * Is there a fatal network level error? 1688 * If so release the layout, but flag the error. 1689 */ 1690 if ((task->tk_rpc_status == -ENETDOWN || 1691 task->tk_rpc_status == -ENETUNREACH) && 1692 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1693 *ret = 0; 1694 (*respp)->lrs_present = 0; 1695 retval = -EIO; 1696 break; 1697 } 1698 /* If the call was not sent, let caller handle it */ 1699 if (!RPC_WAS_SENT(task)) 1700 return 0; 1701 switch (task->tk_rpc_status) { 1702 default: 1703 /* 1704 * Defer the layoutreturn if it was due 1705 * to the server being down. 1706 */ 1707 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1708 break; 1709 case -EACCES: 1710 case -EIO: 1711 case -EKEYEXPIRED: 1712 case -ERESTARTSYS: 1713 case -EINTR: 1714 /* Don't retry */ 1715 *ret = 0; 1716 break; 1717 } 1718 (*respp)->lrs_present = 0; 1719 retval = 0; 1720 break; 1721 case -NFS4ERR_DELAY: 1722 /* Let the caller handle the retry */ 1723 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1724 return 0; 1725 case -NFS4ERR_OLD_STATEID: 1726 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1727 &arg->range, arg->inode)) 1728 break; 1729 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1730 return -EAGAIN; 1731 } 1732 *argpp = NULL; 1733 *respp = NULL; 1734 return retval; 1735 } 1736 1737 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1738 struct nfs4_layoutreturn_res *res, int ret) 1739 { 1740 struct pnfs_layout_hdr *lo = args->layout; 1741 struct inode *inode = args->inode; 1742 const nfs4_stateid *res_stateid = NULL; 1743 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1744 LIST_HEAD(freeme); 1745 1746 switch (ret) { 1747 case -NFS4ERR_BADSESSION: 1748 case -NFS4ERR_DEADSESSION: 1749 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1750 case -NFS4ERR_NOMATCHING_LAYOUT: 1751 spin_lock(&inode->i_lock); 1752 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1753 &args->range, &freeme); 1754 spin_unlock(&inode->i_lock); 1755 pnfs_free_lseg_list(&freeme); 1756 break; 1757 case 0: 1758 if (res->lrs_present) 1759 res_stateid = &res->stateid; 1760 fallthrough; 1761 default: 1762 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1763 res_stateid); 1764 } 1765 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1766 if (ld_private && ld_private->ops && ld_private->ops->free) 1767 ld_private->ops->free(ld_private); 1768 pnfs_put_layout_hdr(lo); 1769 } 1770 1771 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1772 { 1773 struct nfs_inode *nfsi = NFS_I(ino); 1774 struct pnfs_layout_hdr *lo; 1775 bool sleep = false; 1776 1777 /* we might not have grabbed lo reference. so need to check under 1778 * i_lock */ 1779 spin_lock(&ino->i_lock); 1780 lo = nfsi->layout; 1781 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1782 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1783 sleep = true; 1784 } 1785 spin_unlock(&ino->i_lock); 1786 return sleep; 1787 } 1788 1789 /* 1790 * Compare two layout segments for sorting into layout cache. 1791 * We want to preferentially return RW over RO layouts, so ensure those 1792 * are seen first. 1793 */ 1794 static s64 1795 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1796 const struct pnfs_layout_range *l2) 1797 { 1798 s64 d; 1799 1800 /* high offset > low offset */ 1801 d = l1->offset - l2->offset; 1802 if (d) 1803 return d; 1804 1805 /* short length > long length */ 1806 d = l2->length - l1->length; 1807 if (d) 1808 return d; 1809 1810 /* read > read/write */ 1811 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1812 } 1813 1814 static bool 1815 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1816 const struct pnfs_layout_range *l2) 1817 { 1818 return pnfs_lseg_range_cmp(l1, l2) > 0; 1819 } 1820 1821 static bool 1822 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1823 struct pnfs_layout_segment *old) 1824 { 1825 return false; 1826 } 1827 1828 void 1829 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1830 struct pnfs_layout_segment *lseg, 1831 bool (*is_after)(const struct pnfs_layout_range *, 1832 const struct pnfs_layout_range *), 1833 bool (*do_merge)(struct pnfs_layout_segment *, 1834 struct pnfs_layout_segment *), 1835 struct list_head *free_me) 1836 { 1837 struct pnfs_layout_segment *lp, *tmp; 1838 1839 dprintk("%s:Begin\n", __func__); 1840 1841 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1842 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1843 continue; 1844 if (do_merge(lseg, lp)) { 1845 mark_lseg_invalid(lp, free_me); 1846 continue; 1847 } 1848 if (is_after(&lseg->pls_range, &lp->pls_range)) 1849 continue; 1850 list_add_tail(&lseg->pls_list, &lp->pls_list); 1851 dprintk("%s: inserted lseg %p " 1852 "iomode %d offset %llu length %llu before " 1853 "lp %p iomode %d offset %llu length %llu\n", 1854 __func__, lseg, lseg->pls_range.iomode, 1855 lseg->pls_range.offset, lseg->pls_range.length, 1856 lp, lp->pls_range.iomode, lp->pls_range.offset, 1857 lp->pls_range.length); 1858 goto out; 1859 } 1860 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1861 dprintk("%s: inserted lseg %p " 1862 "iomode %d offset %llu length %llu at tail\n", 1863 __func__, lseg, lseg->pls_range.iomode, 1864 lseg->pls_range.offset, lseg->pls_range.length); 1865 out: 1866 pnfs_get_layout_hdr(lo); 1867 1868 dprintk("%s:Return\n", __func__); 1869 } 1870 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1871 1872 static void 1873 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1874 struct pnfs_layout_segment *lseg, 1875 struct list_head *free_me) 1876 { 1877 struct inode *inode = lo->plh_inode; 1878 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1879 1880 if (ld->add_lseg != NULL) 1881 ld->add_lseg(lo, lseg, free_me); 1882 else 1883 pnfs_generic_layout_insert_lseg(lo, lseg, 1884 pnfs_lseg_range_is_after, 1885 pnfs_lseg_no_merge, 1886 free_me); 1887 } 1888 1889 static struct pnfs_layout_hdr * 1890 alloc_init_layout_hdr(struct inode *ino, 1891 struct nfs_open_context *ctx, 1892 gfp_t gfp_flags) 1893 { 1894 struct pnfs_layout_hdr *lo; 1895 1896 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1897 if (!lo) 1898 return NULL; 1899 refcount_set(&lo->plh_refcount, 1); 1900 INIT_LIST_HEAD(&lo->plh_layouts); 1901 INIT_LIST_HEAD(&lo->plh_segs); 1902 INIT_LIST_HEAD(&lo->plh_return_segs); 1903 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1904 lo->plh_inode = ino; 1905 lo->plh_lc_cred = get_cred(ctx->cred); 1906 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1907 return lo; 1908 } 1909 1910 static struct pnfs_layout_hdr * 1911 pnfs_find_alloc_layout(struct inode *ino, 1912 struct nfs_open_context *ctx, 1913 gfp_t gfp_flags) 1914 __releases(&ino->i_lock) 1915 __acquires(&ino->i_lock) 1916 { 1917 struct nfs_inode *nfsi = NFS_I(ino); 1918 struct pnfs_layout_hdr *new = NULL; 1919 1920 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1921 1922 if (nfsi->layout != NULL) 1923 goto out_existing; 1924 spin_unlock(&ino->i_lock); 1925 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1926 spin_lock(&ino->i_lock); 1927 1928 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1929 nfsi->layout = new; 1930 return new; 1931 } else if (new != NULL) 1932 pnfs_free_layout_hdr(new); 1933 out_existing: 1934 pnfs_get_layout_hdr(nfsi->layout); 1935 return nfsi->layout; 1936 } 1937 1938 /* 1939 * iomode matching rules: 1940 * iomode lseg strict match 1941 * iomode 1942 * ----- ----- ------ ----- 1943 * ANY READ N/A true 1944 * ANY RW N/A true 1945 * RW READ N/A false 1946 * RW RW N/A true 1947 * READ READ N/A true 1948 * READ RW true false 1949 * READ RW false true 1950 */ 1951 static bool 1952 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1953 const struct pnfs_layout_range *range, 1954 bool strict_iomode) 1955 { 1956 struct pnfs_layout_range range1; 1957 1958 if ((range->iomode == IOMODE_RW && 1959 ls_range->iomode != IOMODE_RW) || 1960 (range->iomode != ls_range->iomode && 1961 strict_iomode) || 1962 !pnfs_lseg_range_intersecting(ls_range, range)) 1963 return false; 1964 1965 /* range1 covers only the first byte in the range */ 1966 range1 = *range; 1967 range1.length = 1; 1968 return pnfs_lseg_range_contained(ls_range, &range1); 1969 } 1970 1971 /* 1972 * lookup range in layout 1973 */ 1974 static struct pnfs_layout_segment * 1975 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1976 struct pnfs_layout_range *range, 1977 bool strict_iomode) 1978 { 1979 struct pnfs_layout_segment *lseg, *ret = NULL; 1980 1981 dprintk("%s:Begin\n", __func__); 1982 1983 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1984 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1985 pnfs_lseg_range_match(&lseg->pls_range, range, 1986 strict_iomode)) { 1987 ret = pnfs_get_lseg(lseg); 1988 break; 1989 } 1990 } 1991 1992 dprintk("%s:Return lseg %p ref %d\n", 1993 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 1994 return ret; 1995 } 1996 1997 /* 1998 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1999 * to the MDS or over pNFS 2000 * 2001 * The nfs_inode read_io and write_io fields are cumulative counters reset 2002 * when there are no layout segments. Note that in pnfs_update_layout iomode 2003 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 2004 * WRITE request. 2005 * 2006 * A return of true means use MDS I/O. 2007 * 2008 * From rfc 5661: 2009 * If a file's size is smaller than the file size threshold, data accesses 2010 * SHOULD be sent to the metadata server. If an I/O request has a length that 2011 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 2012 * server. If both file size and I/O size are provided, the client SHOULD 2013 * reach or exceed both thresholds before sending its read or write 2014 * requests to the data server. 2015 */ 2016 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 2017 struct inode *ino, int iomode) 2018 { 2019 struct nfs4_threshold *t = ctx->mdsthreshold; 2020 struct nfs_inode *nfsi = NFS_I(ino); 2021 loff_t fsize = i_size_read(ino); 2022 bool size = false, size_set = false, io = false, io_set = false, ret = false; 2023 2024 if (t == NULL) 2025 return ret; 2026 2027 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 2028 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 2029 2030 switch (iomode) { 2031 case IOMODE_READ: 2032 if (t->bm & THRESHOLD_RD) { 2033 dprintk("%s fsize %llu\n", __func__, fsize); 2034 size_set = true; 2035 if (fsize < t->rd_sz) 2036 size = true; 2037 } 2038 if (t->bm & THRESHOLD_RD_IO) { 2039 dprintk("%s nfsi->read_io %llu\n", __func__, 2040 nfsi->read_io); 2041 io_set = true; 2042 if (nfsi->read_io < t->rd_io_sz) 2043 io = true; 2044 } 2045 break; 2046 case IOMODE_RW: 2047 if (t->bm & THRESHOLD_WR) { 2048 dprintk("%s fsize %llu\n", __func__, fsize); 2049 size_set = true; 2050 if (fsize < t->wr_sz) 2051 size = true; 2052 } 2053 if (t->bm & THRESHOLD_WR_IO) { 2054 dprintk("%s nfsi->write_io %llu\n", __func__, 2055 nfsi->write_io); 2056 io_set = true; 2057 if (nfsi->write_io < t->wr_io_sz) 2058 io = true; 2059 } 2060 break; 2061 } 2062 if (size_set && io_set) { 2063 if (size && io) 2064 ret = true; 2065 } else if (size || io) 2066 ret = true; 2067 2068 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2069 return ret; 2070 } 2071 2072 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2073 { 2074 /* 2075 * send layoutcommit as it can hold up layoutreturn due to lseg 2076 * reference 2077 */ 2078 pnfs_layoutcommit_inode(lo->plh_inode, false); 2079 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2080 nfs_wait_bit_killable, 2081 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2082 } 2083 2084 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2085 { 2086 atomic_inc(&lo->plh_outstanding); 2087 } 2088 2089 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2090 { 2091 if (atomic_dec_and_test(&lo->plh_outstanding) && 2092 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2093 smp_mb__after_atomic(); 2094 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2095 } 2096 } 2097 2098 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2099 { 2100 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2101 } 2102 2103 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 2104 { 2105 unsigned long *bitlock = &lo->plh_flags; 2106 2107 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 2108 smp_mb__after_atomic(); 2109 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 2110 } 2111 2112 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2113 struct nfs_server *server) 2114 { 2115 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2116 struct nfs_client *clp = server->nfs_client; 2117 2118 /* The lo must be on the clp list if there is any 2119 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2120 */ 2121 spin_lock(&clp->cl_lock); 2122 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2123 spin_unlock(&clp->cl_lock); 2124 } 2125 } 2126 2127 /* 2128 * Layout segment is retreived from the server if not cached. 2129 * The appropriate layout segment is referenced and returned to the caller. 2130 */ 2131 struct pnfs_layout_segment * 2132 pnfs_update_layout(struct inode *ino, 2133 struct nfs_open_context *ctx, 2134 loff_t pos, 2135 u64 count, 2136 enum pnfs_iomode iomode, 2137 bool strict_iomode, 2138 gfp_t gfp_flags) 2139 { 2140 struct pnfs_layout_range arg = { 2141 .iomode = iomode, 2142 .offset = pos, 2143 .length = count, 2144 }; 2145 unsigned pg_offset; 2146 struct nfs_server *server = NFS_SERVER(ino); 2147 struct nfs_client *clp = server->nfs_client; 2148 struct pnfs_layout_hdr *lo = NULL; 2149 struct pnfs_layout_segment *lseg = NULL; 2150 struct nfs4_layoutget *lgp; 2151 nfs4_stateid stateid; 2152 struct nfs4_exception exception = { 2153 .inode = ino, 2154 }; 2155 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2156 bool first; 2157 2158 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2159 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2160 PNFS_UPDATE_LAYOUT_NO_PNFS); 2161 goto out; 2162 } 2163 2164 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2165 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2166 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2167 goto out; 2168 } 2169 2170 lookup_again: 2171 if (!nfs4_valid_open_stateid(ctx->state)) { 2172 trace_pnfs_update_layout(ino, pos, count, 2173 iomode, lo, lseg, 2174 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2175 lseg = ERR_PTR(-EIO); 2176 goto out; 2177 } 2178 2179 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2180 if (IS_ERR(lseg)) 2181 goto out; 2182 first = false; 2183 spin_lock(&ino->i_lock); 2184 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2185 if (lo == NULL) { 2186 spin_unlock(&ino->i_lock); 2187 lseg = ERR_PTR(-ENOMEM); 2188 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2189 PNFS_UPDATE_LAYOUT_NOMEM); 2190 goto out; 2191 } 2192 2193 /* Do we even need to bother with this? */ 2194 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2195 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2196 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2197 dprintk("%s matches recall, use MDS\n", __func__); 2198 goto out_unlock; 2199 } 2200 2201 /* if LAYOUTGET already failed once we don't try again */ 2202 if (pnfs_layout_io_test_failed(lo, iomode)) { 2203 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2204 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2205 goto out_unlock; 2206 } 2207 2208 /* 2209 * If the layout segment list is empty, but there are outstanding 2210 * layoutget calls, then they might be subject to a layoutrecall. 2211 */ 2212 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2213 atomic_read(&lo->plh_outstanding) != 0) { 2214 spin_unlock(&ino->i_lock); 2215 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2216 TASK_KILLABLE)); 2217 if (IS_ERR(lseg)) 2218 goto out_put_layout_hdr; 2219 pnfs_put_layout_hdr(lo); 2220 goto lookup_again; 2221 } 2222 2223 /* 2224 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2225 * for LAYOUTRETURN. 2226 */ 2227 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2228 spin_unlock(&ino->i_lock); 2229 dprintk("%s wait for layoutreturn\n", __func__); 2230 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2231 if (!IS_ERR(lseg)) { 2232 dprintk("%s retrying\n", __func__); 2233 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2234 lseg, 2235 PNFS_UPDATE_LAYOUT_RETRY); 2236 pnfs_put_layout_hdr(lo); 2237 goto lookup_again; 2238 } 2239 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2240 PNFS_UPDATE_LAYOUT_RETURN); 2241 goto out_put_layout_hdr; 2242 } 2243 2244 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2245 if (lseg) { 2246 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2247 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2248 goto out_unlock; 2249 } 2250 2251 /* 2252 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2253 * stateid, or it has been invalidated, then we must use the open 2254 * stateid. 2255 */ 2256 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2257 int status; 2258 2259 /* 2260 * The first layoutget for the file. Need to serialize per 2261 * RFC 5661 Errata 3208. 2262 */ 2263 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2264 &lo->plh_flags)) { 2265 spin_unlock(&ino->i_lock); 2266 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2267 NFS_LAYOUT_FIRST_LAYOUTGET, 2268 TASK_KILLABLE)); 2269 if (IS_ERR(lseg)) 2270 goto out_put_layout_hdr; 2271 pnfs_put_layout_hdr(lo); 2272 dprintk("%s retrying\n", __func__); 2273 goto lookup_again; 2274 } 2275 2276 spin_unlock(&ino->i_lock); 2277 first = true; 2278 status = nfs4_select_rw_stateid(ctx->state, 2279 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2280 NULL, &stateid, NULL); 2281 if (status != 0) { 2282 lseg = ERR_PTR(status); 2283 trace_pnfs_update_layout(ino, pos, count, 2284 iomode, lo, lseg, 2285 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2286 nfs4_schedule_stateid_recovery(server, ctx->state); 2287 pnfs_clear_first_layoutget(lo); 2288 pnfs_put_layout_hdr(lo); 2289 goto lookup_again; 2290 } 2291 spin_lock(&ino->i_lock); 2292 } else { 2293 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2294 } 2295 2296 if (pnfs_layoutgets_blocked(lo)) { 2297 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2298 PNFS_UPDATE_LAYOUT_BLOCKED); 2299 goto out_unlock; 2300 } 2301 nfs_layoutget_begin(lo); 2302 spin_unlock(&ino->i_lock); 2303 2304 _add_to_server_list(lo, server); 2305 2306 pg_offset = arg.offset & ~PAGE_MASK; 2307 if (pg_offset) { 2308 arg.offset -= pg_offset; 2309 arg.length += pg_offset; 2310 } 2311 if (arg.length != NFS4_MAX_UINT64) 2312 arg.length = PAGE_ALIGN(arg.length); 2313 2314 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); 2315 if (!lgp) { 2316 lseg = ERR_PTR(-ENOMEM); 2317 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2318 PNFS_UPDATE_LAYOUT_NOMEM); 2319 nfs_layoutget_end(lo); 2320 goto out_put_layout_hdr; 2321 } 2322 2323 lgp->lo = lo; 2324 pnfs_get_layout_hdr(lo); 2325 2326 lseg = nfs4_proc_layoutget(lgp, &exception); 2327 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2328 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2329 nfs_layoutget_end(lo); 2330 if (IS_ERR(lseg)) { 2331 switch(PTR_ERR(lseg)) { 2332 case -EBUSY: 2333 if (time_after(jiffies, giveup)) 2334 lseg = NULL; 2335 break; 2336 case -ERECALLCONFLICT: 2337 case -EAGAIN: 2338 break; 2339 case -ENODATA: 2340 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2341 pnfs_layout_set_fail_bit( 2342 lo, pnfs_iomode_to_fail_bit(iomode)); 2343 lseg = NULL; 2344 goto out_put_layout_hdr; 2345 default: 2346 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2347 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2348 lseg = NULL; 2349 } 2350 goto out_put_layout_hdr; 2351 } 2352 if (lseg) { 2353 if (!exception.retry) 2354 goto out_put_layout_hdr; 2355 if (first) 2356 pnfs_clear_first_layoutget(lo); 2357 trace_pnfs_update_layout(ino, pos, count, 2358 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2359 pnfs_put_layout_hdr(lo); 2360 goto lookup_again; 2361 } 2362 } else { 2363 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2364 } 2365 2366 out_put_layout_hdr: 2367 if (first) 2368 pnfs_clear_first_layoutget(lo); 2369 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2370 PNFS_UPDATE_LAYOUT_EXIT); 2371 pnfs_put_layout_hdr(lo); 2372 out: 2373 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2374 "(%s, offset: %llu, length: %llu)\n", 2375 __func__, ino->i_sb->s_id, 2376 (unsigned long long)ino->i_ino, 2377 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2378 iomode==IOMODE_RW ? "read/write" : "read-only", 2379 (unsigned long long)pos, 2380 (unsigned long long)count); 2381 return lseg; 2382 out_unlock: 2383 spin_unlock(&ino->i_lock); 2384 goto out_put_layout_hdr; 2385 } 2386 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2387 2388 static bool 2389 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2390 { 2391 switch (range->iomode) { 2392 case IOMODE_READ: 2393 case IOMODE_RW: 2394 break; 2395 default: 2396 return false; 2397 } 2398 if (range->offset == NFS4_MAX_UINT64) 2399 return false; 2400 if (range->length == 0) 2401 return false; 2402 if (range->length != NFS4_MAX_UINT64 && 2403 range->length > NFS4_MAX_UINT64 - range->offset) 2404 return false; 2405 return true; 2406 } 2407 2408 static struct pnfs_layout_hdr * 2409 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2410 { 2411 struct pnfs_layout_hdr *lo; 2412 2413 spin_lock(&ino->i_lock); 2414 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2415 if (!lo) 2416 goto out_unlock; 2417 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2418 goto out_unlock; 2419 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2420 goto out_unlock; 2421 if (pnfs_layoutgets_blocked(lo)) 2422 goto out_unlock; 2423 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2424 goto out_unlock; 2425 nfs_layoutget_begin(lo); 2426 spin_unlock(&ino->i_lock); 2427 _add_to_server_list(lo, NFS_SERVER(ino)); 2428 return lo; 2429 2430 out_unlock: 2431 spin_unlock(&ino->i_lock); 2432 pnfs_put_layout_hdr(lo); 2433 return NULL; 2434 } 2435 2436 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2437 struct nfs_open_context *ctx) 2438 { 2439 struct inode *ino = data->dentry->d_inode; 2440 struct pnfs_layout_range rng = { 2441 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2442 IOMODE_RW: IOMODE_READ, 2443 .offset = 0, 2444 .length = NFS4_MAX_UINT64, 2445 }; 2446 struct nfs4_layoutget *lgp; 2447 struct pnfs_layout_hdr *lo; 2448 2449 /* Heuristic: don't send layoutget if we have cached data */ 2450 if (rng.iomode == IOMODE_READ && 2451 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2452 return; 2453 2454 lo = _pnfs_grab_empty_layout(ino, ctx); 2455 if (!lo) 2456 return; 2457 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2458 nfs_io_gfp_mask()); 2459 if (!lgp) { 2460 pnfs_clear_first_layoutget(lo); 2461 nfs_layoutget_end(lo); 2462 pnfs_put_layout_hdr(lo); 2463 return; 2464 } 2465 lgp->lo = lo; 2466 data->lgp = lgp; 2467 data->o_arg.lg_args = &lgp->args; 2468 data->o_res.lg_res = &lgp->res; 2469 } 2470 2471 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2472 struct nfs_open_context *ctx) 2473 { 2474 struct inode *ino = data->dentry->d_inode; 2475 struct pnfs_layout_range rng = { 2476 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2477 IOMODE_RW: IOMODE_READ, 2478 .offset = 0, 2479 .length = NFS4_MAX_UINT64, 2480 }; 2481 struct nfs4_layoutget *lgp; 2482 2483 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 2484 nfs_io_gfp_mask()); 2485 if (!lgp) 2486 return; 2487 data->lgp = lgp; 2488 data->o_arg.lg_args = &lgp->args; 2489 data->o_res.lg_res = &lgp->res; 2490 } 2491 2492 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2493 struct nfs_open_context *ctx) 2494 { 2495 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2496 2497 if (!(pnfs_enabled_sb(server) && 2498 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2499 return; 2500 /* Could check on max_ops, but currently hardcoded high enough */ 2501 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2502 return; 2503 if (data->lgp) 2504 return; 2505 if (data->state) 2506 _lgopen_prepare_attached(data, ctx); 2507 else 2508 _lgopen_prepare_floating(data, ctx); 2509 } 2510 2511 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2512 struct nfs_open_context *ctx) 2513 { 2514 struct pnfs_layout_hdr *lo; 2515 struct pnfs_layout_segment *lseg; 2516 struct nfs_server *srv = NFS_SERVER(ino); 2517 u32 iomode; 2518 2519 if (!lgp) 2520 return; 2521 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2522 if (lgp->res.status) { 2523 switch (lgp->res.status) { 2524 default: 2525 break; 2526 /* 2527 * Halt lgopen attempts if the server doesn't recognise 2528 * the "current stateid" value, the layout type, or the 2529 * layoutget operation as being valid. 2530 * Also if it complains about too many ops in the compound 2531 * or of the request/reply being too big. 2532 */ 2533 case -NFS4ERR_BAD_STATEID: 2534 case -NFS4ERR_NOTSUPP: 2535 case -NFS4ERR_REP_TOO_BIG: 2536 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2537 case -NFS4ERR_REQ_TOO_BIG: 2538 case -NFS4ERR_TOO_MANY_OPS: 2539 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2540 srv->caps &= ~NFS_CAP_LGOPEN; 2541 } 2542 return; 2543 } 2544 if (!lgp->lo) { 2545 lo = _pnfs_grab_empty_layout(ino, ctx); 2546 if (!lo) 2547 return; 2548 lgp->lo = lo; 2549 } else 2550 lo = lgp->lo; 2551 2552 lseg = pnfs_layout_process(lgp); 2553 if (!IS_ERR(lseg)) { 2554 iomode = lgp->args.range.iomode; 2555 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2556 pnfs_put_lseg(lseg); 2557 } 2558 } 2559 2560 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2561 { 2562 if (lgp != NULL) { 2563 if (lgp->lo) { 2564 pnfs_clear_first_layoutget(lgp->lo); 2565 nfs_layoutget_end(lgp->lo); 2566 } 2567 pnfs_layoutget_free(lgp); 2568 } 2569 } 2570 2571 struct pnfs_layout_segment * 2572 pnfs_layout_process(struct nfs4_layoutget *lgp) 2573 { 2574 struct pnfs_layout_hdr *lo = lgp->lo; 2575 struct nfs4_layoutget_res *res = &lgp->res; 2576 struct pnfs_layout_segment *lseg; 2577 struct inode *ino = lo->plh_inode; 2578 LIST_HEAD(free_me); 2579 2580 if (!pnfs_sanity_check_layout_range(&res->range)) 2581 return ERR_PTR(-EINVAL); 2582 2583 /* Inject layout blob into I/O device driver */ 2584 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2585 if (IS_ERR_OR_NULL(lseg)) { 2586 if (!lseg) 2587 lseg = ERR_PTR(-ENOMEM); 2588 2589 dprintk("%s: Could not allocate layout: error %ld\n", 2590 __func__, PTR_ERR(lseg)); 2591 return lseg; 2592 } 2593 2594 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2595 2596 spin_lock(&ino->i_lock); 2597 if (pnfs_layoutgets_blocked(lo)) { 2598 dprintk("%s forget reply due to state\n", __func__); 2599 goto out_forget; 2600 } 2601 2602 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2603 !pnfs_is_first_layoutget(lo)) 2604 goto out_forget; 2605 2606 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2607 /* existing state ID, make sure the sequence number matches. */ 2608 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2609 if (!pnfs_layout_is_valid(lo)) 2610 lo->plh_barrier = 0; 2611 dprintk("%s forget reply due to sequence\n", __func__); 2612 goto out_forget; 2613 } 2614 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2615 } else if (pnfs_layout_is_valid(lo)) { 2616 /* 2617 * We got an entirely new state ID. Mark all segments for the 2618 * inode invalid, and retry the layoutget 2619 */ 2620 struct pnfs_layout_range range = { 2621 .iomode = IOMODE_ANY, 2622 .length = NFS4_MAX_UINT64, 2623 }; 2624 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); 2625 goto out_forget; 2626 } else { 2627 /* We have a completely new layout */ 2628 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2629 } 2630 2631 pnfs_get_lseg(lseg); 2632 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2633 2634 2635 if (res->return_on_close) 2636 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2637 2638 spin_unlock(&ino->i_lock); 2639 pnfs_free_lseg_list(&free_me); 2640 return lseg; 2641 2642 out_forget: 2643 spin_unlock(&ino->i_lock); 2644 lseg->pls_layout = lo; 2645 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2646 return ERR_PTR(-EAGAIN); 2647 } 2648 2649 /** 2650 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2651 * @lo: pointer to layout header 2652 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2653 * @return_range: describe layout segment ranges to be returned 2654 * @seq: stateid seqid to match 2655 * 2656 * This function is mainly intended for use by layoutrecall. It attempts 2657 * to free the layout segment immediately, or else to mark it for return 2658 * as soon as its reference count drops to zero. 2659 * 2660 * Returns 2661 * - 0: a layoutreturn needs to be scheduled. 2662 * - EBUSY: there are layout segment that are still in use. 2663 * - ENOENT: there are no layout segments that need to be returned. 2664 */ 2665 int 2666 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2667 struct list_head *tmp_list, 2668 const struct pnfs_layout_range *return_range, 2669 u32 seq) 2670 { 2671 struct pnfs_layout_segment *lseg, *next; 2672 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2673 int remaining = 0; 2674 2675 dprintk("%s:Begin lo %p\n", __func__, lo); 2676 2677 assert_spin_locked(&lo->plh_inode->i_lock); 2678 2679 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2680 tmp_list = &lo->plh_return_segs; 2681 2682 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2683 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2684 dprintk("%s: marking lseg %p iomode %d " 2685 "offset %llu length %llu\n", __func__, 2686 lseg, lseg->pls_range.iomode, 2687 lseg->pls_range.offset, 2688 lseg->pls_range.length); 2689 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2690 tmp_list = &lo->plh_return_segs; 2691 if (mark_lseg_invalid(lseg, tmp_list)) 2692 continue; 2693 remaining++; 2694 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2695 pnfs_lseg_cancel_io(server, lseg); 2696 } 2697 2698 if (remaining) { 2699 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2700 return -EBUSY; 2701 } 2702 2703 if (!list_empty(&lo->plh_return_segs)) { 2704 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2705 return 0; 2706 } 2707 2708 return -ENOENT; 2709 } 2710 2711 static void 2712 pnfs_mark_layout_for_return(struct inode *inode, 2713 const struct pnfs_layout_range *range) 2714 { 2715 struct pnfs_layout_hdr *lo; 2716 bool return_now = false; 2717 2718 spin_lock(&inode->i_lock); 2719 lo = NFS_I(inode)->layout; 2720 if (!pnfs_layout_is_valid(lo)) { 2721 spin_unlock(&inode->i_lock); 2722 return; 2723 } 2724 pnfs_set_plh_return_info(lo, range->iomode, 0); 2725 /* 2726 * mark all matching lsegs so that we are sure to have no live 2727 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2728 * for how it works. 2729 */ 2730 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { 2731 const struct cred *cred; 2732 nfs4_stateid stateid; 2733 enum pnfs_iomode iomode; 2734 2735 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2736 spin_unlock(&inode->i_lock); 2737 if (return_now) 2738 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2739 PNFS_FL_LAYOUTRETURN_ASYNC); 2740 } else { 2741 spin_unlock(&inode->i_lock); 2742 nfs_commit_inode(inode, 0); 2743 } 2744 } 2745 2746 void pnfs_error_mark_layout_for_return(struct inode *inode, 2747 struct pnfs_layout_segment *lseg) 2748 { 2749 struct pnfs_layout_range range = { 2750 .iomode = lseg->pls_range.iomode, 2751 .offset = 0, 2752 .length = NFS4_MAX_UINT64, 2753 }; 2754 2755 pnfs_mark_layout_for_return(inode, &range); 2756 } 2757 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2758 2759 static bool 2760 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2761 { 2762 return pnfs_layout_is_valid(lo) && 2763 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2764 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2765 } 2766 2767 static struct pnfs_layout_segment * 2768 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2769 const struct pnfs_layout_range *range, 2770 enum pnfs_iomode iomode) 2771 { 2772 struct pnfs_layout_segment *lseg; 2773 2774 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2775 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2776 continue; 2777 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2778 continue; 2779 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2780 continue; 2781 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2782 return lseg; 2783 } 2784 return NULL; 2785 } 2786 2787 /* Find open file states whose mode matches that of the range */ 2788 static bool 2789 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2790 const struct pnfs_layout_range *range) 2791 { 2792 struct list_head *head; 2793 struct nfs_open_context *ctx; 2794 fmode_t mode = 0; 2795 2796 if (!pnfs_layout_can_be_returned(lo) || 2797 !pnfs_find_first_lseg(lo, range, range->iomode)) 2798 return false; 2799 2800 head = &NFS_I(lo->plh_inode)->open_files; 2801 list_for_each_entry_rcu(ctx, head, list) { 2802 if (ctx->state) 2803 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2804 } 2805 2806 switch (range->iomode) { 2807 default: 2808 break; 2809 case IOMODE_READ: 2810 mode &= ~FMODE_WRITE; 2811 break; 2812 case IOMODE_RW: 2813 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2814 mode &= ~FMODE_READ; 2815 } 2816 return mode == 0; 2817 } 2818 2819 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2820 void *data) 2821 { 2822 const struct pnfs_layout_range *range = data; 2823 const struct cred *cred; 2824 struct pnfs_layout_hdr *lo; 2825 struct inode *inode; 2826 nfs4_stateid stateid; 2827 enum pnfs_iomode iomode; 2828 2829 restart: 2830 rcu_read_lock(); 2831 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2832 inode = lo->plh_inode; 2833 if (!inode || !pnfs_layout_can_be_returned(lo) || 2834 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2835 continue; 2836 spin_lock(&inode->i_lock); 2837 if (!lo->plh_inode || 2838 !pnfs_should_return_unused_layout(lo, range)) { 2839 spin_unlock(&inode->i_lock); 2840 continue; 2841 } 2842 pnfs_get_layout_hdr(lo); 2843 pnfs_set_plh_return_info(lo, range->iomode, 0); 2844 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2845 range, 0) != 0 || 2846 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2847 spin_unlock(&inode->i_lock); 2848 rcu_read_unlock(); 2849 pnfs_put_layout_hdr(lo); 2850 cond_resched(); 2851 goto restart; 2852 } 2853 spin_unlock(&inode->i_lock); 2854 rcu_read_unlock(); 2855 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2856 PNFS_FL_LAYOUTRETURN_ASYNC); 2857 pnfs_put_layout_hdr(lo); 2858 cond_resched(); 2859 goto restart; 2860 } 2861 rcu_read_unlock(); 2862 return 0; 2863 } 2864 2865 void 2866 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2867 enum pnfs_iomode iomode) 2868 { 2869 struct pnfs_layout_range range = { 2870 .iomode = iomode, 2871 .offset = 0, 2872 .length = NFS4_MAX_UINT64, 2873 }; 2874 2875 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2876 &range); 2877 } 2878 2879 /* Check if we have we have a valid layout but if there isn't an intersection 2880 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 2881 */ 2882 void 2883 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 2884 struct nfs_page *req) 2885 { 2886 if (pgio->pg_lseg == NULL || 2887 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 2888 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 2889 return; 2890 pnfs_put_lseg(pgio->pg_lseg); 2891 pgio->pg_lseg = NULL; 2892 } 2893 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2894 2895 void 2896 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2897 { 2898 u64 rd_size; 2899 2900 pnfs_generic_pg_check_layout(pgio, req); 2901 if (pgio->pg_lseg == NULL) { 2902 if (pgio->pg_dreq == NULL) 2903 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2904 else 2905 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 2906 req_offset(req)); 2907 2908 pgio->pg_lseg = 2909 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2910 req_offset(req), rd_size, 2911 IOMODE_READ, false, 2912 nfs_io_gfp_mask()); 2913 if (IS_ERR(pgio->pg_lseg)) { 2914 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2915 pgio->pg_lseg = NULL; 2916 return; 2917 } 2918 } 2919 /* If no lseg, fall back to read through mds */ 2920 if (pgio->pg_lseg == NULL) 2921 nfs_pageio_reset_read_mds(pgio); 2922 2923 } 2924 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2925 2926 void 2927 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2928 struct nfs_page *req, u64 wb_size) 2929 { 2930 pnfs_generic_pg_check_layout(pgio, req); 2931 if (pgio->pg_lseg == NULL) { 2932 pgio->pg_lseg = 2933 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 2934 req_offset(req), wb_size, IOMODE_RW, 2935 false, nfs_io_gfp_mask()); 2936 if (IS_ERR(pgio->pg_lseg)) { 2937 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2938 pgio->pg_lseg = NULL; 2939 return; 2940 } 2941 } 2942 /* If no lseg, fall back to write through mds */ 2943 if (pgio->pg_lseg == NULL) 2944 nfs_pageio_reset_write_mds(pgio); 2945 } 2946 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2947 2948 void 2949 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2950 { 2951 if (desc->pg_lseg) { 2952 pnfs_put_lseg(desc->pg_lseg); 2953 desc->pg_lseg = NULL; 2954 } 2955 } 2956 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2957 2958 /* 2959 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2960 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2961 */ 2962 size_t 2963 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2964 struct nfs_page *prev, struct nfs_page *req) 2965 { 2966 unsigned int size; 2967 u64 seg_end, req_start, seg_left; 2968 2969 size = nfs_generic_pg_test(pgio, prev, req); 2970 if (!size) 2971 return 0; 2972 2973 /* 2974 * 'size' contains the number of bytes left in the current page (up 2975 * to the original size asked for in @req->wb_bytes). 2976 * 2977 * Calculate how many bytes are left in the layout segment 2978 * and if there are less bytes than 'size', return that instead. 2979 * 2980 * Please also note that 'end_offset' is actually the offset of the 2981 * first byte that lies outside the pnfs_layout_range. FIXME? 2982 * 2983 */ 2984 if (pgio->pg_lseg) { 2985 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2986 pgio->pg_lseg->pls_range.length); 2987 req_start = req_offset(req); 2988 2989 /* start of request is past the last byte of this segment */ 2990 if (req_start >= seg_end) 2991 return 0; 2992 2993 /* adjust 'size' iff there are fewer bytes left in the 2994 * segment than what nfs_generic_pg_test returned */ 2995 seg_left = seg_end - req_start; 2996 if (seg_left < size) 2997 size = (unsigned int)seg_left; 2998 } 2999 3000 return size; 3001 } 3002 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 3003 3004 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 3005 { 3006 struct nfs_pageio_descriptor pgio; 3007 3008 /* Resend all requests through the MDS */ 3009 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 3010 hdr->completion_ops); 3011 return nfs_pageio_resend(&pgio, hdr); 3012 } 3013 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 3014 3015 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 3016 { 3017 3018 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 3019 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3020 PNFS_LAYOUTRET_ON_ERROR) { 3021 pnfs_return_layout(hdr->inode); 3022 } 3023 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3024 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 3025 } 3026 3027 /* 3028 * Called by non rpc-based layout drivers 3029 */ 3030 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 3031 { 3032 if (likely(!hdr->pnfs_error)) { 3033 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 3034 hdr->mds_offset + hdr->res.count); 3035 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3036 } 3037 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3038 if (unlikely(hdr->pnfs_error)) 3039 pnfs_ld_handle_write_error(hdr); 3040 hdr->mds_ops->rpc_release(hdr); 3041 } 3042 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3043 3044 static void 3045 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3046 struct nfs_pgio_header *hdr) 3047 { 3048 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3049 3050 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3051 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3052 nfs_pageio_reset_write_mds(desc); 3053 mirror->pg_recoalesce = 1; 3054 } 3055 hdr->completion_ops->completion(hdr); 3056 } 3057 3058 static enum pnfs_try_status 3059 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3060 const struct rpc_call_ops *call_ops, 3061 struct pnfs_layout_segment *lseg, 3062 int how) 3063 { 3064 struct inode *inode = hdr->inode; 3065 enum pnfs_try_status trypnfs; 3066 struct nfs_server *nfss = NFS_SERVER(inode); 3067 3068 hdr->mds_ops = call_ops; 3069 3070 dprintk("%s: Writing ino:%llu %u@%llu (how %d)\n", __func__, 3071 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3072 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3073 if (trypnfs != PNFS_NOT_ATTEMPTED) 3074 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3075 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3076 return trypnfs; 3077 } 3078 3079 static void 3080 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3081 struct nfs_pgio_header *hdr, int how) 3082 { 3083 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3084 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3085 enum pnfs_try_status trypnfs; 3086 3087 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3088 switch (trypnfs) { 3089 case PNFS_NOT_ATTEMPTED: 3090 pnfs_write_through_mds(desc, hdr); 3091 break; 3092 case PNFS_ATTEMPTED: 3093 break; 3094 case PNFS_TRY_AGAIN: 3095 /* cleanup hdr and prepare to redo pnfs */ 3096 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3097 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3098 list_splice_init(&hdr->pages, &mirror->pg_list); 3099 mirror->pg_recoalesce = 1; 3100 } 3101 hdr->mds_ops->rpc_release(hdr); 3102 } 3103 } 3104 3105 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3106 { 3107 pnfs_put_lseg(hdr->lseg); 3108 nfs_pgio_header_free(hdr); 3109 } 3110 3111 int 3112 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3113 { 3114 struct nfs_pgio_header *hdr; 3115 int ret; 3116 3117 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3118 if (!hdr) { 3119 desc->pg_error = -ENOMEM; 3120 return desc->pg_error; 3121 } 3122 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3123 3124 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3125 ret = nfs_generic_pgio(desc, hdr); 3126 if (!ret) 3127 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3128 3129 return ret; 3130 } 3131 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3132 3133 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3134 { 3135 struct nfs_pageio_descriptor pgio; 3136 3137 /* Resend all requests through the MDS */ 3138 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3139 return nfs_pageio_resend(&pgio, hdr); 3140 } 3141 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3142 3143 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3144 { 3145 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3146 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3147 PNFS_LAYOUTRET_ON_ERROR) { 3148 pnfs_return_layout(hdr->inode); 3149 } 3150 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3151 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3152 } 3153 3154 /* 3155 * Called by non rpc-based layout drivers 3156 */ 3157 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3158 { 3159 if (likely(!hdr->pnfs_error)) 3160 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3161 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3162 if (unlikely(hdr->pnfs_error)) 3163 pnfs_ld_handle_read_error(hdr); 3164 hdr->mds_ops->rpc_release(hdr); 3165 } 3166 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3167 3168 static void 3169 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3170 struct nfs_pgio_header *hdr) 3171 { 3172 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3173 3174 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3175 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3176 nfs_pageio_reset_read_mds(desc); 3177 mirror->pg_recoalesce = 1; 3178 } 3179 hdr->completion_ops->completion(hdr); 3180 } 3181 3182 /* 3183 * Call the appropriate parallel I/O subsystem read function. 3184 */ 3185 static enum pnfs_try_status 3186 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3187 const struct rpc_call_ops *call_ops, 3188 struct pnfs_layout_segment *lseg) 3189 { 3190 struct inode *inode = hdr->inode; 3191 struct nfs_server *nfss = NFS_SERVER(inode); 3192 enum pnfs_try_status trypnfs; 3193 3194 hdr->mds_ops = call_ops; 3195 3196 dprintk("%s: Reading ino:%llu %u@%llu\n", 3197 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3198 3199 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3200 if (trypnfs != PNFS_NOT_ATTEMPTED) 3201 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3202 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3203 return trypnfs; 3204 } 3205 3206 /* Resend all requests through pnfs. */ 3207 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3208 unsigned int mirror_idx) 3209 { 3210 struct nfs_pageio_descriptor pgio; 3211 3212 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3213 /* Prevent deadlocks with layoutreturn! */ 3214 pnfs_put_lseg(hdr->lseg); 3215 hdr->lseg = NULL; 3216 3217 nfs_pageio_init_read(&pgio, hdr->inode, false, 3218 hdr->completion_ops); 3219 pgio.pg_mirror_idx = mirror_idx; 3220 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3221 } 3222 } 3223 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3224 3225 static void 3226 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3227 { 3228 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3229 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3230 enum pnfs_try_status trypnfs; 3231 3232 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3233 switch (trypnfs) { 3234 case PNFS_NOT_ATTEMPTED: 3235 pnfs_read_through_mds(desc, hdr); 3236 break; 3237 case PNFS_ATTEMPTED: 3238 break; 3239 case PNFS_TRY_AGAIN: 3240 /* cleanup hdr and prepare to redo pnfs */ 3241 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3242 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3243 list_splice_init(&hdr->pages, &mirror->pg_list); 3244 mirror->pg_recoalesce = 1; 3245 } 3246 hdr->mds_ops->rpc_release(hdr); 3247 } 3248 } 3249 3250 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3251 { 3252 pnfs_put_lseg(hdr->lseg); 3253 nfs_pgio_header_free(hdr); 3254 } 3255 3256 int 3257 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3258 { 3259 struct nfs_pgio_header *hdr; 3260 int ret; 3261 3262 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3263 if (!hdr) { 3264 desc->pg_error = -ENOMEM; 3265 return desc->pg_error; 3266 } 3267 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3268 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3269 ret = nfs_generic_pgio(desc, hdr); 3270 if (!ret) 3271 pnfs_do_read(desc, hdr); 3272 return ret; 3273 } 3274 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3275 3276 static void pnfs_clear_layoutcommitting(struct inode *inode) 3277 { 3278 unsigned long *bitlock = &NFS_I(inode)->flags; 3279 3280 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 3281 smp_mb__after_atomic(); 3282 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 3283 } 3284 3285 /* 3286 * There can be multiple RW segments. 3287 */ 3288 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3289 { 3290 struct pnfs_layout_segment *lseg; 3291 3292 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3293 if (lseg->pls_range.iomode == IOMODE_RW && 3294 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3295 list_add(&lseg->pls_lc_list, listp); 3296 } 3297 } 3298 3299 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3300 { 3301 struct pnfs_layout_segment *lseg, *tmp; 3302 3303 /* Matched by references in pnfs_set_layoutcommit */ 3304 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3305 list_del_init(&lseg->pls_lc_list); 3306 pnfs_put_lseg(lseg); 3307 } 3308 3309 pnfs_clear_layoutcommitting(inode); 3310 } 3311 3312 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3313 { 3314 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3315 } 3316 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3317 3318 void 3319 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3320 loff_t end_pos) 3321 { 3322 struct nfs_inode *nfsi = NFS_I(inode); 3323 bool mark_as_dirty = false; 3324 3325 spin_lock(&inode->i_lock); 3326 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3327 nfsi->layout->plh_lwb = end_pos; 3328 mark_as_dirty = true; 3329 dprintk("%s: Set layoutcommit for inode %llu ", 3330 __func__, inode->i_ino); 3331 } else if (end_pos > nfsi->layout->plh_lwb) 3332 nfsi->layout->plh_lwb = end_pos; 3333 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3334 /* references matched in nfs4_layoutcommit_release */ 3335 pnfs_get_lseg(lseg); 3336 } 3337 spin_unlock(&inode->i_lock); 3338 dprintk("%s: lseg %p end_pos %llu\n", 3339 __func__, lseg, nfsi->layout->plh_lwb); 3340 3341 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3342 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3343 if (mark_as_dirty) 3344 mark_inode_dirty_sync(inode); 3345 } 3346 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3347 3348 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3349 { 3350 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3351 3352 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3353 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3354 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3355 } 3356 3357 /* 3358 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3359 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3360 * data to disk to allow the server to recover the data if it crashes. 3361 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3362 * is off, and a COMMIT is sent to a data server, or 3363 * if WRITEs to a data server return NFS_DATA_SYNC. 3364 */ 3365 int 3366 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3367 { 3368 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3369 struct nfs4_layoutcommit_data *data; 3370 struct nfs_inode *nfsi = NFS_I(inode); 3371 loff_t end_pos; 3372 int status; 3373 bool mark_as_dirty = false; 3374 3375 if (!pnfs_layoutcommit_outstanding(inode)) 3376 return 0; 3377 3378 dprintk("--> %s inode %llu\n", __func__, inode->i_ino); 3379 3380 status = -EAGAIN; 3381 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3382 if (!sync) 3383 goto out; 3384 status = wait_on_bit_lock_action(&nfsi->flags, 3385 NFS_INO_LAYOUTCOMMITTING, 3386 nfs_wait_bit_killable, 3387 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3388 if (status) 3389 goto out; 3390 } 3391 3392 status = -ENOMEM; 3393 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3394 data = kzalloc_obj(*data, nfs_io_gfp_mask()); 3395 if (!data) 3396 goto clear_layoutcommitting; 3397 3398 status = 0; 3399 spin_lock(&inode->i_lock); 3400 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3401 goto out_unlock; 3402 3403 INIT_LIST_HEAD(&data->lseg_list); 3404 pnfs_list_write_lseg(inode, &data->lseg_list); 3405 3406 end_pos = nfsi->layout->plh_lwb; 3407 3408 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3409 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3410 spin_unlock(&inode->i_lock); 3411 3412 data->args.inode = inode; 3413 nfs_fattr_init(&data->fattr); 3414 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3415 data->res.fattr = &data->fattr; 3416 if (end_pos != 0) 3417 data->args.lastbytewritten = end_pos - 1; 3418 else 3419 data->args.lastbytewritten = U64_MAX; 3420 data->res.server = NFS_SERVER(inode); 3421 3422 if (ld->prepare_layoutcommit) { 3423 status = ld->prepare_layoutcommit(&data->args); 3424 if (status) { 3425 if (status != -ENOSPC) 3426 put_cred(data->cred); 3427 spin_lock(&inode->i_lock); 3428 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3429 if (end_pos > nfsi->layout->plh_lwb) 3430 nfsi->layout->plh_lwb = end_pos; 3431 if (status != -ENOSPC) 3432 goto out_unlock; 3433 spin_unlock(&inode->i_lock); 3434 mark_as_dirty = true; 3435 } 3436 } 3437 3438 3439 status = nfs4_proc_layoutcommit(data, sync); 3440 out: 3441 if (status || mark_as_dirty) 3442 mark_inode_dirty_sync(inode); 3443 dprintk("<-- %s status %d\n", __func__, status); 3444 return status; 3445 out_unlock: 3446 spin_unlock(&inode->i_lock); 3447 kfree(data); 3448 clear_layoutcommitting: 3449 pnfs_clear_layoutcommitting(inode); 3450 goto out; 3451 } 3452 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3453 3454 int 3455 pnfs_generic_sync(struct inode *inode, bool datasync) 3456 { 3457 return pnfs_layoutcommit_inode(inode, true); 3458 } 3459 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3460 3461 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3462 { 3463 struct nfs4_threshold *thp; 3464 3465 thp = kzalloc_obj(*thp, nfs_io_gfp_mask()); 3466 if (!thp) { 3467 dprintk("%s mdsthreshold allocation failed\n", __func__); 3468 return NULL; 3469 } 3470 return thp; 3471 } 3472 3473 #if IS_ENABLED(CONFIG_NFS_V4_2) 3474 int 3475 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3476 { 3477 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3478 struct nfs_server *server = NFS_SERVER(inode); 3479 struct nfs_inode *nfsi = NFS_I(inode); 3480 struct nfs42_layoutstat_data *data; 3481 struct pnfs_layout_hdr *hdr; 3482 int status = 0; 3483 3484 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3485 goto out; 3486 3487 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3488 goto out; 3489 3490 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3491 goto out; 3492 3493 spin_lock(&inode->i_lock); 3494 if (!NFS_I(inode)->layout) { 3495 spin_unlock(&inode->i_lock); 3496 goto out_clear_layoutstats; 3497 } 3498 hdr = NFS_I(inode)->layout; 3499 pnfs_get_layout_hdr(hdr); 3500 spin_unlock(&inode->i_lock); 3501 3502 data = kzalloc_obj(*data, gfp_flags); 3503 if (!data) { 3504 status = -ENOMEM; 3505 goto out_put; 3506 } 3507 3508 data->args.fh = NFS_FH(inode); 3509 data->args.inode = inode; 3510 status = ld->prepare_layoutstats(&data->args); 3511 if (status) 3512 goto out_free; 3513 3514 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3515 3516 out: 3517 dprintk("%s returns %d\n", __func__, status); 3518 return status; 3519 3520 out_free: 3521 kfree(data); 3522 out_put: 3523 pnfs_put_layout_hdr(hdr); 3524 out_clear_layoutstats: 3525 smp_mb__before_atomic(); 3526 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3527 smp_mb__after_atomic(); 3528 goto out; 3529 } 3530 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3531 #endif 3532 3533 unsigned int layoutstats_timer; 3534 module_param(layoutstats_timer, uint, 0644); 3535 EXPORT_SYMBOL_GPL(layoutstats_timer); 3536