1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 #include "nfs4_fs.h" 41 42 #define NFSDBG_FACILITY NFSDBG_PNFS 43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 44 45 /* Locking: 46 * 47 * pnfs_spinlock: 48 * protects pnfs_modules_tbl. 49 */ 50 static DEFINE_SPINLOCK(pnfs_spinlock); 51 52 /* 53 * pnfs_modules_tbl holds all pnfs modules 54 */ 55 static LIST_HEAD(pnfs_modules_tbl); 56 57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 59 struct list_head *free_me, 60 const struct pnfs_layout_range *range, 61 u32 seq); 62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 63 struct list_head *tmp_list); 64 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); 65 66 /* Return the registered pnfs layout driver module matching given id */ 67 static struct pnfs_layoutdriver_type * 68 find_pnfs_driver_locked(u32 id) 69 { 70 struct pnfs_layoutdriver_type *local; 71 72 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 73 if (local->id == id) 74 goto out; 75 local = NULL; 76 out: 77 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 78 return local; 79 } 80 81 static struct pnfs_layoutdriver_type * 82 find_pnfs_driver(u32 id) 83 { 84 struct pnfs_layoutdriver_type *local; 85 86 spin_lock(&pnfs_spinlock); 87 local = find_pnfs_driver_locked(id); 88 if (local != NULL && !try_module_get(local->owner)) { 89 dprintk("%s: Could not grab reference on module\n", __func__); 90 local = NULL; 91 } 92 spin_unlock(&pnfs_spinlock); 93 return local; 94 } 95 96 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) 97 { 98 return find_pnfs_driver(id); 99 } 100 101 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) 102 { 103 if (ld) 104 module_put(ld->owner); 105 } 106 107 void 108 unset_pnfs_layoutdriver(struct nfs_server *nfss) 109 { 110 if (nfss->pnfs_curr_ld) { 111 if (nfss->pnfs_curr_ld->clear_layoutdriver) 112 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 113 /* Decrement the MDS count. Purge the deviceid cache if zero */ 114 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 115 nfs4_deviceid_purge_client(nfss->nfs_client); 116 module_put(nfss->pnfs_curr_ld->owner); 117 } 118 nfss->pnfs_curr_ld = NULL; 119 } 120 121 /* 122 * When the server sends a list of layout types, we choose one in the order 123 * given in the list below. 124 * 125 * FIXME: should this list be configurable in some fashion? module param? 126 * mount option? something else? 127 */ 128 static const u32 ld_prefs[] = { 129 LAYOUT_SCSI, 130 LAYOUT_BLOCK_VOLUME, 131 LAYOUT_OSD2_OBJECTS, 132 LAYOUT_FLEX_FILES, 133 LAYOUT_NFSV4_1_FILES, 134 0 135 }; 136 137 static int 138 ld_cmp(const void *e1, const void *e2) 139 { 140 u32 ld1 = *((u32 *)e1); 141 u32 ld2 = *((u32 *)e2); 142 int i; 143 144 for (i = 0; ld_prefs[i] != 0; i++) { 145 if (ld1 == ld_prefs[i]) 146 return -1; 147 148 if (ld2 == ld_prefs[i]) 149 return 1; 150 } 151 return 0; 152 } 153 154 /* 155 * Try to set the server's pnfs module to the pnfs layout type specified by id. 156 * Currently only one pNFS layout driver per filesystem is supported. 157 * 158 * @ids array of layout types supported by MDS. 159 */ 160 void 161 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 162 struct nfs_fsinfo *fsinfo) 163 { 164 struct pnfs_layoutdriver_type *ld_type = NULL; 165 u32 id; 166 int i; 167 168 if (fsinfo->nlayouttypes == 0) 169 goto out_no_driver; 170 if (!(server->nfs_client->cl_exchange_flags & 171 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 172 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 173 __func__, server->nfs_client->cl_exchange_flags); 174 goto out_no_driver; 175 } 176 177 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 178 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 179 180 for (i = 0; i < fsinfo->nlayouttypes; i++) { 181 id = fsinfo->layouttype[i]; 182 ld_type = find_pnfs_driver(id); 183 if (!ld_type) { 184 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 185 id); 186 ld_type = find_pnfs_driver(id); 187 } 188 if (ld_type) 189 break; 190 } 191 192 if (!ld_type) { 193 dprintk("%s: No pNFS module found!\n", __func__); 194 goto out_no_driver; 195 } 196 197 server->pnfs_curr_ld = ld_type; 198 if (ld_type->set_layoutdriver 199 && ld_type->set_layoutdriver(server, mntfh)) { 200 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 201 "driver %u.\n", __func__, id); 202 module_put(ld_type->owner); 203 goto out_no_driver; 204 } 205 /* Bump the MDS count */ 206 atomic_inc(&server->nfs_client->cl_mds_count); 207 208 dprintk("%s: pNFS module for %u set\n", __func__, id); 209 return; 210 211 out_no_driver: 212 dprintk("%s: Using NFSv4 I/O\n", __func__); 213 server->pnfs_curr_ld = NULL; 214 } 215 216 int 217 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 218 { 219 int status = -EINVAL; 220 struct pnfs_layoutdriver_type *tmp; 221 222 if (ld_type->id == 0) { 223 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 224 return status; 225 } 226 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 227 printk(KERN_ERR "NFS: %s Layout driver must provide " 228 "alloc_lseg and free_lseg.\n", __func__); 229 return status; 230 } 231 232 spin_lock(&pnfs_spinlock); 233 tmp = find_pnfs_driver_locked(ld_type->id); 234 if (!tmp) { 235 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 236 status = 0; 237 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 238 ld_type->name); 239 } else { 240 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 241 __func__, ld_type->id); 242 } 243 spin_unlock(&pnfs_spinlock); 244 245 return status; 246 } 247 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 248 249 void 250 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 251 { 252 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 253 spin_lock(&pnfs_spinlock); 254 list_del(&ld_type->pnfs_tblid); 255 spin_unlock(&pnfs_spinlock); 256 } 257 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 258 259 /* 260 * pNFS client layout cache 261 */ 262 263 /* Need to hold i_lock if caller does not already hold reference */ 264 void 265 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 refcount_inc(&lo->plh_refcount); 268 } 269 270 static struct pnfs_layout_hdr * 271 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 272 { 273 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 274 return ld->alloc_layout_hdr(ino, gfp_flags); 275 } 276 277 static void 278 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 279 { 280 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 282 283 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 284 struct nfs_client *clp = server->nfs_client; 285 286 spin_lock(&clp->cl_lock); 287 list_del_rcu(&lo->plh_layouts); 288 spin_unlock(&clp->cl_lock); 289 } 290 put_cred(lo->plh_lc_cred); 291 return ld->free_layout_hdr(lo); 292 } 293 294 static void 295 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 296 { 297 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 298 dprintk("%s: freeing layout cache %p\n", __func__, lo); 299 nfsi->layout = NULL; 300 /* Reset MDS Threshold I/O counters */ 301 nfsi->write_io = 0; 302 nfsi->read_io = 0; 303 } 304 305 void 306 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 307 { 308 struct inode *inode; 309 310 if (!lo) 311 return; 312 inode = lo->plh_inode; 313 pnfs_layoutreturn_before_put_layout_hdr(lo); 314 315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 316 if (!list_empty(&lo->plh_segs)) 317 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 318 pnfs_detach_layout_hdr(lo); 319 /* Notify pnfs_destroy_layout_final() that we're done */ 320 if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) 321 wake_up_var_locked(lo, &inode->i_lock); 322 spin_unlock(&inode->i_lock); 323 pnfs_free_layout_hdr(lo); 324 } 325 } 326 327 static struct inode * 328 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) 329 { 330 struct inode *inode = igrab(lo->plh_inode); 331 if (inode) 332 return inode; 333 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); 334 return NULL; 335 } 336 337 /* 338 * Compare 2 layout stateid sequence ids, to see which is newer, 339 * taking into account wraparound issues. 340 */ 341 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 342 { 343 return (s32)(s1 - s2) > 0; 344 } 345 346 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) 347 { 348 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) 349 lo->plh_barrier = newseq; 350 } 351 352 static void 353 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 354 u32 seq) 355 { 356 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 357 iomode = IOMODE_ANY; 358 lo->plh_return_iomode = iomode; 359 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 360 /* 361 * We must set lo->plh_return_seq to avoid livelocks with 362 * pnfs_layout_need_return() 363 */ 364 if (seq == 0) 365 seq = be32_to_cpu(lo->plh_stateid.seqid); 366 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) 367 lo->plh_return_seq = seq; 368 pnfs_barrier_update(lo, seq); 369 } 370 371 static void 372 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 373 { 374 struct pnfs_layout_segment *lseg; 375 lo->plh_return_iomode = 0; 376 lo->plh_return_seq = 0; 377 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 378 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 379 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 380 continue; 381 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 382 } 383 } 384 385 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 386 { 387 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 388 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 389 smp_mb__after_atomic(); 390 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 391 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 392 } 393 394 static void 395 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 396 struct list_head *free_me) 397 { 398 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 399 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 400 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 401 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 402 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 403 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 404 } 405 406 /* 407 * Update the seqid of a layout stateid after receiving 408 * NFS4ERR_OLD_STATEID 409 */ 410 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, 411 struct pnfs_layout_range *dst_range, 412 struct inode *inode) 413 { 414 struct pnfs_layout_hdr *lo; 415 struct pnfs_layout_range range = { 416 .iomode = IOMODE_ANY, 417 .offset = 0, 418 .length = NFS4_MAX_UINT64, 419 }; 420 bool ret = false; 421 LIST_HEAD(head); 422 int err; 423 424 spin_lock(&inode->i_lock); 425 lo = NFS_I(inode)->layout; 426 if (lo && pnfs_layout_is_valid(lo) && 427 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { 428 /* Is our call using the most recent seqid? If so, bump it */ 429 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { 430 nfs4_stateid_seqid_inc(dst); 431 ret = true; 432 goto out; 433 } 434 /* Try to update the seqid to the most recent */ 435 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, 436 true, NULL); 437 if (err != -EBUSY) { 438 dst->seqid = lo->plh_stateid.seqid; 439 *dst_range = range; 440 ret = true; 441 } 442 } 443 out: 444 spin_unlock(&inode->i_lock); 445 pnfs_free_lseg_list(&head); 446 return ret; 447 } 448 449 /* 450 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 451 * 452 * In order to continue using the pnfs_layout_hdr, a full recovery 453 * is required. 454 * Note that caller must hold inode->i_lock. 455 */ 456 int 457 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 458 struct list_head *lseg_list) 459 { 460 struct pnfs_layout_range range = { 461 .iomode = IOMODE_ANY, 462 .offset = 0, 463 .length = NFS4_MAX_UINT64, 464 }; 465 struct pnfs_layout_segment *lseg, *next; 466 467 if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 468 return !list_empty(&lo->plh_segs); 469 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); 470 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 471 pnfs_clear_lseg_state(lseg, lseg_list); 472 pnfs_clear_layoutreturn_info(lo); 473 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 474 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); 475 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 476 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 477 pnfs_clear_layoutreturn_waitbit(lo); 478 return !list_empty(&lo->plh_segs); 479 } 480 481 static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, 482 struct list_head *lseg_list, 483 enum pnfs_iomode iomode, u32 seq) 484 { 485 struct pnfs_layout_range range = { 486 .iomode = iomode, 487 .length = NFS4_MAX_UINT64, 488 }; 489 490 return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq, true, 491 NULL); 492 } 493 494 static int 495 pnfs_iomode_to_fail_bit(u32 iomode) 496 { 497 return iomode == IOMODE_RW ? 498 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 499 } 500 501 static void 502 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 503 { 504 lo->plh_retry_timestamp = jiffies; 505 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 506 refcount_inc(&lo->plh_refcount); 507 } 508 509 static void 510 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 511 { 512 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 513 refcount_dec(&lo->plh_refcount); 514 } 515 516 static void 517 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 518 { 519 struct inode *inode = lo->plh_inode; 520 struct pnfs_layout_range range = { 521 .iomode = iomode, 522 .offset = 0, 523 .length = NFS4_MAX_UINT64, 524 }; 525 LIST_HEAD(head); 526 527 spin_lock(&inode->i_lock); 528 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 529 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, true, NULL); 530 spin_unlock(&inode->i_lock); 531 pnfs_free_lseg_list(&head); 532 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 533 iomode == IOMODE_RW ? "RW" : "READ"); 534 } 535 536 static bool 537 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 538 { 539 unsigned long start, end; 540 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 541 542 if (test_bit(fail_bit, &lo->plh_flags) == 0) 543 return false; 544 end = jiffies; 545 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 546 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 547 /* It is time to retry the failed layoutgets */ 548 pnfs_layout_clear_fail_bit(lo, fail_bit); 549 return false; 550 } 551 return true; 552 } 553 554 static void 555 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 556 const struct pnfs_layout_range *range, 557 const nfs4_stateid *stateid) 558 { 559 INIT_LIST_HEAD(&lseg->pls_list); 560 INIT_LIST_HEAD(&lseg->pls_lc_list); 561 INIT_LIST_HEAD(&lseg->pls_commits); 562 refcount_set(&lseg->pls_refcount, 1); 563 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 564 lseg->pls_layout = lo; 565 lseg->pls_range = *range; 566 lseg->pls_seq = be32_to_cpu(stateid->seqid); 567 } 568 569 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 570 { 571 if (lseg != NULL) { 572 struct inode *inode = lseg->pls_layout->plh_inode; 573 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 574 } 575 } 576 577 static void 578 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 579 struct pnfs_layout_segment *lseg) 580 { 581 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 582 list_del_init(&lseg->pls_list); 583 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 584 refcount_dec(&lo->plh_refcount); 585 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 586 return; 587 if (list_empty(&lo->plh_segs) && 588 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 589 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 590 if (atomic_read(&lo->plh_outstanding) == 0) 591 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 592 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 593 } 594 } 595 596 static bool 597 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 598 struct pnfs_layout_segment *lseg) 599 { 600 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 601 pnfs_layout_is_valid(lo)) { 602 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 603 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 604 return true; 605 } 606 return false; 607 } 608 609 void 610 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 611 { 612 struct pnfs_layout_hdr *lo; 613 struct inode *inode; 614 615 if (!lseg) 616 return; 617 618 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 619 refcount_read(&lseg->pls_refcount), 620 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 621 622 lo = lseg->pls_layout; 623 inode = lo->plh_inode; 624 625 if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 626 pnfs_get_layout_hdr(lo); 627 pnfs_layout_remove_lseg(lo, lseg); 628 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 629 lseg = NULL; 630 spin_unlock(&inode->i_lock); 631 pnfs_free_lseg(lseg); 632 pnfs_put_layout_hdr(lo); 633 } 634 } 635 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 636 637 /* 638 * is l2 fully contained in l1? 639 * start1 end1 640 * [----------------------------------) 641 * start2 end2 642 * [----------------) 643 */ 644 static bool 645 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 646 const struct pnfs_layout_range *l2) 647 { 648 u64 start1 = l1->offset; 649 u64 end1 = pnfs_end_offset(start1, l1->length); 650 u64 start2 = l2->offset; 651 u64 end2 = pnfs_end_offset(start2, l2->length); 652 653 return (start1 <= start2) && (end1 >= end2); 654 } 655 656 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 657 struct list_head *tmp_list) 658 { 659 if (!refcount_dec_and_test(&lseg->pls_refcount)) 660 return false; 661 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 662 list_add(&lseg->pls_list, tmp_list); 663 return true; 664 } 665 666 /* Returns 1 if lseg is removed from list, 0 otherwise */ 667 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 668 struct list_head *tmp_list) 669 { 670 int rv = 0; 671 672 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 673 /* Remove the reference keeping the lseg in the 674 * list. It will now be removed when all 675 * outstanding io is finished. 676 */ 677 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 678 refcount_read(&lseg->pls_refcount)); 679 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 680 rv = 1; 681 } 682 return rv; 683 } 684 685 static bool 686 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 687 const struct pnfs_layout_range *recall_range) 688 { 689 return (recall_range->iomode == IOMODE_ANY || 690 lseg_range->iomode == recall_range->iomode) && 691 pnfs_lseg_range_intersecting(lseg_range, recall_range); 692 } 693 694 static bool 695 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 696 const struct pnfs_layout_range *recall_range, 697 u32 seq) 698 { 699 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 700 return false; 701 if (recall_range == NULL) 702 return true; 703 return pnfs_should_free_range(&lseg->pls_range, recall_range); 704 } 705 706 /** 707 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 708 * @lo: layout header containing the lsegs 709 * @tmp_list: list head where doomed lsegs should go 710 * @recall_range: optional recall range argument to match (may be NULL) 711 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 712 * 713 * Walk the list of lsegs in the layout header, and tear down any that should 714 * be destroyed. If "recall_range" is specified then the segment must match 715 * that range. If "seq" is non-zero, then only match segments that were handed 716 * out at or before that sequence. 717 * 718 * Returns number of matching invalid lsegs remaining in list after scanning 719 * it and purging them. 720 */ 721 int 722 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 723 struct list_head *tmp_list, 724 const struct pnfs_layout_range *recall_range, 725 u32 seq) 726 { 727 struct pnfs_layout_segment *lseg, *next; 728 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 729 int remaining = 0; 730 731 dprintk("%s:Begin lo %p\n", __func__, lo); 732 733 if (list_empty(&lo->plh_segs)) 734 return 0; 735 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 736 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 737 dprintk("%s: freeing lseg %p iomode %d seq %u " 738 "offset %llu length %llu\n", __func__, 739 lseg, lseg->pls_range.iomode, lseg->pls_seq, 740 lseg->pls_range.offset, lseg->pls_range.length); 741 if (mark_lseg_invalid(lseg, tmp_list)) 742 continue; 743 remaining++; 744 pnfs_lseg_cancel_io(server, lseg, NULL); 745 } 746 dprintk("%s:Return %i\n", __func__, remaining); 747 return remaining; 748 } 749 750 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) 751 { 752 struct pnfs_layout_segment *lseg; 753 754 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) 755 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 756 } 757 758 static void 759 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 760 struct list_head *free_me, 761 const struct pnfs_layout_range *range, 762 u32 seq) 763 { 764 struct pnfs_layout_segment *lseg, *next; 765 766 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 767 if (pnfs_match_lseg_recall(lseg, range, seq)) 768 list_move_tail(&lseg->pls_list, free_me); 769 } 770 } 771 772 /* note free_me must contain lsegs from a single layout_hdr */ 773 void 774 pnfs_free_lseg_list(struct list_head *free_me) 775 { 776 struct pnfs_layout_segment *lseg, *tmp; 777 778 if (list_empty(free_me)) 779 return; 780 781 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 782 list_del(&lseg->pls_list); 783 pnfs_free_lseg(lseg); 784 } 785 } 786 787 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) 788 { 789 struct pnfs_layout_hdr *lo; 790 LIST_HEAD(tmp_list); 791 792 spin_lock(&nfsi->vfs_inode.i_lock); 793 lo = nfsi->layout; 794 if (lo) { 795 pnfs_get_layout_hdr(lo); 796 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 797 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 798 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 799 spin_unlock(&nfsi->vfs_inode.i_lock); 800 pnfs_free_lseg_list(&tmp_list); 801 nfs_commit_inode(&nfsi->vfs_inode, 0); 802 pnfs_put_layout_hdr(lo); 803 } else 804 spin_unlock(&nfsi->vfs_inode.i_lock); 805 return lo; 806 } 807 808 void pnfs_destroy_layout(struct nfs_inode *nfsi) 809 { 810 __pnfs_destroy_layout(nfsi); 811 } 812 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 813 814 void pnfs_destroy_layout_final(struct nfs_inode *nfsi) 815 { 816 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); 817 struct inode *inode = &nfsi->vfs_inode; 818 819 if (lo) { 820 spin_lock(&inode->i_lock); 821 wait_var_event_spinlock(lo, nfsi->layout != lo, 822 &inode->i_lock); 823 spin_unlock(&inode->i_lock); 824 } 825 } 826 827 static bool 828 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 829 struct list_head *layout_list) 830 { 831 struct pnfs_layout_hdr *lo; 832 bool ret = false; 833 834 spin_lock(&inode->i_lock); 835 lo = NFS_I(inode)->layout; 836 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 837 pnfs_get_layout_hdr(lo); 838 list_add(&lo->plh_bulk_destroy, layout_list); 839 ret = true; 840 } 841 spin_unlock(&inode->i_lock); 842 return ret; 843 } 844 845 /* Caller must hold rcu_read_lock and clp->cl_lock */ 846 static int 847 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 848 struct nfs_server *server, 849 struct list_head *layout_list) 850 __must_hold(&clp->cl_lock) 851 __must_hold(RCU) 852 { 853 struct pnfs_layout_hdr *lo, *next; 854 struct inode *inode; 855 856 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 857 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || 858 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || 859 !list_empty(&lo->plh_bulk_destroy)) 860 continue; 861 /* If the sb is being destroyed, just bail */ 862 if (!nfs_sb_active(server->super)) 863 break; 864 inode = pnfs_grab_inode_layout_hdr(lo); 865 if (inode != NULL) { 866 if (pnfs_layout_add_bulk_destroy_list(inode, 867 layout_list)) 868 continue; 869 rcu_read_unlock(); 870 spin_unlock(&clp->cl_lock); 871 iput(inode); 872 } else { 873 rcu_read_unlock(); 874 spin_unlock(&clp->cl_lock); 875 } 876 nfs_sb_deactive(server->super); 877 spin_lock(&clp->cl_lock); 878 rcu_read_lock(); 879 return -EAGAIN; 880 } 881 return 0; 882 } 883 884 static int 885 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 886 enum pnfs_layout_destroy_mode mode) 887 { 888 struct pnfs_layout_hdr *lo; 889 struct inode *inode; 890 LIST_HEAD(lseg_list); 891 int ret = 0; 892 893 while (!list_empty(layout_list)) { 894 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 895 plh_bulk_destroy); 896 dprintk("%s freeing layout for inode %llu\n", __func__, 897 lo->plh_inode->i_ino); 898 inode = lo->plh_inode; 899 900 pnfs_layoutcommit_inode(inode, false); 901 902 spin_lock(&inode->i_lock); 903 list_del_init(&lo->plh_bulk_destroy); 904 if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { 905 pnfs_mark_layout_stateid_return(lo, &lseg_list, 906 IOMODE_ANY, 0); 907 } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 908 if (mode == PNFS_LAYOUT_BULK_RETURN) 909 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 910 ret = -EAGAIN; 911 } 912 spin_unlock(&inode->i_lock); 913 pnfs_free_lseg_list(&lseg_list); 914 /* Free all lsegs that are attached to commit buckets */ 915 nfs_commit_inode(inode, 0); 916 pnfs_put_layout_hdr(lo); 917 nfs_iput_and_deactive(inode); 918 } 919 return ret; 920 } 921 922 int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, 923 enum pnfs_layout_destroy_mode mode) 924 { 925 struct nfs_server *server; 926 LIST_HEAD(layout_list); 927 928 spin_lock(&clp->cl_lock); 929 rcu_read_lock(); 930 restart: 931 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 932 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 933 continue; 934 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 935 server, 936 &layout_list) != 0) 937 goto restart; 938 } 939 rcu_read_unlock(); 940 spin_unlock(&clp->cl_lock); 941 942 return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); 943 } 944 945 static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, 946 struct list_head *list) 947 { 948 struct nfs_server *server; 949 950 spin_lock(&clp->cl_lock); 951 rcu_read_lock(); 952 restart: 953 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 954 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 955 list) != 0) 956 goto restart; 957 } 958 rcu_read_unlock(); 959 spin_unlock(&clp->cl_lock); 960 } 961 962 static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, 963 struct list_head *list, 964 enum pnfs_layout_destroy_mode mode) 965 { 966 pnfs_layout_build_destroy_list_byclient(clp, list); 967 return pnfs_layout_free_bulk_destroy_list(list, mode); 968 } 969 970 int pnfs_layout_destroy_byclid(struct nfs_client *clp, 971 enum pnfs_layout_destroy_mode mode) 972 { 973 LIST_HEAD(layout_list); 974 975 return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); 976 } 977 978 /* 979 * Called by the state manager to remove all layouts established under an 980 * expired lease. 981 */ 982 void 983 pnfs_destroy_all_layouts(struct nfs_client *clp) 984 { 985 nfs4_deviceid_mark_client_invalid(clp); 986 nfs4_deviceid_purge_client(clp); 987 988 pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); 989 } 990 991 static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, 992 struct list_head *list) 993 { 994 struct nfs_server *server; 995 996 spin_lock(&clp->cl_lock); 997 rcu_read_lock(); 998 restart: 999 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 1000 if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) 1001 continue; 1002 if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, 1003 list) != 0) 1004 goto restart; 1005 } 1006 rcu_read_unlock(); 1007 spin_unlock(&clp->cl_lock); 1008 } 1009 1010 static int pnfs_layout_bulk_list_reboot(struct list_head *list) 1011 { 1012 struct pnfs_layout_hdr *lo; 1013 struct nfs_server *server; 1014 int ret; 1015 1016 list_for_each_entry(lo, list, plh_bulk_destroy) { 1017 server = NFS_SERVER(lo->plh_inode); 1018 ret = pnfs_layout_return_on_reboot(lo); 1019 switch (ret) { 1020 case 0: 1021 continue; 1022 case -NFS4ERR_BAD_STATEID: 1023 server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; 1024 break; 1025 case -NFS4ERR_NO_GRACE: 1026 break; 1027 default: 1028 goto err; 1029 } 1030 break; 1031 } 1032 return 0; 1033 err: 1034 return ret; 1035 } 1036 1037 int pnfs_layout_handle_reboot(struct nfs_client *clp) 1038 { 1039 LIST_HEAD(list); 1040 int ret = 0, ret2; 1041 1042 pnfs_layout_build_recover_list_byclient(clp, &list); 1043 if (!list_empty(&list)) 1044 ret = pnfs_layout_bulk_list_reboot(&list); 1045 ret2 = pnfs_layout_do_destroy_byclid(clp, &list, 1046 PNFS_LAYOUT_INVALIDATE); 1047 if (!ret) 1048 ret = ret2; 1049 return (ret == 0) ? 0 : -EAGAIN; 1050 } 1051 1052 static void 1053 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) 1054 { 1055 const struct cred *old; 1056 1057 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { 1058 old = xchg(&lo->plh_lc_cred, get_cred(cred)); 1059 put_cred(old); 1060 } 1061 } 1062 1063 /* update lo->plh_stateid with new if is more recent */ 1064 void 1065 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 1066 const struct cred *cred, bool update_barrier) 1067 { 1068 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 1069 u32 newseq = be32_to_cpu(new->seqid); 1070 1071 if (!pnfs_layout_is_valid(lo)) { 1072 pnfs_set_layout_cred(lo, cred); 1073 nfs4_stateid_copy(&lo->plh_stateid, new); 1074 lo->plh_barrier = newseq; 1075 pnfs_clear_layoutreturn_info(lo); 1076 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1077 return; 1078 } 1079 1080 if (pnfs_seqid_is_newer(newseq, oldseq)) 1081 nfs4_stateid_copy(&lo->plh_stateid, new); 1082 1083 if (update_barrier) { 1084 pnfs_barrier_update(lo, newseq); 1085 return; 1086 } 1087 /* 1088 * Because of wraparound, we want to keep the barrier 1089 * "close" to the current seqids. We really only want to 1090 * get here from a layoutget call. 1091 */ 1092 if (atomic_read(&lo->plh_outstanding) == 1) 1093 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); 1094 } 1095 1096 static bool 1097 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 1098 const nfs4_stateid *stateid) 1099 { 1100 u32 seqid = be32_to_cpu(stateid->seqid); 1101 1102 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); 1103 } 1104 1105 /* lget is set to 1 if called from inside send_layoutget call chain */ 1106 static bool 1107 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 1108 { 1109 return lo->plh_block_lgets || 1110 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 1111 } 1112 1113 static struct nfs_server * 1114 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) 1115 { 1116 struct nfs_server *server; 1117 1118 if (inode) { 1119 server = NFS_SERVER(inode); 1120 } else { 1121 struct dentry *parent_dir = dget_parent(ctx->dentry); 1122 server = NFS_SERVER(parent_dir->d_inode); 1123 dput(parent_dir); 1124 } 1125 return server; 1126 } 1127 1128 static void nfs4_free_pages(struct page **pages, size_t size) 1129 { 1130 int i; 1131 1132 if (!pages) 1133 return; 1134 1135 for (i = 0; i < size; i++) { 1136 if (!pages[i]) 1137 break; 1138 __free_page(pages[i]); 1139 } 1140 kfree(pages); 1141 } 1142 1143 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) 1144 { 1145 struct page **pages; 1146 int i; 1147 1148 pages = kmalloc_objs(struct page *, size, gfp_flags); 1149 if (!pages) { 1150 dprintk("%s: can't alloc array of %zu pages\n", __func__, size); 1151 return NULL; 1152 } 1153 1154 for (i = 0; i < size; i++) { 1155 pages[i] = alloc_page(gfp_flags); 1156 if (!pages[i]) { 1157 dprintk("%s: failed to allocate page\n", __func__); 1158 nfs4_free_pages(pages, i); 1159 return NULL; 1160 } 1161 } 1162 1163 return pages; 1164 } 1165 1166 static struct nfs4_layoutget * 1167 pnfs_alloc_init_layoutget_args(struct inode *ino, 1168 struct nfs_open_context *ctx, 1169 const nfs4_stateid *stateid, 1170 const struct pnfs_layout_range *range, 1171 size_t min_reply_sz, gfp_t gfp_flags) 1172 { 1173 struct nfs_server *server = pnfs_find_server(ino, ctx); 1174 size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; 1175 size_t session_pages = max_response_pages(server); 1176 size_t max_pages = session_pages; 1177 struct nfs4_layoutget *lgp; 1178 1179 dprintk("--> %s\n", __func__); 1180 1181 lgp = kzalloc_obj(*lgp, gfp_flags); 1182 if (lgp == NULL) 1183 return NULL; 1184 1185 if (max_reply_sz) { 1186 size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1187 if (npages < max_pages) 1188 max_pages = npages; 1189 } 1190 1191 /* 1192 * A previous LAYOUTGET on this layout or on this server did not 1193 * fit the reply buffer: raise the layout driver's default up to 1194 * the session's maximum response size. 1195 */ 1196 if (!min_reply_sz) 1197 min_reply_sz = READ_ONCE(server->lg_reply_sz); 1198 if (min_reply_sz) { 1199 size_t npages = (min_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; 1200 if (npages > max_pages) 1201 max_pages = min(npages, session_pages); 1202 } 1203 1204 lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); 1205 if (!lgp->args.layout.pages) { 1206 kfree(lgp); 1207 return NULL; 1208 } 1209 lgp->args.layout.pglen = max_pages * PAGE_SIZE; 1210 lgp->res.layoutp = &lgp->args.layout; 1211 1212 /* Don't confuse uninitialised result and success */ 1213 lgp->res.status = -NFS4ERR_DELAY; 1214 1215 lgp->args.minlength = PAGE_SIZE; 1216 if (lgp->args.minlength > range->length) 1217 lgp->args.minlength = range->length; 1218 if (ino) { 1219 loff_t i_size = i_size_read(ino); 1220 1221 if (range->iomode == IOMODE_READ) { 1222 if (range->offset >= i_size) 1223 lgp->args.minlength = 0; 1224 else if (i_size - range->offset < lgp->args.minlength) 1225 lgp->args.minlength = i_size - range->offset; 1226 } 1227 } 1228 lgp->args.maxcount = lgp->args.layout.pglen; 1229 pnfs_copy_range(&lgp->args.range, range); 1230 lgp->args.type = server->pnfs_curr_ld->id; 1231 lgp->args.inode = ino; 1232 lgp->args.ctx = get_nfs_open_context(ctx); 1233 nfs4_stateid_copy(&lgp->args.stateid, stateid); 1234 lgp->gfp_flags = gfp_flags; 1235 lgp->cred = ctx->cred; 1236 return lgp; 1237 } 1238 1239 void pnfs_layoutget_free(struct nfs4_layoutget *lgp) 1240 { 1241 size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; 1242 1243 nfs4_free_pages(lgp->args.layout.pages, max_pages); 1244 pnfs_put_layout_hdr(lgp->lo); 1245 put_nfs_open_context(lgp->args.ctx); 1246 kfree(lgp); 1247 } 1248 1249 static void pnfs_clear_layoutcommit(struct inode *inode, 1250 struct list_head *head) 1251 { 1252 struct nfs_inode *nfsi = NFS_I(inode); 1253 struct pnfs_layout_segment *lseg, *tmp; 1254 1255 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 1256 return; 1257 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 1258 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1259 continue; 1260 pnfs_lseg_dec_and_remove_zero(lseg, head); 1261 } 1262 } 1263 1264 static void 1265 pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, 1266 const nfs4_stateid *arg_stateid, 1267 const struct pnfs_layout_range *range, 1268 struct list_head *freeme) 1269 { 1270 if (pnfs_layout_is_valid(lo) && 1271 nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1272 pnfs_reset_return_info(lo); 1273 else 1274 pnfs_mark_layout_stateid_invalid(lo, freeme); 1275 pnfs_clear_layoutreturn_waitbit(lo); 1276 } 1277 1278 void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, 1279 const nfs4_stateid *arg_stateid, 1280 const struct pnfs_layout_range *range) 1281 { 1282 struct inode *inode = lo->plh_inode; 1283 LIST_HEAD(freeme); 1284 1285 spin_lock(&inode->i_lock); 1286 pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); 1287 spin_unlock(&inode->i_lock); 1288 pnfs_free_lseg_list(&freeme); 1289 } 1290 1291 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1292 const nfs4_stateid *arg_stateid, 1293 const struct pnfs_layout_range *range, 1294 const nfs4_stateid *stateid) 1295 { 1296 struct inode *inode = lo->plh_inode; 1297 LIST_HEAD(freeme); 1298 1299 spin_lock(&inode->i_lock); 1300 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1301 goto out_unlock; 1302 if (stateid && pnfs_layout_is_valid(lo)) { 1303 u32 seq = be32_to_cpu(arg_stateid->seqid); 1304 1305 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1306 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1307 pnfs_set_layout_stateid(lo, stateid, NULL, true); 1308 pnfs_reset_return_info(lo); 1309 } else 1310 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1311 out_unlock: 1312 pnfs_clear_layoutreturn_waitbit(lo); 1313 spin_unlock(&inode->i_lock); 1314 pnfs_free_lseg_list(&freeme); 1315 1316 } 1317 1318 static bool 1319 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1320 nfs4_stateid *stateid, 1321 const struct cred **cred, 1322 enum pnfs_iomode *iomode) 1323 { 1324 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1325 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) 1326 return false; 1327 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1328 return false; 1329 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1330 pnfs_get_layout_hdr(lo); 1331 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1332 *cred = get_cred(lo->plh_lc_cred); 1333 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1334 if (lo->plh_return_seq != 0) 1335 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1336 if (iomode != NULL) 1337 *iomode = lo->plh_return_iomode; 1338 pnfs_clear_layoutreturn_info(lo); 1339 } else if (iomode != NULL) 1340 *iomode = IOMODE_ANY; 1341 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); 1342 return true; 1343 } 1344 1345 static void 1346 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1347 struct pnfs_layout_hdr *lo, 1348 const nfs4_stateid *stateid, 1349 enum pnfs_iomode iomode) 1350 { 1351 struct inode *inode = lo->plh_inode; 1352 1353 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1354 args->inode = inode; 1355 args->range.iomode = iomode; 1356 args->range.offset = 0; 1357 args->range.length = NFS4_MAX_UINT64; 1358 args->layout = lo; 1359 nfs4_stateid_copy(&args->stateid, stateid); 1360 } 1361 1362 static int 1363 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, 1364 const nfs4_stateid *stateid, 1365 const struct cred **pcred, 1366 enum pnfs_iomode iomode, 1367 unsigned int flags) 1368 { 1369 struct inode *ino = lo->plh_inode; 1370 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1371 struct nfs4_layoutreturn *lrp; 1372 const struct cred *cred = *pcred; 1373 int status = 0; 1374 1375 *pcred = NULL; 1376 lrp = kzalloc_obj(*lrp, nfs_io_gfp_mask()); 1377 if (unlikely(lrp == NULL)) { 1378 status = -ENOMEM; 1379 spin_lock(&ino->i_lock); 1380 pnfs_clear_layoutreturn_waitbit(lo); 1381 spin_unlock(&ino->i_lock); 1382 put_cred(cred); 1383 pnfs_put_layout_hdr(lo); 1384 goto out; 1385 } 1386 1387 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1388 lrp->args.ld_private = &lrp->ld_private; 1389 lrp->clp = NFS_SERVER(ino)->nfs_client; 1390 lrp->cred = cred; 1391 if (ld->prepare_layoutreturn) 1392 ld->prepare_layoutreturn(&lrp->args); 1393 1394 status = nfs4_proc_layoutreturn(lrp, flags); 1395 out: 1396 dprintk("<-- %s status: %d\n", __func__, status); 1397 return status; 1398 } 1399 1400 /* Return true if layoutreturn is needed */ 1401 static bool 1402 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1403 { 1404 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1405 return false; 1406 return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, 1407 lo->plh_return_iomode, 1408 lo->plh_return_seq) != -EBUSY; 1409 } 1410 1411 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1412 { 1413 struct inode *inode= lo->plh_inode; 1414 1415 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1416 return; 1417 spin_lock(&inode->i_lock); 1418 if (pnfs_layout_need_return(lo)) { 1419 const struct cred *cred; 1420 nfs4_stateid stateid; 1421 enum pnfs_iomode iomode; 1422 bool send; 1423 1424 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 1425 spin_unlock(&inode->i_lock); 1426 if (send) { 1427 /* Send an async layoutreturn so we dont deadlock */ 1428 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 1429 PNFS_FL_LAYOUTRETURN_ASYNC); 1430 } 1431 } else 1432 spin_unlock(&inode->i_lock); 1433 } 1434 1435 /* 1436 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1437 * when the layout segment list is empty. 1438 * 1439 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1440 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1441 * deviceid is marked invalid. 1442 */ 1443 int 1444 _pnfs_return_layout(struct inode *ino) 1445 { 1446 struct pnfs_layout_hdr *lo = NULL; 1447 struct nfs_inode *nfsi = NFS_I(ino); 1448 struct pnfs_layout_range range = { 1449 .iomode = IOMODE_ANY, 1450 .offset = 0, 1451 .length = NFS4_MAX_UINT64, 1452 }; 1453 LIST_HEAD(tmp_list); 1454 const struct cred *cred; 1455 nfs4_stateid stateid; 1456 int status = 0; 1457 bool send, valid_layout; 1458 1459 dprintk("NFS: %s for inode %llu\n", __func__, ino->i_ino); 1460 1461 spin_lock(&ino->i_lock); 1462 lo = nfsi->layout; 1463 if (!lo) { 1464 spin_unlock(&ino->i_lock); 1465 dprintk("NFS: %s no layout to return\n", __func__); 1466 goto out; 1467 } 1468 /* Reference matched in nfs4_layoutreturn_release */ 1469 pnfs_get_layout_hdr(lo); 1470 /* Is there an outstanding layoutreturn ? */ 1471 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1472 spin_unlock(&ino->i_lock); 1473 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1474 TASK_UNINTERRUPTIBLE)) 1475 goto out_put_layout_hdr; 1476 spin_lock(&ino->i_lock); 1477 } 1478 valid_layout = pnfs_layout_is_valid(lo); 1479 pnfs_clear_layoutcommit(ino, &tmp_list); 1480 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0, true, NULL); 1481 1482 1483 /* Don't send a LAYOUTRETURN if list was initially empty */ 1484 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || 1485 !valid_layout) { 1486 spin_unlock(&ino->i_lock); 1487 dprintk("NFS: %s no layout segments to return\n", __func__); 1488 goto out_wait_layoutreturn; 1489 } 1490 1491 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); 1492 spin_unlock(&ino->i_lock); 1493 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) 1494 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1495 if (send) 1496 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 1497 0); 1498 out_wait_layoutreturn: 1499 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); 1500 out_put_layout_hdr: 1501 pnfs_free_lseg_list(&tmp_list); 1502 pnfs_put_layout_hdr(lo); 1503 out: 1504 dprintk("<-- %s status: %d\n", __func__, status); 1505 return status; 1506 } 1507 1508 int 1509 pnfs_commit_and_return_layout(struct inode *inode) 1510 { 1511 struct pnfs_layout_hdr *lo; 1512 int ret; 1513 1514 spin_lock(&inode->i_lock); 1515 lo = NFS_I(inode)->layout; 1516 if (lo == NULL) { 1517 spin_unlock(&inode->i_lock); 1518 return 0; 1519 } 1520 pnfs_get_layout_hdr(lo); 1521 /* Block new layoutgets and read/write to ds */ 1522 lo->plh_block_lgets++; 1523 spin_unlock(&inode->i_lock); 1524 filemap_fdatawait(inode->i_mapping); 1525 ret = pnfs_layoutcommit_inode(inode, true); 1526 if (ret == 0) 1527 ret = _pnfs_return_layout(inode); 1528 spin_lock(&inode->i_lock); 1529 lo->plh_block_lgets--; 1530 spin_unlock(&inode->i_lock); 1531 pnfs_put_layout_hdr(lo); 1532 return ret; 1533 } 1534 1535 static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) 1536 { 1537 struct inode *inode = lo->plh_inode; 1538 const struct cred *cred; 1539 1540 spin_lock(&inode->i_lock); 1541 if (!pnfs_layout_is_valid(lo)) { 1542 spin_unlock(&inode->i_lock); 1543 return 0; 1544 } 1545 cred = get_cred(lo->plh_lc_cred); 1546 pnfs_get_layout_hdr(lo); 1547 spin_unlock(&inode->i_lock); 1548 1549 return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, 1550 PNFS_FL_LAYOUTRETURN_PRIVILEGED); 1551 } 1552 1553 bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, 1554 struct nfs4_layoutreturn_res *res, const struct cred *cred, 1555 bool sync) 1556 { 1557 struct nfs_inode *nfsi = NFS_I(ino); 1558 struct nfs_open_context *ctx; 1559 struct nfs4_state *state; 1560 struct pnfs_layout_hdr *lo; 1561 struct pnfs_layout_segment *lseg, *next; 1562 const struct cred *lc_cred; 1563 nfs4_stateid stateid; 1564 enum pnfs_iomode iomode = 0; 1565 bool layoutreturn = false, roc = false; 1566 bool skip_read; 1567 1568 if (!nfs_have_layout(ino)) 1569 return false; 1570 retry: 1571 rcu_read_lock(); 1572 spin_lock(&ino->i_lock); 1573 lo = nfsi->layout; 1574 if (!lo || !pnfs_layout_is_valid(lo) || 1575 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1576 lo = NULL; 1577 goto out_noroc; 1578 } 1579 1580 /* no roc if we hold a delegation */ 1581 skip_read = false; 1582 if (nfs4_check_delegation(ino, FMODE_READ)) { 1583 if (nfs4_check_delegation(ino, FMODE_WRITE)) { 1584 lo = NULL; 1585 goto out_noroc; 1586 } 1587 skip_read = true; 1588 } 1589 1590 list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { 1591 state = ctx->state; 1592 if (state == NULL) 1593 continue; 1594 /* Don't return layout if there is open file state */ 1595 if (state->state & FMODE_WRITE) { 1596 lo = NULL; 1597 goto out_noroc; 1598 } 1599 if (state->state & FMODE_READ) 1600 skip_read = true; 1601 } 1602 1603 if (skip_read) { 1604 bool writes = false; 1605 1606 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1607 if (lseg->pls_range.iomode != IOMODE_READ) { 1608 writes = true; 1609 break; 1610 } 1611 } 1612 if (!writes) { 1613 lo = NULL; 1614 goto out_noroc; 1615 } 1616 } 1617 1618 pnfs_get_layout_hdr(lo); 1619 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1620 if (!sync) { 1621 pnfs_set_plh_return_info( 1622 lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); 1623 goto out_noroc; 1624 } 1625 spin_unlock(&ino->i_lock); 1626 rcu_read_unlock(); 1627 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1628 TASK_UNINTERRUPTIBLE); 1629 pnfs_put_layout_hdr(lo); 1630 goto retry; 1631 } 1632 1633 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1634 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1635 continue; 1636 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1637 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1638 continue; 1639 /* 1640 * Note: mark lseg for return so pnfs_layout_remove_lseg 1641 * doesn't invalidate the layout for us. 1642 */ 1643 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1644 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1645 continue; 1646 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1647 } 1648 1649 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1650 goto out_noroc; 1651 1652 /* ROC in two conditions: 1653 * 1. there are ROC lsegs 1654 * 2. we don't send layoutreturn 1655 */ 1656 /* lo ref dropped in pnfs_roc_release() */ 1657 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); 1658 /* If the creds don't match, we can't compound the layoutreturn */ 1659 if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) 1660 goto out_noroc; 1661 1662 roc = layoutreturn; 1663 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1664 res->lrs_present = 0; 1665 layoutreturn = false; 1666 put_cred(lc_cred); 1667 1668 out_noroc: 1669 spin_unlock(&ino->i_lock); 1670 rcu_read_unlock(); 1671 pnfs_layoutcommit_inode(ino, sync); 1672 if (roc) { 1673 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1674 if (ld->prepare_layoutreturn) 1675 ld->prepare_layoutreturn(args); 1676 pnfs_put_layout_hdr(lo); 1677 return true; 1678 } 1679 if (layoutreturn) 1680 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); 1681 pnfs_put_layout_hdr(lo); 1682 return false; 1683 } 1684 1685 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, 1686 struct nfs4_layoutreturn_res **respp, int *ret) 1687 { 1688 struct nfs4_layoutreturn_args *arg = *argpp; 1689 int retval = -EAGAIN; 1690 1691 if (!arg) 1692 return 0; 1693 /* Handle Layoutreturn errors */ 1694 switch (*ret) { 1695 case 0: 1696 retval = 0; 1697 break; 1698 case -NFS4ERR_NOMATCHING_LAYOUT: 1699 /* Was there an RPC level error? If not, retry */ 1700 if (task->tk_rpc_status == 0) 1701 break; 1702 /* 1703 * Is there a fatal network level error? 1704 * If so release the layout, but flag the error. 1705 */ 1706 if ((task->tk_rpc_status == -ENETDOWN || 1707 task->tk_rpc_status == -ENETUNREACH) && 1708 task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { 1709 *ret = 0; 1710 (*respp)->lrs_present = 0; 1711 retval = -EIO; 1712 break; 1713 } 1714 /* If the call was not sent, let caller handle it */ 1715 if (!RPC_WAS_SENT(task)) 1716 return 0; 1717 switch (task->tk_rpc_status) { 1718 default: 1719 /* 1720 * Defer the layoutreturn if it was due 1721 * to the server being down. 1722 */ 1723 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1724 break; 1725 case -EACCES: 1726 case -EIO: 1727 case -EKEYEXPIRED: 1728 case -ERESTARTSYS: 1729 case -EINTR: 1730 /* Don't retry */ 1731 *ret = 0; 1732 break; 1733 } 1734 (*respp)->lrs_present = 0; 1735 retval = 0; 1736 break; 1737 case -NFS4ERR_DELAY: 1738 /* Let the caller handle the retry */ 1739 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1740 return 0; 1741 case -NFS4ERR_OLD_STATEID: 1742 if (!nfs4_layout_refresh_old_stateid(&arg->stateid, 1743 &arg->range, arg->inode)) 1744 break; 1745 *ret = -NFS4ERR_NOMATCHING_LAYOUT; 1746 return -EAGAIN; 1747 } 1748 *argpp = NULL; 1749 *respp = NULL; 1750 return retval; 1751 } 1752 1753 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1754 struct nfs4_layoutreturn_res *res, int ret) 1755 { 1756 struct pnfs_layout_hdr *lo = args->layout; 1757 struct inode *inode = args->inode; 1758 const nfs4_stateid *res_stateid = NULL; 1759 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1760 LIST_HEAD(freeme); 1761 1762 switch (ret) { 1763 case -NFS4ERR_BADSESSION: 1764 case -NFS4ERR_DEADSESSION: 1765 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1766 case -NFS4ERR_NOMATCHING_LAYOUT: 1767 spin_lock(&inode->i_lock); 1768 pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, 1769 &args->range, &freeme); 1770 spin_unlock(&inode->i_lock); 1771 pnfs_free_lseg_list(&freeme); 1772 break; 1773 case 0: 1774 if (res->lrs_present) 1775 res_stateid = &res->stateid; 1776 fallthrough; 1777 default: 1778 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, 1779 res_stateid); 1780 } 1781 trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); 1782 if (ld_private && ld_private->ops && ld_private->ops->free) 1783 ld_private->ops->free(ld_private); 1784 pnfs_put_layout_hdr(lo); 1785 } 1786 1787 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1788 { 1789 struct nfs_inode *nfsi = NFS_I(ino); 1790 struct pnfs_layout_hdr *lo; 1791 bool sleep = false; 1792 1793 /* we might not have grabbed lo reference. so need to check under 1794 * i_lock */ 1795 spin_lock(&ino->i_lock); 1796 lo = nfsi->layout; 1797 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1798 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1799 sleep = true; 1800 } 1801 spin_unlock(&ino->i_lock); 1802 return sleep; 1803 } 1804 1805 /* 1806 * Compare two layout segments for sorting into layout cache. 1807 * We want to preferentially return RW over RO layouts, so ensure those 1808 * are seen first. 1809 */ 1810 static s64 1811 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1812 const struct pnfs_layout_range *l2) 1813 { 1814 s64 d; 1815 1816 /* high offset > low offset */ 1817 d = l1->offset - l2->offset; 1818 if (d) 1819 return d; 1820 1821 /* short length > long length */ 1822 d = l2->length - l1->length; 1823 if (d) 1824 return d; 1825 1826 /* read > read/write */ 1827 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1828 } 1829 1830 static bool 1831 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1832 const struct pnfs_layout_range *l2) 1833 { 1834 return pnfs_lseg_range_cmp(l1, l2) > 0; 1835 } 1836 1837 static bool 1838 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1839 struct pnfs_layout_segment *old) 1840 { 1841 return false; 1842 } 1843 1844 void 1845 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1846 struct pnfs_layout_segment *lseg, 1847 bool (*is_after)(const struct pnfs_layout_range *, 1848 const struct pnfs_layout_range *), 1849 bool (*do_merge)(struct pnfs_layout_segment *, 1850 struct pnfs_layout_segment *), 1851 struct list_head *free_me) 1852 { 1853 struct pnfs_layout_segment *lp, *tmp; 1854 1855 dprintk("%s:Begin\n", __func__); 1856 1857 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1858 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1859 continue; 1860 if (do_merge(lseg, lp)) { 1861 mark_lseg_invalid(lp, free_me); 1862 continue; 1863 } 1864 if (is_after(&lseg->pls_range, &lp->pls_range)) 1865 continue; 1866 list_add_tail(&lseg->pls_list, &lp->pls_list); 1867 dprintk("%s: inserted lseg %p " 1868 "iomode %d offset %llu length %llu before " 1869 "lp %p iomode %d offset %llu length %llu\n", 1870 __func__, lseg, lseg->pls_range.iomode, 1871 lseg->pls_range.offset, lseg->pls_range.length, 1872 lp, lp->pls_range.iomode, lp->pls_range.offset, 1873 lp->pls_range.length); 1874 goto out; 1875 } 1876 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1877 dprintk("%s: inserted lseg %p " 1878 "iomode %d offset %llu length %llu at tail\n", 1879 __func__, lseg, lseg->pls_range.iomode, 1880 lseg->pls_range.offset, lseg->pls_range.length); 1881 out: 1882 pnfs_get_layout_hdr(lo); 1883 1884 dprintk("%s:Return\n", __func__); 1885 } 1886 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1887 1888 static void 1889 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1890 struct pnfs_layout_segment *lseg, 1891 struct list_head *free_me) 1892 { 1893 struct inode *inode = lo->plh_inode; 1894 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1895 1896 if (ld->add_lseg != NULL) 1897 ld->add_lseg(lo, lseg, free_me); 1898 else 1899 pnfs_generic_layout_insert_lseg(lo, lseg, 1900 pnfs_lseg_range_is_after, 1901 pnfs_lseg_no_merge, 1902 free_me); 1903 } 1904 1905 static struct pnfs_layout_hdr * 1906 alloc_init_layout_hdr(struct inode *ino, 1907 struct nfs_open_context *ctx, 1908 gfp_t gfp_flags) 1909 { 1910 struct pnfs_layout_hdr *lo; 1911 1912 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1913 if (!lo) 1914 return NULL; 1915 refcount_set(&lo->plh_refcount, 1); 1916 INIT_LIST_HEAD(&lo->plh_layouts); 1917 INIT_LIST_HEAD(&lo->plh_segs); 1918 INIT_LIST_HEAD(&lo->plh_return_segs); 1919 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1920 lo->plh_inode = ino; 1921 lo->plh_lc_cred = get_cred(ctx->cred); 1922 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1923 return lo; 1924 } 1925 1926 static struct pnfs_layout_hdr * 1927 pnfs_find_alloc_layout(struct inode *ino, 1928 struct nfs_open_context *ctx, 1929 gfp_t gfp_flags) 1930 __releases(&ino->i_lock) 1931 __acquires(&ino->i_lock) 1932 { 1933 struct nfs_inode *nfsi = NFS_I(ino); 1934 struct pnfs_layout_hdr *new = NULL; 1935 1936 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1937 1938 if (nfsi->layout != NULL) 1939 goto out_existing; 1940 spin_unlock(&ino->i_lock); 1941 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1942 spin_lock(&ino->i_lock); 1943 1944 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1945 nfsi->layout = new; 1946 return new; 1947 } else if (new != NULL) 1948 pnfs_free_layout_hdr(new); 1949 out_existing: 1950 pnfs_get_layout_hdr(nfsi->layout); 1951 return nfsi->layout; 1952 } 1953 1954 /* 1955 * iomode matching rules: 1956 * iomode lseg strict match 1957 * iomode 1958 * ----- ----- ------ ----- 1959 * ANY READ N/A true 1960 * ANY RW N/A true 1961 * RW READ N/A false 1962 * RW RW N/A true 1963 * READ READ N/A true 1964 * READ RW true false 1965 * READ RW false true 1966 */ 1967 static bool 1968 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1969 const struct pnfs_layout_range *range, 1970 bool strict_iomode) 1971 { 1972 struct pnfs_layout_range range1; 1973 1974 if ((range->iomode == IOMODE_RW && 1975 ls_range->iomode != IOMODE_RW) || 1976 (range->iomode != ls_range->iomode && 1977 strict_iomode) || 1978 !pnfs_lseg_range_intersecting(ls_range, range)) 1979 return false; 1980 1981 /* range1 covers only the first byte in the range */ 1982 range1 = *range; 1983 range1.length = 1; 1984 return pnfs_lseg_range_contained(ls_range, &range1); 1985 } 1986 1987 /* 1988 * lookup range in layout 1989 */ 1990 static struct pnfs_layout_segment * 1991 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1992 struct pnfs_layout_range *range, 1993 bool strict_iomode) 1994 { 1995 struct pnfs_layout_segment *lseg, *ret = NULL; 1996 1997 dprintk("%s:Begin\n", __func__); 1998 1999 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2000 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 2001 pnfs_lseg_range_match(&lseg->pls_range, range, 2002 strict_iomode)) { 2003 ret = pnfs_get_lseg(lseg); 2004 break; 2005 } 2006 } 2007 2008 dprintk("%s:Return lseg %p ref %d\n", 2009 __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); 2010 return ret; 2011 } 2012 2013 /* 2014 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 2015 * to the MDS or over pNFS 2016 * 2017 * The nfs_inode read_io and write_io fields are cumulative counters reset 2018 * when there are no layout segments. Note that in pnfs_update_layout iomode 2019 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 2020 * WRITE request. 2021 * 2022 * A return of true means use MDS I/O, so for a file whose layout forbids 2023 * that (see NFS_INO_NO_IO_THRU_MDS) the hint is not evaluated at all: 2024 * there is no answer it could give that this client may act on. 2025 * 2026 * From rfc 5661: 2027 * If a file's size is smaller than the file size threshold, data accesses 2028 * SHOULD be sent to the metadata server. If an I/O request has a length that 2029 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 2030 * server. If both file size and I/O size are provided, the client SHOULD 2031 * reach or exceed both thresholds before sending its read or write 2032 * requests to the data server. 2033 */ 2034 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 2035 struct inode *ino, int iomode) 2036 { 2037 struct nfs4_threshold *t = ctx->mdsthreshold; 2038 struct nfs_inode *nfsi = NFS_I(ino); 2039 loff_t fsize = i_size_read(ino); 2040 bool size = false, size_set = false, io = false, io_set = false, ret = false; 2041 2042 if (t == NULL) 2043 return ret; 2044 2045 /* 2046 * The server has told a layout driver that this file's I/O may not 2047 * go through the MDS (flexfiles FF_FLAGS_NO_IO_THRU_MDS). Honor 2048 * that over its own mdsthreshold hint. 2049 */ 2050 if (nfs_no_io_thru_mds(ino)) 2051 return ret; 2052 2053 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 2054 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 2055 2056 switch (iomode) { 2057 case IOMODE_READ: 2058 if (t->bm & THRESHOLD_RD) { 2059 dprintk("%s fsize %llu\n", __func__, fsize); 2060 size_set = true; 2061 if (fsize < t->rd_sz) 2062 size = true; 2063 } 2064 if (t->bm & THRESHOLD_RD_IO) { 2065 dprintk("%s nfsi->read_io %llu\n", __func__, 2066 nfsi->read_io); 2067 io_set = true; 2068 if (nfsi->read_io < t->rd_io_sz) 2069 io = true; 2070 } 2071 break; 2072 case IOMODE_RW: 2073 if (t->bm & THRESHOLD_WR) { 2074 dprintk("%s fsize %llu\n", __func__, fsize); 2075 size_set = true; 2076 if (fsize < t->wr_sz) 2077 size = true; 2078 } 2079 if (t->bm & THRESHOLD_WR_IO) { 2080 dprintk("%s nfsi->write_io %llu\n", __func__, 2081 nfsi->write_io); 2082 io_set = true; 2083 if (nfsi->write_io < t->wr_io_sz) 2084 io = true; 2085 } 2086 break; 2087 } 2088 if (size_set && io_set) { 2089 if (size && io) 2090 ret = true; 2091 } else if (size || io) 2092 ret = true; 2093 2094 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 2095 return ret; 2096 } 2097 2098 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 2099 { 2100 /* 2101 * send layoutcommit as it can hold up layoutreturn due to lseg 2102 * reference 2103 */ 2104 pnfs_layoutcommit_inode(lo->plh_inode, false); 2105 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 2106 nfs_wait_bit_killable, 2107 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 2108 } 2109 2110 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) 2111 { 2112 atomic_inc(&lo->plh_outstanding); 2113 } 2114 2115 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) 2116 { 2117 if (atomic_dec_and_test(&lo->plh_outstanding) && 2118 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { 2119 smp_mb__after_atomic(); 2120 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); 2121 } 2122 } 2123 2124 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) 2125 { 2126 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2127 } 2128 2129 static void _add_to_server_list(struct pnfs_layout_hdr *lo, 2130 struct nfs_server *server) 2131 { 2132 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { 2133 struct nfs_client *clp = server->nfs_client; 2134 2135 /* The lo must be on the clp list if there is any 2136 * chance of a CB_LAYOUTRECALL(FILE) coming in. 2137 */ 2138 spin_lock(&clp->cl_lock); 2139 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); 2140 spin_unlock(&clp->cl_lock); 2141 } 2142 } 2143 2144 /* 2145 * Layout segment is retreived from the server if not cached. 2146 * The appropriate layout segment is referenced and returned to the caller. 2147 */ 2148 struct pnfs_layout_segment * 2149 pnfs_update_layout(struct inode *ino, 2150 struct nfs_open_context *ctx, 2151 loff_t pos, 2152 u64 count, 2153 enum pnfs_iomode iomode, 2154 bool strict_iomode, 2155 gfp_t gfp_flags) 2156 { 2157 struct pnfs_layout_range arg = { 2158 .iomode = iomode, 2159 .offset = pos, 2160 .length = count, 2161 }; 2162 unsigned pg_offset; 2163 struct nfs_server *server = NFS_SERVER(ino); 2164 struct nfs_client *clp = server->nfs_client; 2165 struct pnfs_layout_hdr *lo = NULL; 2166 struct pnfs_layout_segment *lseg = NULL; 2167 struct nfs4_layoutget *lgp; 2168 nfs4_stateid stateid; 2169 struct nfs4_exception exception = { 2170 .inode = ino, 2171 }; 2172 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 2173 size_t reply_sz = 0; 2174 bool first; 2175 2176 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 2177 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2178 PNFS_UPDATE_LAYOUT_NO_PNFS); 2179 goto out; 2180 } 2181 2182 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 2183 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2184 PNFS_UPDATE_LAYOUT_MDSTHRESH); 2185 goto out; 2186 } 2187 2188 lookup_again: 2189 if (!nfs4_valid_open_stateid(ctx->state)) { 2190 trace_pnfs_update_layout(ino, pos, count, 2191 iomode, lo, lseg, 2192 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2193 lseg = ERR_PTR(-EIO); 2194 goto out; 2195 } 2196 2197 lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); 2198 if (IS_ERR(lseg)) 2199 goto out; 2200 first = false; 2201 spin_lock(&ino->i_lock); 2202 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 2203 if (lo == NULL) { 2204 spin_unlock(&ino->i_lock); 2205 lseg = ERR_PTR(-ENOMEM); 2206 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2207 PNFS_UPDATE_LAYOUT_NOMEM); 2208 goto out; 2209 } 2210 2211 /* Do we even need to bother with this? */ 2212 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 2213 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2214 PNFS_UPDATE_LAYOUT_BULK_RECALL); 2215 dprintk("%s matches recall, use MDS\n", __func__); 2216 goto out_unlock; 2217 } 2218 2219 /* if LAYOUTGET already failed once we don't try again */ 2220 if (pnfs_layout_io_test_failed(lo, iomode)) { 2221 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2222 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 2223 goto out_unlock; 2224 } 2225 2226 /* 2227 * If the layout segment list is empty, but there are outstanding 2228 * layoutget calls, then they might be subject to a layoutrecall. 2229 */ 2230 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2231 atomic_read(&lo->plh_outstanding) != 0) { 2232 spin_unlock(&ino->i_lock); 2233 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, 2234 TASK_KILLABLE)); 2235 if (IS_ERR(lseg)) 2236 goto out_put_layout_hdr; 2237 pnfs_put_layout_hdr(lo); 2238 goto lookup_again; 2239 } 2240 2241 /* 2242 * Because we free lsegs when sending LAYOUTRETURN, we need to wait 2243 * for LAYOUTRETURN. 2244 */ 2245 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 2246 spin_unlock(&ino->i_lock); 2247 dprintk("%s wait for layoutreturn\n", __func__); 2248 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); 2249 if (!IS_ERR(lseg)) { 2250 dprintk("%s retrying\n", __func__); 2251 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 2252 lseg, 2253 PNFS_UPDATE_LAYOUT_RETRY); 2254 pnfs_put_layout_hdr(lo); 2255 goto lookup_again; 2256 } 2257 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2258 PNFS_UPDATE_LAYOUT_RETURN); 2259 goto out_put_layout_hdr; 2260 } 2261 2262 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 2263 if (lseg) { 2264 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2265 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 2266 goto out_unlock; 2267 } 2268 2269 /* 2270 * Choose a stateid for the LAYOUTGET. If we don't have a layout 2271 * stateid, or it has been invalidated, then we must use the open 2272 * stateid. 2273 */ 2274 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 2275 int status; 2276 2277 /* 2278 * The first layoutget for the file. Need to serialize per 2279 * RFC 5661 Errata 3208. 2280 */ 2281 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2282 &lo->plh_flags)) { 2283 spin_unlock(&ino->i_lock); 2284 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, 2285 NFS_LAYOUT_FIRST_LAYOUTGET, 2286 TASK_KILLABLE)); 2287 if (IS_ERR(lseg)) 2288 goto out_put_layout_hdr; 2289 pnfs_put_layout_hdr(lo); 2290 dprintk("%s retrying\n", __func__); 2291 goto lookup_again; 2292 } 2293 2294 spin_unlock(&ino->i_lock); 2295 first = true; 2296 status = nfs4_select_rw_stateid(ctx->state, 2297 iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, 2298 NULL, &stateid, NULL); 2299 if (status != 0) { 2300 lseg = ERR_PTR(status); 2301 trace_pnfs_update_layout(ino, pos, count, 2302 iomode, lo, lseg, 2303 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 2304 nfs4_schedule_stateid_recovery(server, ctx->state); 2305 clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2306 &lo->plh_flags); 2307 pnfs_put_layout_hdr(lo); 2308 goto lookup_again; 2309 } 2310 spin_lock(&ino->i_lock); 2311 } else { 2312 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 2313 } 2314 2315 if (pnfs_layoutgets_blocked(lo)) { 2316 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2317 PNFS_UPDATE_LAYOUT_BLOCKED); 2318 goto out_unlock; 2319 } 2320 nfs_layoutget_begin(lo); 2321 spin_unlock(&ino->i_lock); 2322 2323 _add_to_server_list(lo, server); 2324 2325 pg_offset = arg.offset & ~PAGE_MASK; 2326 if (pg_offset) { 2327 arg.offset -= pg_offset; 2328 arg.length += pg_offset; 2329 } 2330 if (arg.length != NFS4_MAX_UINT64) 2331 arg.length = PAGE_ALIGN(arg.length); 2332 2333 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, reply_sz, 2334 gfp_flags); 2335 if (!lgp) { 2336 lseg = ERR_PTR(-ENOMEM); 2337 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, 2338 PNFS_UPDATE_LAYOUT_NOMEM); 2339 nfs_layoutget_end(lo); 2340 goto out_put_layout_hdr; 2341 } 2342 2343 lgp->lo = lo; 2344 pnfs_get_layout_hdr(lo); 2345 2346 lseg = nfs4_proc_layoutget(lgp, &exception); 2347 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2348 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 2349 nfs_layoutget_end(lo); 2350 if (IS_ERR(lseg)) { 2351 switch(PTR_ERR(lseg)) { 2352 case -EBUSY: 2353 if (time_after(jiffies, giveup)) 2354 lseg = NULL; 2355 break; 2356 case -ERECALLCONFLICT: 2357 case -EAGAIN: 2358 break; 2359 case -ENODATA: 2360 /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ 2361 pnfs_layout_set_fail_bit( 2362 lo, pnfs_iomode_to_fail_bit(iomode)); 2363 lseg = NULL; 2364 goto out_put_layout_hdr; 2365 case -EMSGSIZE: { 2366 /* 2367 * The layout exceeded loga_maxcount (NFS4ERR_TOOSMALL): 2368 * retry once with the reply buffer raised to the 2369 * session's maximum response size before falling back 2370 * to I/O through the MDS. 2371 */ 2372 size_t max = max_response_pages(server) << PAGE_SHIFT; 2373 2374 if (reply_sz < max) { 2375 reply_sz = max; 2376 exception.retry = 1; 2377 break; 2378 } 2379 pnfs_layout_set_fail_bit( 2380 lo, pnfs_iomode_to_fail_bit(iomode)); 2381 lseg = NULL; 2382 goto out_put_layout_hdr; 2383 } 2384 default: 2385 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 2386 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2387 lseg = NULL; 2388 } 2389 goto out_put_layout_hdr; 2390 } 2391 if (lseg) { 2392 if (!exception.retry) 2393 goto out_put_layout_hdr; 2394 if (first) 2395 clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2396 &lo->plh_flags); 2397 trace_pnfs_update_layout(ino, pos, count, 2398 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 2399 pnfs_put_layout_hdr(lo); 2400 goto lookup_again; 2401 } 2402 } else { 2403 /* 2404 * A LAYOUTGET that only succeeded with an escalated reply 2405 * buffer: remember the size so that future LAYOUTGETs to 2406 * this server skip the attempt at the driver's default. 2407 */ 2408 if (reply_sz) 2409 WRITE_ONCE(server->lg_reply_sz, reply_sz); 2410 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2411 } 2412 2413 out_put_layout_hdr: 2414 if (first) 2415 clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2416 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 2417 PNFS_UPDATE_LAYOUT_EXIT); 2418 pnfs_put_layout_hdr(lo); 2419 out: 2420 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 2421 "(%s, offset: %llu, length: %llu)\n", 2422 __func__, ino->i_sb->s_id, 2423 (unsigned long long)ino->i_ino, 2424 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 2425 iomode==IOMODE_RW ? "read/write" : "read-only", 2426 (unsigned long long)pos, 2427 (unsigned long long)count); 2428 return lseg; 2429 out_unlock: 2430 spin_unlock(&ino->i_lock); 2431 goto out_put_layout_hdr; 2432 } 2433 EXPORT_SYMBOL_GPL(pnfs_update_layout); 2434 2435 static bool 2436 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 2437 { 2438 switch (range->iomode) { 2439 case IOMODE_READ: 2440 case IOMODE_RW: 2441 break; 2442 default: 2443 return false; 2444 } 2445 if (range->offset == NFS4_MAX_UINT64) 2446 return false; 2447 if (range->length == 0) 2448 return false; 2449 if (range->length != NFS4_MAX_UINT64 && 2450 range->length > NFS4_MAX_UINT64 - range->offset) 2451 return false; 2452 return true; 2453 } 2454 2455 static struct pnfs_layout_hdr * 2456 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) 2457 { 2458 struct pnfs_layout_hdr *lo; 2459 2460 spin_lock(&ino->i_lock); 2461 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); 2462 if (!lo) 2463 goto out_unlock; 2464 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 2465 goto out_unlock; 2466 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 2467 goto out_unlock; 2468 if (pnfs_layoutgets_blocked(lo)) 2469 goto out_unlock; 2470 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) 2471 goto out_unlock; 2472 nfs_layoutget_begin(lo); 2473 spin_unlock(&ino->i_lock); 2474 _add_to_server_list(lo, NFS_SERVER(ino)); 2475 return lo; 2476 2477 out_unlock: 2478 spin_unlock(&ino->i_lock); 2479 pnfs_put_layout_hdr(lo); 2480 return NULL; 2481 } 2482 2483 static void _lgopen_prepare_attached(struct nfs4_opendata *data, 2484 struct nfs_open_context *ctx) 2485 { 2486 struct inode *ino = data->dentry->d_inode; 2487 struct pnfs_layout_range rng = { 2488 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2489 IOMODE_RW: IOMODE_READ, 2490 .offset = 0, 2491 .length = NFS4_MAX_UINT64, 2492 }; 2493 struct nfs4_layoutget *lgp; 2494 struct pnfs_layout_hdr *lo; 2495 2496 /* Heuristic: don't send layoutget if we have cached data */ 2497 if (rng.iomode == IOMODE_READ && 2498 (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) 2499 return; 2500 2501 lo = _pnfs_grab_empty_layout(ino, ctx); 2502 if (!lo) 2503 return; 2504 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 0, 2505 nfs_io_gfp_mask()); 2506 if (!lgp) { 2507 clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); 2508 nfs_layoutget_end(lo); 2509 pnfs_put_layout_hdr(lo); 2510 return; 2511 } 2512 lgp->lo = lo; 2513 data->lgp = lgp; 2514 data->o_arg.lg_args = &lgp->args; 2515 data->o_res.lg_res = &lgp->res; 2516 } 2517 2518 static void _lgopen_prepare_floating(struct nfs4_opendata *data, 2519 struct nfs_open_context *ctx) 2520 { 2521 struct inode *ino = data->dentry->d_inode; 2522 struct pnfs_layout_range rng = { 2523 .iomode = (data->o_arg.fmode & FMODE_WRITE) ? 2524 IOMODE_RW: IOMODE_READ, 2525 .offset = 0, 2526 .length = NFS4_MAX_UINT64, 2527 }; 2528 struct nfs4_layoutget *lgp; 2529 2530 lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 0, 2531 nfs_io_gfp_mask()); 2532 if (!lgp) 2533 return; 2534 data->lgp = lgp; 2535 data->o_arg.lg_args = &lgp->args; 2536 data->o_res.lg_res = &lgp->res; 2537 } 2538 2539 void pnfs_lgopen_prepare(struct nfs4_opendata *data, 2540 struct nfs_open_context *ctx) 2541 { 2542 struct nfs_server *server = NFS_SERVER(data->dir->d_inode); 2543 2544 if (!(pnfs_enabled_sb(server) && 2545 server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) 2546 return; 2547 /* Could check on max_ops, but currently hardcoded high enough */ 2548 if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) 2549 return; 2550 if (data->lgp) 2551 return; 2552 if (data->state) 2553 _lgopen_prepare_attached(data, ctx); 2554 else 2555 _lgopen_prepare_floating(data, ctx); 2556 } 2557 2558 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, 2559 struct nfs_open_context *ctx) 2560 { 2561 struct pnfs_layout_hdr *lo; 2562 struct pnfs_layout_segment *lseg; 2563 struct nfs_server *srv = NFS_SERVER(ino); 2564 u32 iomode; 2565 2566 if (!lgp) 2567 return; 2568 dprintk("%s: entered with status %i\n", __func__, lgp->res.status); 2569 if (lgp->res.status) { 2570 switch (lgp->res.status) { 2571 default: 2572 break; 2573 /* 2574 * Halt lgopen attempts if the server doesn't recognise 2575 * the "current stateid" value, the layout type, or the 2576 * layoutget operation as being valid. 2577 * Also if it complains about too many ops in the compound 2578 * or of the request/reply being too big. 2579 */ 2580 case -NFS4ERR_BAD_STATEID: 2581 case -NFS4ERR_NOTSUPP: 2582 case -NFS4ERR_REP_TOO_BIG: 2583 case -NFS4ERR_REP_TOO_BIG_TO_CACHE: 2584 case -NFS4ERR_REQ_TOO_BIG: 2585 case -NFS4ERR_TOO_MANY_OPS: 2586 case -NFS4ERR_UNKNOWN_LAYOUTTYPE: 2587 srv->caps &= ~NFS_CAP_LGOPEN; 2588 } 2589 return; 2590 } 2591 if (!lgp->lo) { 2592 lo = _pnfs_grab_empty_layout(ino, ctx); 2593 if (!lo) 2594 return; 2595 lgp->lo = lo; 2596 } else 2597 lo = lgp->lo; 2598 2599 lseg = pnfs_layout_process(lgp); 2600 if (!IS_ERR(lseg)) { 2601 iomode = lgp->args.range.iomode; 2602 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 2603 pnfs_put_lseg(lseg); 2604 } 2605 } 2606 2607 void nfs4_lgopen_release(struct nfs4_layoutget *lgp) 2608 { 2609 if (lgp != NULL) { 2610 if (lgp->lo) { 2611 clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 2612 &lgp->lo->plh_flags); 2613 nfs_layoutget_end(lgp->lo); 2614 } 2615 pnfs_layoutget_free(lgp); 2616 } 2617 } 2618 2619 struct pnfs_layout_segment * 2620 pnfs_layout_process(struct nfs4_layoutget *lgp) 2621 { 2622 struct pnfs_layout_hdr *lo = lgp->lo; 2623 struct nfs4_layoutget_res *res = &lgp->res; 2624 struct pnfs_layout_segment *lseg; 2625 struct inode *ino = lo->plh_inode; 2626 LIST_HEAD(free_me); 2627 2628 if (!pnfs_sanity_check_layout_range(&res->range)) 2629 return ERR_PTR(-EINVAL); 2630 2631 /* Inject layout blob into I/O device driver */ 2632 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 2633 if (IS_ERR_OR_NULL(lseg)) { 2634 if (!lseg) 2635 lseg = ERR_PTR(-ENOMEM); 2636 2637 dprintk("%s: Could not allocate layout: error %ld\n", 2638 __func__, PTR_ERR(lseg)); 2639 return lseg; 2640 } 2641 2642 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 2643 2644 spin_lock(&ino->i_lock); 2645 if (pnfs_layoutgets_blocked(lo)) { 2646 dprintk("%s forget reply due to state\n", __func__); 2647 goto out_forget; 2648 } 2649 2650 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && 2651 !pnfs_is_first_layoutget(lo)) 2652 goto out_forget; 2653 2654 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 2655 /* existing state ID, make sure the sequence number matches. */ 2656 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 2657 if (!pnfs_layout_is_valid(lo)) 2658 lo->plh_barrier = 0; 2659 dprintk("%s forget reply due to sequence\n", __func__); 2660 goto out_forget; 2661 } 2662 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); 2663 } else if (pnfs_layout_is_valid(lo)) { 2664 /* 2665 * We got an entirely new state ID. Mark all segments for the 2666 * inode invalid, and retry the layoutget 2667 */ 2668 struct pnfs_layout_range range = { 2669 .iomode = IOMODE_ANY, 2670 .length = NFS4_MAX_UINT64, 2671 }; 2672 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0, true, 2673 NULL); 2674 goto out_forget; 2675 } else { 2676 /* We have a completely new layout */ 2677 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); 2678 } 2679 2680 pnfs_get_lseg(lseg); 2681 pnfs_layout_insert_lseg(lo, lseg, &free_me); 2682 2683 2684 if (res->return_on_close) 2685 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 2686 2687 spin_unlock(&ino->i_lock); 2688 pnfs_free_lseg_list(&free_me); 2689 return lseg; 2690 2691 out_forget: 2692 spin_unlock(&ino->i_lock); 2693 lseg->pls_layout = lo; 2694 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 2695 pnfs_free_lseg_list(&free_me); 2696 return ERR_PTR(-EAGAIN); 2697 } 2698 2699 /** 2700 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2701 * @lo: pointer to layout header 2702 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2703 * @return_range: describe layout segment ranges to be returned 2704 * @seq: stateid seqid to match 2705 * @cancel_io: signal io be cancelled 2706 * @devid: only cancel io directed at this device (all devices if NULL) 2707 * 2708 * This function is mainly intended for use by layoutrecall. It attempts 2709 * to free the layout segment immediately, or else to mark it for return 2710 * as soon as its reference count drops to zero. 2711 * 2712 * Returns 2713 * - 0: a layoutreturn needs to be scheduled. 2714 * - EBUSY: there are layout segment that are still in use. 2715 * - ENOENT: there are no layout segments that need to be returned. 2716 */ 2717 int 2718 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2719 struct list_head *tmp_list, 2720 const struct pnfs_layout_range *return_range, 2721 u32 seq, bool cancel_io, 2722 const struct nfs4_deviceid *devid) 2723 { 2724 struct pnfs_layout_segment *lseg, *next; 2725 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 2726 int remaining = 0; 2727 2728 dprintk("%s:Begin lo %p\n", __func__, lo); 2729 2730 assert_spin_locked(&lo->plh_inode->i_lock); 2731 2732 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2733 tmp_list = &lo->plh_return_segs; 2734 2735 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2736 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2737 dprintk("%s: marking lseg %p iomode %d " 2738 "offset %llu length %llu\n", __func__, 2739 lseg, lseg->pls_range.iomode, 2740 lseg->pls_range.offset, 2741 lseg->pls_range.length); 2742 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2743 tmp_list = &lo->plh_return_segs; 2744 if (mark_lseg_invalid(lseg, tmp_list)) 2745 continue; 2746 remaining++; 2747 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2748 if (cancel_io) 2749 pnfs_lseg_cancel_io(server, lseg, devid); 2750 } 2751 2752 if (remaining) { 2753 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2754 return -EBUSY; 2755 } 2756 2757 if (!list_empty(&lo->plh_return_segs)) { 2758 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2759 return 0; 2760 } 2761 2762 return -ENOENT; 2763 } 2764 2765 static void 2766 pnfs_mark_layout_for_return(struct inode *inode, 2767 const struct pnfs_layout_range *range, 2768 const struct nfs4_deviceid *devid) 2769 { 2770 struct pnfs_layout_hdr *lo; 2771 bool return_now = false; 2772 2773 spin_lock(&inode->i_lock); 2774 lo = NFS_I(inode)->layout; 2775 if (!pnfs_layout_is_valid(lo)) { 2776 spin_unlock(&inode->i_lock); 2777 return; 2778 } 2779 pnfs_set_plh_return_info(lo, range->iomode, 0); 2780 /* 2781 * mark all matching lsegs so that we are sure to have no live 2782 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2783 * for how it works. 2784 */ 2785 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0, 2786 true, devid) != -EBUSY) { 2787 const struct cred *cred; 2788 nfs4_stateid stateid; 2789 enum pnfs_iomode iomode; 2790 2791 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); 2792 spin_unlock(&inode->i_lock); 2793 if (return_now) 2794 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2795 PNFS_FL_LAYOUTRETURN_ASYNC); 2796 } else { 2797 spin_unlock(&inode->i_lock); 2798 nfs_commit_inode(inode, 0); 2799 } 2800 } 2801 2802 void pnfs_error_mark_layout_for_return(struct inode *inode, 2803 struct pnfs_layout_segment *lseg, 2804 const struct nfs4_deviceid *devid) 2805 { 2806 struct pnfs_layout_range range = { 2807 .iomode = lseg->pls_range.iomode, 2808 .offset = 0, 2809 .length = NFS4_MAX_UINT64, 2810 }; 2811 2812 pnfs_mark_layout_for_return(inode, &range, devid); 2813 } 2814 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2815 2816 static bool 2817 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) 2818 { 2819 return pnfs_layout_is_valid(lo) && 2820 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && 2821 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 2822 } 2823 2824 static struct pnfs_layout_segment * 2825 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, 2826 const struct pnfs_layout_range *range, 2827 enum pnfs_iomode iomode) 2828 { 2829 struct pnfs_layout_segment *lseg; 2830 2831 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 2832 if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 2833 continue; 2834 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 2835 continue; 2836 if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) 2837 continue; 2838 if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) 2839 return lseg; 2840 } 2841 return NULL; 2842 } 2843 2844 /* Find open file states whose mode matches that of the range */ 2845 static bool 2846 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, 2847 const struct pnfs_layout_range *range) 2848 { 2849 struct list_head *head; 2850 struct nfs_open_context *ctx; 2851 fmode_t mode = 0; 2852 2853 if (!pnfs_layout_can_be_returned(lo) || 2854 !pnfs_find_first_lseg(lo, range, range->iomode)) 2855 return false; 2856 2857 head = &NFS_I(lo->plh_inode)->open_files; 2858 list_for_each_entry_rcu(ctx, head, list) { 2859 if (ctx->state) 2860 mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); 2861 } 2862 2863 switch (range->iomode) { 2864 default: 2865 break; 2866 case IOMODE_READ: 2867 mode &= ~FMODE_WRITE; 2868 break; 2869 case IOMODE_RW: 2870 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) 2871 mode &= ~FMODE_READ; 2872 } 2873 return mode == 0; 2874 } 2875 2876 static int pnfs_layout_return_unused_byserver(struct nfs_server *server, 2877 void *data) 2878 { 2879 const struct pnfs_layout_range *range = data; 2880 const struct cred *cred; 2881 struct pnfs_layout_hdr *lo; 2882 struct inode *inode; 2883 nfs4_stateid stateid; 2884 enum pnfs_iomode iomode; 2885 2886 restart: 2887 rcu_read_lock(); 2888 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2889 inode = lo->plh_inode; 2890 if (!inode || !pnfs_layout_can_be_returned(lo) || 2891 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 2892 continue; 2893 spin_lock(&inode->i_lock); 2894 if (!lo->plh_inode || 2895 !pnfs_should_return_unused_layout(lo, range)) { 2896 spin_unlock(&inode->i_lock); 2897 continue; 2898 } 2899 pnfs_get_layout_hdr(lo); 2900 pnfs_set_plh_return_info(lo, range->iomode, 0); 2901 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, 2902 range, 0, true, NULL) != 0 || 2903 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { 2904 spin_unlock(&inode->i_lock); 2905 rcu_read_unlock(); 2906 pnfs_put_layout_hdr(lo); 2907 cond_resched(); 2908 goto restart; 2909 } 2910 spin_unlock(&inode->i_lock); 2911 rcu_read_unlock(); 2912 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, 2913 PNFS_FL_LAYOUTRETURN_ASYNC); 2914 pnfs_put_layout_hdr(lo); 2915 cond_resched(); 2916 goto restart; 2917 } 2918 rcu_read_unlock(); 2919 return 0; 2920 } 2921 2922 void 2923 pnfs_layout_return_unused_byclid(struct nfs_client *clp, 2924 enum pnfs_iomode iomode) 2925 { 2926 struct pnfs_layout_range range = { 2927 .iomode = iomode, 2928 .offset = 0, 2929 .length = NFS4_MAX_UINT64, 2930 }; 2931 2932 nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, 2933 &range); 2934 } 2935 2936 struct pnfs_reresolve_deviceid_args { 2937 const struct pnfs_layoutdriver_type *ld; 2938 const struct nfs4_deviceid *devid; 2939 bool immediate; 2940 struct list_head put_list; 2941 }; 2942 2943 static int pnfs_layout_reresolve_deviceid_byserver(struct nfs_server *server, 2944 void *data) 2945 { 2946 struct pnfs_reresolve_deviceid_args *args = data; 2947 struct pnfs_layout_hdr *lo; 2948 struct inode *inode; 2949 2950 if (server->pnfs_curr_ld != args->ld) 2951 return 0; 2952 2953 rcu_read_lock(); 2954 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 2955 inode = lo->plh_inode; 2956 if (!inode) 2957 continue; 2958 spin_lock(&inode->i_lock); 2959 args->ld->reresolve_deviceid(lo, args->devid, args->immediate, 2960 &args->put_list); 2961 spin_unlock(&inode->i_lock); 2962 } 2963 rcu_read_unlock(); 2964 return 0; 2965 } 2966 2967 /* 2968 * Invoke @ld's reresolve_deviceid hook for @devid on every layout of @clp's 2969 * servers, then drain the put_list once the locks are dropped. 2970 */ 2971 void 2972 pnfs_layout_reresolve_deviceid_byclid(struct nfs_client *clp, 2973 const struct pnfs_layoutdriver_type *ld, 2974 const struct nfs4_deviceid *devid, 2975 bool immediate) 2976 { 2977 struct pnfs_reresolve_deviceid_args args = { 2978 .ld = ld, 2979 .devid = devid, 2980 .immediate = immediate, 2981 .put_list = LIST_HEAD_INIT(args.put_list), 2982 }; 2983 struct nfs4_deviceid_put *put, *tmp; 2984 2985 if (!ld->reresolve_deviceid) 2986 return; 2987 2988 nfs_client_for_each_server(clp, 2989 pnfs_layout_reresolve_deviceid_byserver, &args); 2990 2991 list_for_each_entry_safe(put, tmp, &args.put_list, node) { 2992 list_del(&put->node); 2993 nfs4_put_deviceid_node(put->dev); 2994 kfree(put); 2995 } 2996 } 2997 2998 struct pnfs_deviceid_ref_args { 2999 const struct pnfs_layoutdriver_type *ld; 3000 const struct nfs4_deviceid *devid; 3001 struct list_head *result; 3002 bool found; 3003 }; 3004 3005 static int pnfs_layout_deviceid_referenced_byserver( 3006 struct nfs_server *server, void *data) 3007 { 3008 struct pnfs_deviceid_ref_args *args = data; 3009 struct pnfs_layout_hdr *lo; 3010 struct inode *inode; 3011 3012 if (server->pnfs_curr_ld != args->ld) 3013 return 0; 3014 3015 rcu_read_lock(); 3016 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 3017 inode = lo->plh_inode; 3018 if (!inode) 3019 continue; 3020 spin_lock(&inode->i_lock); 3021 if (NFS_I(inode)->layout == lo && pnfs_layout_is_valid(lo) && 3022 args->ld->layout_references_deviceid(lo, args->devid)) 3023 args->found = true; 3024 spin_unlock(&inode->i_lock); 3025 if (args->found) 3026 break; 3027 } 3028 rcu_read_unlock(); 3029 return args->found; 3030 } 3031 3032 /* 3033 * pnfs_layout_deviceid_referenced_byclid - does any live layout of 3034 * @clp's servers using @ld still reference deviceid @devid? 3035 */ 3036 bool 3037 pnfs_layout_deviceid_referenced_byclid(struct nfs_client *clp, 3038 const struct pnfs_layoutdriver_type *ld, 3039 const struct nfs4_deviceid *devid) 3040 { 3041 struct pnfs_deviceid_ref_args args = { 3042 .ld = ld, 3043 .devid = devid, 3044 }; 3045 3046 if (!ld->layout_references_deviceid) 3047 return false; 3048 3049 nfs_client_for_each_server(clp, 3050 pnfs_layout_deviceid_referenced_byserver, &args); 3051 return args.found; 3052 } 3053 3054 static int pnfs_layout_collect_deviceid_refs_byserver( 3055 struct nfs_server *server, void *data) 3056 { 3057 struct pnfs_deviceid_ref_args *args = data; 3058 struct nfs4_deviceid_ref *ref, *tmp; 3059 struct pnfs_layout_hdr *lo; 3060 struct inode *inode; 3061 LIST_HEAD(putme); 3062 bool matched; 3063 int ret = 0; 3064 3065 if (server->pnfs_curr_ld != args->ld) 3066 return 0; 3067 3068 rcu_read_lock(); 3069 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { 3070 inode = lo->plh_inode; 3071 if (!inode) 3072 continue; 3073 3074 spin_lock(&inode->i_lock); 3075 matched = NFS_I(inode)->layout == lo && 3076 pnfs_layout_is_valid(lo) && 3077 args->ld->layout_references_deviceid(lo, args->devid); 3078 if (!matched) { 3079 spin_unlock(&inode->i_lock); 3080 continue; 3081 } 3082 ref = kzalloc_obj(*ref, GFP_ATOMIC); 3083 if (!ref) { 3084 spin_unlock(&inode->i_lock); 3085 ret = -ENOMEM; 3086 break; 3087 } 3088 /* NFS_I()->layout == lo under i_lock means the refcount has 3089 * not reached zero: pnfs_put_layout_hdr() decrements to zero 3090 * and detaches in the same critical section. 3091 */ 3092 pnfs_get_layout_hdr(lo); 3093 ref->lo = lo; 3094 nfs4_stateid_copy(&ref->stateid, &lo->plh_stateid); 3095 ref->cred = get_cred(lo->plh_lc_cred); 3096 spin_unlock(&inode->i_lock); 3097 3098 /* the pinned hdr does not hold the inode: grab it (and 3099 * keep the superblock active) for use across RPCs 3100 */ 3101 ref->inode = nfs_igrab_and_active(inode); 3102 if (!ref->inode) { 3103 /* The layout may still name the deviceID, so report a 3104 * partial list rather than silently shortening it. 3105 * Defer the put: it can layoutreturn and sleep. 3106 */ 3107 list_add(&ref->node, &putme); 3108 ret = -EAGAIN; 3109 break; 3110 } 3111 list_add_tail(&ref->node, args->result); 3112 } 3113 rcu_read_unlock(); 3114 3115 list_for_each_entry_safe(ref, tmp, &putme, node) { 3116 list_del(&ref->node); 3117 pnfs_put_layout_hdr(ref->lo); 3118 put_cred(ref->cred); 3119 kfree(ref); 3120 } 3121 return ret; 3122 } 3123 3124 /* 3125 * Collect @clp's layouts referencing @devid onto @result as entries usable 3126 * across sleeping RPCs; release with pnfs_layout_put_deviceid_refs(). 3127 * A negative return means @result is only a partial set. 3128 */ 3129 int 3130 pnfs_layout_collect_deviceid_refs(struct nfs_client *clp, 3131 const struct pnfs_layoutdriver_type *ld, 3132 const struct nfs4_deviceid *devid, 3133 struct list_head *result) 3134 { 3135 struct pnfs_deviceid_ref_args args = { 3136 .ld = ld, 3137 .devid = devid, 3138 .result = result, 3139 }; 3140 3141 if (!ld->layout_references_deviceid) 3142 return 0; 3143 3144 return nfs_client_for_each_server(clp, 3145 pnfs_layout_collect_deviceid_refs_byserver, &args); 3146 } 3147 3148 void 3149 pnfs_layout_put_deviceid_refs(struct list_head *result) 3150 { 3151 struct nfs4_deviceid_ref *ref, *tmp; 3152 3153 list_for_each_entry_safe(ref, tmp, result, node) { 3154 list_del(&ref->node); 3155 put_cred(ref->cred); 3156 pnfs_put_layout_hdr(ref->lo); 3157 nfs_iput_and_deactive(ref->inode); 3158 kfree(ref); 3159 } 3160 } 3161 3162 /* 3163 * Queue @id for the state manager's Section 18.40.4 recovery, 3164 * dropping duplicates of an already-queued suspect. 3165 */ 3166 void pnfs_deviceid_delete_mark(struct nfs_client *clp, 3167 const struct pnfs_layoutdriver_type *ld, 3168 const struct nfs4_deviceid *id) 3169 { 3170 struct nfs4_deviceid_delete *dd, *new; 3171 3172 new = kzalloc_obj(*new, GFP_KERNEL); 3173 if (!new) 3174 return; /* lost notification; recovery waits for the next */ 3175 new->ld = pnfs_find_layoutdriver(ld->id); 3176 if (!new->ld) { 3177 kfree(new); 3178 return; 3179 } 3180 memcpy(&new->id, id, sizeof(new->id)); 3181 3182 spin_lock(&clp->cl_lock); 3183 list_for_each_entry(dd, &clp->cl_deviceid_deletes, list) { 3184 if (dd->ld == new->ld && 3185 !memcmp(&dd->id, &new->id, sizeof(dd->id))) { 3186 spin_unlock(&clp->cl_lock); 3187 pnfs_put_layoutdriver(new->ld); 3188 kfree(new); 3189 return; 3190 } 3191 } 3192 list_add_tail(&new->list, &clp->cl_deviceid_deletes); 3193 spin_unlock(&clp->cl_lock); 3194 3195 set_bit(NFS4CLNT_DEVICEID_DELETE, &clp->cl_state); 3196 nfs4_schedule_state_manager(clp); 3197 } 3198 3199 struct nfs4_deviceid_delete *pnfs_deviceid_delete_dequeue( 3200 struct nfs_client *clp) 3201 { 3202 struct nfs4_deviceid_delete *dd = NULL; 3203 3204 spin_lock(&clp->cl_lock); 3205 if (!list_empty(&clp->cl_deviceid_deletes)) { 3206 dd = list_first_entry(&clp->cl_deviceid_deletes, 3207 struct nfs4_deviceid_delete, list); 3208 list_del(&dd->list); 3209 } 3210 spin_unlock(&clp->cl_lock); 3211 return dd; 3212 } 3213 3214 void pnfs_deviceid_delete_queue_free(struct nfs_client *clp) 3215 { 3216 struct nfs4_deviceid_delete *dd; 3217 3218 while ((dd = pnfs_deviceid_delete_dequeue(clp)) != NULL) { 3219 pnfs_put_layoutdriver(dd->ld); 3220 kfree(dd); 3221 } 3222 } 3223 3224 /* Check if we have we have a valid layout but if there isn't an intersection 3225 * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. 3226 */ 3227 void 3228 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, 3229 struct nfs_page *req) 3230 { 3231 if (pgio->pg_lseg == NULL || 3232 (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && 3233 pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) 3234 return; 3235 pnfs_put_lseg(pgio->pg_lseg); 3236 pgio->pg_lseg = NULL; 3237 } 3238 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 3239 3240 void 3241 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 3242 { 3243 u64 rd_size; 3244 3245 pnfs_generic_pg_check_layout(pgio, req); 3246 if (pgio->pg_lseg == NULL) { 3247 if (pgio->pg_dreq == NULL) 3248 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 3249 else 3250 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, 3251 req_offset(req)); 3252 3253 pgio->pg_lseg = 3254 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 3255 req_offset(req), rd_size, 3256 IOMODE_READ, false, 3257 nfs_io_gfp_mask()); 3258 if (IS_ERR(pgio->pg_lseg)) { 3259 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 3260 pgio->pg_lseg = NULL; 3261 return; 3262 } 3263 } 3264 /* If no lseg, fall back to read through mds */ 3265 if (pgio->pg_lseg == NULL) 3266 nfs_pageio_reset_read_mds(pgio); 3267 3268 } 3269 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 3270 3271 void 3272 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 3273 struct nfs_page *req, u64 wb_size) 3274 { 3275 pnfs_generic_pg_check_layout(pgio, req); 3276 if (pgio->pg_lseg == NULL) { 3277 pgio->pg_lseg = 3278 pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), 3279 req_offset(req), wb_size, IOMODE_RW, 3280 false, nfs_io_gfp_mask()); 3281 if (IS_ERR(pgio->pg_lseg)) { 3282 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 3283 pgio->pg_lseg = NULL; 3284 return; 3285 } 3286 } 3287 /* If no lseg, fall back to write through mds */ 3288 if (pgio->pg_lseg == NULL) 3289 nfs_pageio_reset_write_mds(pgio); 3290 } 3291 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 3292 3293 void 3294 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 3295 { 3296 if (desc->pg_lseg) { 3297 pnfs_put_lseg(desc->pg_lseg); 3298 desc->pg_lseg = NULL; 3299 } 3300 } 3301 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 3302 3303 /* 3304 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 3305 * of bytes (maximum @req->wb_bytes) that can be coalesced. 3306 */ 3307 size_t 3308 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 3309 struct nfs_page *prev, struct nfs_page *req) 3310 { 3311 unsigned int size; 3312 u64 seg_end, req_start, seg_left; 3313 3314 size = nfs_generic_pg_test(pgio, prev, req); 3315 if (!size) 3316 return 0; 3317 3318 /* 3319 * 'size' contains the number of bytes left in the current page (up 3320 * to the original size asked for in @req->wb_bytes). 3321 * 3322 * Calculate how many bytes are left in the layout segment 3323 * and if there are less bytes than 'size', return that instead. 3324 * 3325 * Please also note that 'end_offset' is actually the offset of the 3326 * first byte that lies outside the pnfs_layout_range. FIXME? 3327 * 3328 */ 3329 if (pgio->pg_lseg) { 3330 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 3331 pgio->pg_lseg->pls_range.length); 3332 req_start = req_offset(req); 3333 3334 /* start of request is past the last byte of this segment */ 3335 if (req_start >= seg_end) 3336 return 0; 3337 3338 /* adjust 'size' iff there are fewer bytes left in the 3339 * segment than what nfs_generic_pg_test returned */ 3340 seg_left = seg_end - req_start; 3341 if (seg_left < size) 3342 size = (unsigned int)seg_left; 3343 } 3344 3345 return size; 3346 } 3347 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 3348 3349 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 3350 { 3351 struct nfs_pageio_descriptor pgio; 3352 3353 /* Resend all requests through the MDS */ 3354 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 3355 hdr->completion_ops); 3356 return nfs_pageio_resend(&pgio, hdr); 3357 } 3358 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 3359 3360 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 3361 { 3362 3363 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 3364 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3365 PNFS_LAYOUTRET_ON_ERROR) { 3366 pnfs_return_layout(hdr->inode); 3367 } 3368 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3369 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 3370 } 3371 3372 /* 3373 * Called by non rpc-based layout drivers 3374 */ 3375 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 3376 { 3377 if (likely(!hdr->pnfs_error)) { 3378 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 3379 hdr->mds_offset + hdr->res.count); 3380 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3381 } 3382 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 3383 if (unlikely(hdr->pnfs_error)) 3384 pnfs_ld_handle_write_error(hdr); 3385 hdr->mds_ops->rpc_release(hdr); 3386 } 3387 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 3388 3389 static void 3390 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 3391 struct nfs_pgio_header *hdr) 3392 { 3393 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3394 3395 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3396 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3397 nfs_pageio_reset_write_mds(desc); 3398 mirror->pg_recoalesce = 1; 3399 } 3400 hdr->completion_ops->completion(hdr); 3401 } 3402 3403 static enum pnfs_try_status 3404 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 3405 const struct rpc_call_ops *call_ops, 3406 struct pnfs_layout_segment *lseg, 3407 int how) 3408 { 3409 struct inode *inode = hdr->inode; 3410 enum pnfs_try_status trypnfs; 3411 struct nfs_server *nfss = NFS_SERVER(inode); 3412 3413 hdr->mds_ops = call_ops; 3414 3415 dprintk("%s: Writing ino:%llu %u@%llu (how %d)\n", __func__, 3416 inode->i_ino, hdr->args.count, hdr->args.offset, how); 3417 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 3418 if (trypnfs != PNFS_NOT_ATTEMPTED) 3419 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 3420 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3421 return trypnfs; 3422 } 3423 3424 static void 3425 pnfs_do_write(struct nfs_pageio_descriptor *desc, 3426 struct nfs_pgio_header *hdr, int how) 3427 { 3428 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3429 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3430 enum pnfs_try_status trypnfs; 3431 3432 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 3433 switch (trypnfs) { 3434 case PNFS_NOT_ATTEMPTED: 3435 pnfs_write_through_mds(desc, hdr); 3436 break; 3437 case PNFS_ATTEMPTED: 3438 break; 3439 case PNFS_TRY_AGAIN: 3440 /* cleanup hdr and prepare to redo pnfs */ 3441 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3442 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3443 list_splice_init(&hdr->pages, &mirror->pg_list); 3444 mirror->pg_recoalesce = 1; 3445 } 3446 hdr->mds_ops->rpc_release(hdr); 3447 } 3448 } 3449 3450 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 3451 { 3452 pnfs_put_ds_dev(hdr->ds_dev); 3453 pnfs_put_lseg(hdr->lseg); 3454 nfs_pgio_header_free(hdr); 3455 } 3456 3457 int 3458 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 3459 { 3460 struct nfs_pgio_header *hdr; 3461 int ret; 3462 3463 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3464 if (!hdr) { 3465 desc->pg_error = -ENOMEM; 3466 return desc->pg_error; 3467 } 3468 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 3469 3470 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3471 ret = nfs_generic_pgio(desc, hdr); 3472 if (!ret) 3473 pnfs_do_write(desc, hdr, desc->pg_ioflags); 3474 3475 return ret; 3476 } 3477 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 3478 3479 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 3480 { 3481 struct nfs_pageio_descriptor pgio; 3482 3483 /* Resend all requests through the MDS */ 3484 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 3485 return nfs_pageio_resend(&pgio, hdr); 3486 } 3487 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 3488 3489 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 3490 { 3491 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 3492 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 3493 PNFS_LAYOUTRET_ON_ERROR) { 3494 pnfs_return_layout(hdr->inode); 3495 } 3496 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 3497 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 3498 } 3499 3500 /* 3501 * Called by non rpc-based layout drivers 3502 */ 3503 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 3504 { 3505 if (likely(!hdr->pnfs_error)) 3506 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 3507 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 3508 if (unlikely(hdr->pnfs_error)) 3509 pnfs_ld_handle_read_error(hdr); 3510 hdr->mds_ops->rpc_release(hdr); 3511 } 3512 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 3513 3514 static void 3515 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 3516 struct nfs_pgio_header *hdr) 3517 { 3518 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3519 3520 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3521 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 3522 nfs_pageio_reset_read_mds(desc); 3523 mirror->pg_recoalesce = 1; 3524 } 3525 hdr->completion_ops->completion(hdr); 3526 } 3527 3528 /* 3529 * Call the appropriate parallel I/O subsystem read function. 3530 */ 3531 static enum pnfs_try_status 3532 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 3533 const struct rpc_call_ops *call_ops, 3534 struct pnfs_layout_segment *lseg) 3535 { 3536 struct inode *inode = hdr->inode; 3537 struct nfs_server *nfss = NFS_SERVER(inode); 3538 enum pnfs_try_status trypnfs; 3539 3540 hdr->mds_ops = call_ops; 3541 3542 dprintk("%s: Reading ino:%llu %u@%llu\n", 3543 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 3544 3545 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 3546 if (trypnfs != PNFS_NOT_ATTEMPTED) 3547 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 3548 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 3549 return trypnfs; 3550 } 3551 3552 /* Resend all requests through pnfs. */ 3553 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, 3554 unsigned int mirror_idx) 3555 { 3556 struct nfs_pageio_descriptor pgio; 3557 3558 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3559 /* Prevent deadlocks with layoutreturn! */ 3560 pnfs_put_lseg(hdr->lseg); 3561 hdr->lseg = NULL; 3562 3563 nfs_pageio_init_read(&pgio, hdr->inode, false, 3564 hdr->completion_ops); 3565 pgio.pg_mirror_idx = mirror_idx; 3566 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 3567 } 3568 } 3569 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 3570 3571 static void 3572 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 3573 { 3574 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 3575 struct pnfs_layout_segment *lseg = desc->pg_lseg; 3576 enum pnfs_try_status trypnfs; 3577 3578 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 3579 switch (trypnfs) { 3580 case PNFS_NOT_ATTEMPTED: 3581 pnfs_read_through_mds(desc, hdr); 3582 break; 3583 case PNFS_ATTEMPTED: 3584 break; 3585 case PNFS_TRY_AGAIN: 3586 /* cleanup hdr and prepare to redo pnfs */ 3587 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 3588 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 3589 list_splice_init(&hdr->pages, &mirror->pg_list); 3590 mirror->pg_recoalesce = 1; 3591 } 3592 hdr->mds_ops->rpc_release(hdr); 3593 } 3594 } 3595 3596 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 3597 { 3598 pnfs_put_ds_dev(hdr->ds_dev); 3599 pnfs_put_lseg(hdr->lseg); 3600 nfs_pgio_header_free(hdr); 3601 } 3602 3603 int 3604 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 3605 { 3606 struct nfs_pgio_header *hdr; 3607 int ret; 3608 3609 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 3610 if (!hdr) { 3611 desc->pg_error = -ENOMEM; 3612 return desc->pg_error; 3613 } 3614 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 3615 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 3616 ret = nfs_generic_pgio(desc, hdr); 3617 if (!ret) 3618 pnfs_do_read(desc, hdr); 3619 return ret; 3620 } 3621 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 3622 3623 /* 3624 * There can be multiple RW segments. 3625 */ 3626 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 3627 { 3628 struct pnfs_layout_segment *lseg; 3629 3630 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 3631 if (lseg->pls_range.iomode == IOMODE_RW && 3632 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 3633 list_add(&lseg->pls_lc_list, listp); 3634 } 3635 } 3636 3637 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 3638 { 3639 struct pnfs_layout_segment *lseg, *tmp; 3640 3641 /* Matched by references in pnfs_set_layoutcommit */ 3642 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 3643 list_del_init(&lseg->pls_lc_list); 3644 pnfs_put_lseg(lseg); 3645 } 3646 3647 clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); 3648 } 3649 3650 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 3651 { 3652 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 3653 } 3654 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 3655 3656 void 3657 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 3658 loff_t end_pos) 3659 { 3660 struct nfs_inode *nfsi = NFS_I(inode); 3661 bool mark_as_dirty = false; 3662 3663 spin_lock(&inode->i_lock); 3664 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 3665 nfsi->layout->plh_lwb = end_pos; 3666 mark_as_dirty = true; 3667 dprintk("%s: Set layoutcommit for inode %llu ", 3668 __func__, inode->i_ino); 3669 } else if (end_pos > nfsi->layout->plh_lwb) 3670 nfsi->layout->plh_lwb = end_pos; 3671 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 3672 /* references matched in nfs4_layoutcommit_release */ 3673 pnfs_get_lseg(lseg); 3674 } 3675 spin_unlock(&inode->i_lock); 3676 dprintk("%s: lseg %p end_pos %llu\n", 3677 __func__, lseg, nfsi->layout->plh_lwb); 3678 3679 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 3680 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 3681 if (mark_as_dirty) 3682 mark_inode_dirty_sync(inode); 3683 } 3684 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 3685 3686 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 3687 { 3688 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 3689 3690 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 3691 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 3692 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 3693 } 3694 3695 /* 3696 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 3697 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 3698 * data to disk to allow the server to recover the data if it crashes. 3699 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 3700 * is off, and a COMMIT is sent to a data server, or 3701 * if WRITEs to a data server return NFS_DATA_SYNC. 3702 */ 3703 int 3704 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 3705 { 3706 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3707 struct nfs4_layoutcommit_data *data; 3708 struct nfs_inode *nfsi = NFS_I(inode); 3709 loff_t end_pos; 3710 int status; 3711 bool mark_as_dirty = false; 3712 3713 if (!pnfs_layoutcommit_outstanding(inode)) 3714 return 0; 3715 3716 dprintk("--> %s inode %llu\n", __func__, inode->i_ino); 3717 3718 status = -EAGAIN; 3719 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 3720 if (!sync) 3721 goto out; 3722 status = wait_on_bit_lock_action(&nfsi->flags, 3723 NFS_INO_LAYOUTCOMMITTING, 3724 nfs_wait_bit_killable, 3725 TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); 3726 if (status) 3727 goto out; 3728 } 3729 3730 status = -ENOMEM; 3731 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 3732 data = kzalloc_obj(*data, nfs_io_gfp_mask()); 3733 if (!data) 3734 goto clear_layoutcommitting; 3735 3736 status = 0; 3737 spin_lock(&inode->i_lock); 3738 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 3739 goto out_unlock; 3740 3741 INIT_LIST_HEAD(&data->lseg_list); 3742 pnfs_list_write_lseg(inode, &data->lseg_list); 3743 3744 end_pos = nfsi->layout->plh_lwb; 3745 3746 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 3747 data->cred = get_cred(nfsi->layout->plh_lc_cred); 3748 spin_unlock(&inode->i_lock); 3749 3750 data->args.inode = inode; 3751 nfs_fattr_init(&data->fattr); 3752 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 3753 data->res.fattr = &data->fattr; 3754 if (end_pos != 0) 3755 data->args.lastbytewritten = end_pos - 1; 3756 else 3757 data->args.lastbytewritten = U64_MAX; 3758 data->res.server = NFS_SERVER(inode); 3759 3760 if (ld->prepare_layoutcommit) { 3761 status = ld->prepare_layoutcommit(&data->args); 3762 if (status) { 3763 if (status != -ENOSPC) 3764 put_cred(data->cred); 3765 spin_lock(&inode->i_lock); 3766 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 3767 if (end_pos > nfsi->layout->plh_lwb) 3768 nfsi->layout->plh_lwb = end_pos; 3769 if (status != -ENOSPC) 3770 goto out_unlock; 3771 spin_unlock(&inode->i_lock); 3772 mark_as_dirty = true; 3773 } 3774 } 3775 3776 3777 status = nfs4_proc_layoutcommit(data, sync); 3778 out: 3779 if (status || mark_as_dirty) 3780 mark_inode_dirty_sync(inode); 3781 dprintk("<-- %s status %d\n", __func__, status); 3782 return status; 3783 out_unlock: 3784 spin_unlock(&inode->i_lock); 3785 kfree(data); 3786 clear_layoutcommitting: 3787 clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); 3788 goto out; 3789 } 3790 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 3791 3792 int 3793 pnfs_generic_sync(struct inode *inode, bool datasync) 3794 { 3795 return pnfs_layoutcommit_inode(inode, true); 3796 } 3797 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 3798 3799 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 3800 { 3801 struct nfs4_threshold *thp; 3802 3803 thp = kzalloc_obj(*thp, nfs_io_gfp_mask()); 3804 if (!thp) { 3805 dprintk("%s mdsthreshold allocation failed\n", __func__); 3806 return NULL; 3807 } 3808 return thp; 3809 } 3810 3811 #if IS_ENABLED(CONFIG_NFS_V4_2) 3812 int 3813 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 3814 { 3815 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 3816 struct nfs_server *server = NFS_SERVER(inode); 3817 struct nfs_inode *nfsi = NFS_I(inode); 3818 struct nfs42_layoutstat_data *data; 3819 struct pnfs_layout_hdr *hdr; 3820 int status = 0; 3821 3822 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 3823 goto out; 3824 3825 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 3826 goto out; 3827 3828 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 3829 goto out; 3830 3831 spin_lock(&inode->i_lock); 3832 if (!NFS_I(inode)->layout) { 3833 spin_unlock(&inode->i_lock); 3834 goto out_clear_layoutstats; 3835 } 3836 hdr = NFS_I(inode)->layout; 3837 pnfs_get_layout_hdr(hdr); 3838 spin_unlock(&inode->i_lock); 3839 3840 data = kzalloc_obj(*data, gfp_flags); 3841 if (!data) { 3842 status = -ENOMEM; 3843 goto out_put; 3844 } 3845 3846 data->args.fh = NFS_FH(inode); 3847 data->args.inode = inode; 3848 status = ld->prepare_layoutstats(&data->args); 3849 if (status) 3850 goto out_free; 3851 3852 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 3853 3854 out: 3855 dprintk("%s returns %d\n", __func__, status); 3856 return status; 3857 3858 out_free: 3859 kfree(data); 3860 out_put: 3861 pnfs_put_layout_hdr(hdr); 3862 out_clear_layoutstats: 3863 smp_mb__before_atomic(); 3864 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 3865 smp_mb__after_atomic(); 3866 goto out; 3867 } 3868 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 3869 #endif 3870 3871 unsigned int layoutstats_timer; 3872 module_param(layoutstats_timer, uint, 0644); 3873 EXPORT_SYMBOL_GPL(layoutstats_timer); 3874