1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * NFS client support for local clients to bypass network stack 4 * 5 * Copyright (C) 2014 Weston Andros Adamson <dros@primarydata.com> 6 * Copyright (C) 2019 Trond Myklebust <trond.myklebust@hammerspace.com> 7 * Copyright (C) 2024 Mike Snitzer <snitzer@hammerspace.com> 8 * Copyright (C) 2024 NeilBrown <neilb@suse.de> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/errno.h> 13 #include <linux/vfs.h> 14 #include <linux/file.h> 15 #include <linux/inet.h> 16 #include <linux/sunrpc/addr.h> 17 #include <linux/inetdevice.h> 18 #include <net/addrconf.h> 19 #include <linux/nfs_common.h> 20 #include <linux/nfslocalio.h> 21 #include <linux/bvec.h> 22 23 #include <linux/nfs.h> 24 #include <linux/nfs_fs.h> 25 #include <linux/nfs_xdr.h> 26 27 #include "internal.h" 28 #include "pnfs.h" 29 #include "nfstrace.h" 30 31 #define NFSDBG_FACILITY NFSDBG_VFS 32 33 struct nfs_local_kiocb { 34 struct kiocb kiocb; 35 struct bio_vec *bvec; 36 struct nfs_pgio_header *hdr; 37 struct work_struct work; 38 struct nfsd_file *localio; 39 }; 40 41 struct nfs_local_fsync_ctx { 42 struct nfsd_file *localio; 43 struct nfs_commit_data *data; 44 struct work_struct work; 45 struct kref kref; 46 struct completion *done; 47 }; 48 static void nfs_local_fsync_work(struct work_struct *work); 49 50 static bool localio_enabled __read_mostly = true; 51 module_param(localio_enabled, bool, 0644); 52 53 static inline bool nfs_client_is_local(const struct nfs_client *clp) 54 { 55 return !!test_bit(NFS_CS_LOCAL_IO, &clp->cl_flags); 56 } 57 58 bool nfs_server_is_local(const struct nfs_client *clp) 59 { 60 return nfs_client_is_local(clp) && localio_enabled; 61 } 62 EXPORT_SYMBOL_GPL(nfs_server_is_local); 63 64 /* 65 * UUID_IS_LOCAL XDR functions 66 */ 67 68 static void localio_xdr_enc_uuidargs(struct rpc_rqst *req, 69 struct xdr_stream *xdr, 70 const void *data) 71 { 72 const u8 *uuid = data; 73 74 encode_opaque_fixed(xdr, uuid, UUID_SIZE); 75 } 76 77 static int localio_xdr_dec_uuidres(struct rpc_rqst *req, 78 struct xdr_stream *xdr, 79 void *result) 80 { 81 /* void return */ 82 return 0; 83 } 84 85 static const struct rpc_procinfo nfs_localio_procedures[] = { 86 [LOCALIOPROC_UUID_IS_LOCAL] = { 87 .p_proc = LOCALIOPROC_UUID_IS_LOCAL, 88 .p_encode = localio_xdr_enc_uuidargs, 89 .p_decode = localio_xdr_dec_uuidres, 90 .p_arglen = XDR_QUADLEN(UUID_SIZE), 91 .p_replen = 0, 92 .p_statidx = LOCALIOPROC_UUID_IS_LOCAL, 93 .p_name = "UUID_IS_LOCAL", 94 }, 95 }; 96 97 static unsigned int nfs_localio_counts[ARRAY_SIZE(nfs_localio_procedures)]; 98 static const struct rpc_version nfslocalio_version1 = { 99 .number = 1, 100 .nrprocs = ARRAY_SIZE(nfs_localio_procedures), 101 .procs = nfs_localio_procedures, 102 .counts = nfs_localio_counts, 103 }; 104 105 static const struct rpc_version *nfslocalio_version[] = { 106 [1] = &nfslocalio_version1, 107 }; 108 109 extern const struct rpc_program nfslocalio_program; 110 static struct rpc_stat nfslocalio_rpcstat = { &nfslocalio_program }; 111 112 const struct rpc_program nfslocalio_program = { 113 .name = "nfslocalio", 114 .number = NFS_LOCALIO_PROGRAM, 115 .nrvers = ARRAY_SIZE(nfslocalio_version), 116 .version = nfslocalio_version, 117 .stats = &nfslocalio_rpcstat, 118 }; 119 120 /* 121 * nfs_local_enable - enable local i/o for an nfs_client 122 */ 123 static void nfs_local_enable(struct nfs_client *clp) 124 { 125 spin_lock(&clp->cl_localio_lock); 126 set_bit(NFS_CS_LOCAL_IO, &clp->cl_flags); 127 trace_nfs_local_enable(clp); 128 spin_unlock(&clp->cl_localio_lock); 129 } 130 131 /* 132 * nfs_local_disable - disable local i/o for an nfs_client 133 */ 134 void nfs_local_disable(struct nfs_client *clp) 135 { 136 spin_lock(&clp->cl_localio_lock); 137 if (test_and_clear_bit(NFS_CS_LOCAL_IO, &clp->cl_flags)) { 138 trace_nfs_local_disable(clp); 139 nfs_uuid_invalidate_one_client(&clp->cl_uuid); 140 } 141 spin_unlock(&clp->cl_localio_lock); 142 } 143 144 /* 145 * nfs_init_localioclient - Initialise an NFS localio client connection 146 */ 147 static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp) 148 { 149 struct rpc_clnt *rpcclient_localio; 150 151 rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient, 152 &nfslocalio_program, 1); 153 154 dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n", 155 __func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR), 156 (IS_ERR(rpcclient_localio) ? "does not support" : "supports")); 157 158 return rpcclient_localio; 159 } 160 161 static bool nfs_server_uuid_is_local(struct nfs_client *clp) 162 { 163 u8 uuid[UUID_SIZE]; 164 struct rpc_message msg = { 165 .rpc_argp = &uuid, 166 }; 167 struct rpc_clnt *rpcclient_localio; 168 int status; 169 170 rpcclient_localio = nfs_init_localioclient(clp); 171 if (IS_ERR(rpcclient_localio)) 172 return false; 173 174 export_uuid(uuid, &clp->cl_uuid.uuid); 175 176 msg.rpc_proc = &nfs_localio_procedures[LOCALIOPROC_UUID_IS_LOCAL]; 177 status = rpc_call_sync(rpcclient_localio, &msg, 0); 178 dprintk("%s: NFS reply UUID_IS_LOCAL: status=%d\n", 179 __func__, status); 180 rpc_shutdown_client(rpcclient_localio); 181 182 /* Server is only local if it initialized required struct members */ 183 if (status || !clp->cl_uuid.net || !clp->cl_uuid.dom) 184 return false; 185 186 return true; 187 } 188 189 /* 190 * nfs_local_probe - probe local i/o support for an nfs_server and nfs_client 191 * - called after alloc_client and init_client (so cl_rpcclient exists) 192 * - this function is idempotent, it can be called for old or new clients 193 */ 194 void nfs_local_probe(struct nfs_client *clp) 195 { 196 /* Disallow localio if disabled via sysfs or AUTH_SYS isn't used */ 197 if (!localio_enabled || 198 clp->cl_rpcclient->cl_auth->au_flavor != RPC_AUTH_UNIX) { 199 nfs_local_disable(clp); 200 return; 201 } 202 203 if (nfs_client_is_local(clp)) { 204 /* If already enabled, disable and re-enable */ 205 nfs_local_disable(clp); 206 } 207 208 if (!nfs_uuid_begin(&clp->cl_uuid)) 209 return; 210 if (nfs_server_uuid_is_local(clp)) 211 nfs_local_enable(clp); 212 nfs_uuid_end(&clp->cl_uuid); 213 } 214 EXPORT_SYMBOL_GPL(nfs_local_probe); 215 216 /* 217 * nfs_local_open_fh - open a local filehandle in terms of nfsd_file 218 * 219 * Returns a pointer to a struct nfsd_file or NULL 220 */ 221 struct nfsd_file * 222 nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, 223 struct nfs_fh *fh, const fmode_t mode) 224 { 225 struct nfsd_file *localio; 226 int status; 227 228 if (!nfs_server_is_local(clp)) 229 return NULL; 230 if (mode & ~(FMODE_READ | FMODE_WRITE)) 231 return NULL; 232 233 localio = nfs_open_local_fh(&clp->cl_uuid, clp->cl_rpcclient, 234 cred, fh, mode); 235 if (IS_ERR(localio)) { 236 status = PTR_ERR(localio); 237 trace_nfs_local_open_fh(fh, mode, status); 238 switch (status) { 239 case -ENOMEM: 240 case -ENXIO: 241 case -ENOENT: 242 /* Revalidate localio, will disable if unsupported */ 243 nfs_local_probe(clp); 244 } 245 return NULL; 246 } 247 return localio; 248 } 249 EXPORT_SYMBOL_GPL(nfs_local_open_fh); 250 251 static struct bio_vec * 252 nfs_bvec_alloc_and_import_pagevec(struct page **pagevec, 253 unsigned int npages, gfp_t flags) 254 { 255 struct bio_vec *bvec, *p; 256 257 bvec = kmalloc_array(npages, sizeof(*bvec), flags); 258 if (bvec != NULL) { 259 for (p = bvec; npages > 0; p++, pagevec++, npages--) { 260 p->bv_page = *pagevec; 261 p->bv_len = PAGE_SIZE; 262 p->bv_offset = 0; 263 } 264 } 265 return bvec; 266 } 267 268 static void 269 nfs_local_iocb_free(struct nfs_local_kiocb *iocb) 270 { 271 kfree(iocb->bvec); 272 kfree(iocb); 273 } 274 275 static struct nfs_local_kiocb * 276 nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, 277 struct nfsd_file *localio, gfp_t flags) 278 { 279 struct nfs_local_kiocb *iocb; 280 281 iocb = kmalloc(sizeof(*iocb), flags); 282 if (iocb == NULL) 283 return NULL; 284 iocb->bvec = nfs_bvec_alloc_and_import_pagevec(hdr->page_array.pagevec, 285 hdr->page_array.npages, flags); 286 if (iocb->bvec == NULL) { 287 kfree(iocb); 288 return NULL; 289 } 290 init_sync_kiocb(&iocb->kiocb, nfs_to->nfsd_file_file(localio)); 291 iocb->kiocb.ki_pos = hdr->args.offset; 292 iocb->localio = localio; 293 iocb->hdr = hdr; 294 iocb->kiocb.ki_flags &= ~IOCB_APPEND; 295 return iocb; 296 } 297 298 static void 299 nfs_local_iter_init(struct iov_iter *i, struct nfs_local_kiocb *iocb, int dir) 300 { 301 struct nfs_pgio_header *hdr = iocb->hdr; 302 303 iov_iter_bvec(i, dir, iocb->bvec, hdr->page_array.npages, 304 hdr->args.count + hdr->args.pgbase); 305 if (hdr->args.pgbase != 0) 306 iov_iter_advance(i, hdr->args.pgbase); 307 } 308 309 static void 310 nfs_local_hdr_release(struct nfs_pgio_header *hdr, 311 const struct rpc_call_ops *call_ops) 312 { 313 call_ops->rpc_call_done(&hdr->task, hdr); 314 call_ops->rpc_release(hdr); 315 } 316 317 static void 318 nfs_local_pgio_init(struct nfs_pgio_header *hdr, 319 const struct rpc_call_ops *call_ops) 320 { 321 hdr->task.tk_ops = call_ops; 322 if (!hdr->task.tk_start) 323 hdr->task.tk_start = ktime_get(); 324 } 325 326 static void 327 nfs_local_pgio_done(struct nfs_pgio_header *hdr, long status) 328 { 329 if (status >= 0) { 330 hdr->res.count = status; 331 hdr->res.op_status = NFS4_OK; 332 hdr->task.tk_status = 0; 333 } else { 334 hdr->res.op_status = nfs4_stat_to_errno(status); 335 hdr->task.tk_status = status; 336 } 337 } 338 339 static void 340 nfs_local_pgio_release(struct nfs_local_kiocb *iocb) 341 { 342 struct nfs_pgio_header *hdr = iocb->hdr; 343 344 nfs_to_nfsd_file_put_local(iocb->localio); 345 nfs_local_iocb_free(iocb); 346 nfs_local_hdr_release(hdr, hdr->task.tk_ops); 347 } 348 349 static void 350 nfs_local_read_done(struct nfs_local_kiocb *iocb, long status) 351 { 352 struct nfs_pgio_header *hdr = iocb->hdr; 353 struct file *filp = iocb->kiocb.ki_filp; 354 355 nfs_local_pgio_done(hdr, status); 356 357 if (hdr->res.count != hdr->args.count || 358 hdr->args.offset + hdr->res.count >= i_size_read(file_inode(filp))) 359 hdr->res.eof = true; 360 361 dprintk("%s: read %ld bytes eof %d.\n", __func__, 362 status > 0 ? status : 0, hdr->res.eof); 363 } 364 365 static void nfs_local_call_read(struct work_struct *work) 366 { 367 struct nfs_local_kiocb *iocb = 368 container_of(work, struct nfs_local_kiocb, work); 369 struct file *filp = iocb->kiocb.ki_filp; 370 const struct cred *save_cred; 371 struct iov_iter iter; 372 ssize_t status; 373 374 save_cred = override_creds(filp->f_cred); 375 376 nfs_local_iter_init(&iter, iocb, READ); 377 378 status = filp->f_op->read_iter(&iocb->kiocb, &iter); 379 WARN_ON_ONCE(status == -EIOCBQUEUED); 380 381 nfs_local_read_done(iocb, status); 382 nfs_local_pgio_release(iocb); 383 384 revert_creds(save_cred); 385 } 386 387 static int 388 nfs_do_local_read(struct nfs_pgio_header *hdr, 389 struct nfsd_file *localio, 390 const struct rpc_call_ops *call_ops) 391 { 392 struct nfs_local_kiocb *iocb; 393 394 dprintk("%s: vfs_read count=%u pos=%llu\n", 395 __func__, hdr->args.count, hdr->args.offset); 396 397 iocb = nfs_local_iocb_alloc(hdr, localio, GFP_KERNEL); 398 if (iocb == NULL) 399 return -ENOMEM; 400 401 nfs_local_pgio_init(hdr, call_ops); 402 hdr->res.eof = false; 403 404 INIT_WORK(&iocb->work, nfs_local_call_read); 405 queue_work(nfslocaliod_workqueue, &iocb->work); 406 407 return 0; 408 } 409 410 static void 411 nfs_copy_boot_verifier(struct nfs_write_verifier *verifier, struct inode *inode) 412 { 413 struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; 414 u32 *verf = (u32 *)verifier->data; 415 int seq = 0; 416 417 do { 418 read_seqbegin_or_lock(&clp->cl_boot_lock, &seq); 419 verf[0] = (u32)clp->cl_nfssvc_boot.tv_sec; 420 verf[1] = (u32)clp->cl_nfssvc_boot.tv_nsec; 421 } while (need_seqretry(&clp->cl_boot_lock, seq)); 422 done_seqretry(&clp->cl_boot_lock, seq); 423 } 424 425 static void 426 nfs_reset_boot_verifier(struct inode *inode) 427 { 428 struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; 429 430 write_seqlock(&clp->cl_boot_lock); 431 ktime_get_real_ts64(&clp->cl_nfssvc_boot); 432 write_sequnlock(&clp->cl_boot_lock); 433 } 434 435 static void 436 nfs_set_local_verifier(struct inode *inode, 437 struct nfs_writeverf *verf, 438 enum nfs3_stable_how how) 439 { 440 nfs_copy_boot_verifier(&verf->verifier, inode); 441 verf->committed = how; 442 } 443 444 /* Factored out from fs/nfsd/vfs.h:fh_getattr() */ 445 static int __vfs_getattr(struct path *p, struct kstat *stat, int version) 446 { 447 u32 request_mask = STATX_BASIC_STATS; 448 449 if (version == 4) 450 request_mask |= (STATX_BTIME | STATX_CHANGE_COOKIE); 451 return vfs_getattr(p, stat, request_mask, AT_STATX_SYNC_AS_STAT); 452 } 453 454 /* Copied from fs/nfsd/nfsfh.c:nfsd4_change_attribute() */ 455 static u64 __nfsd4_change_attribute(const struct kstat *stat, 456 const struct inode *inode) 457 { 458 u64 chattr; 459 460 if (stat->result_mask & STATX_CHANGE_COOKIE) { 461 chattr = stat->change_cookie; 462 if (S_ISREG(inode->i_mode) && 463 !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) { 464 chattr += (u64)stat->ctime.tv_sec << 30; 465 chattr += stat->ctime.tv_nsec; 466 } 467 } else { 468 chattr = time_to_chattr(&stat->ctime); 469 } 470 return chattr; 471 } 472 473 static void nfs_local_vfs_getattr(struct nfs_local_kiocb *iocb) 474 { 475 struct kstat stat; 476 struct file *filp = iocb->kiocb.ki_filp; 477 struct nfs_pgio_header *hdr = iocb->hdr; 478 struct nfs_fattr *fattr = hdr->res.fattr; 479 int version = NFS_PROTO(hdr->inode)->version; 480 481 if (unlikely(!fattr) || __vfs_getattr(&filp->f_path, &stat, version)) 482 return; 483 484 fattr->valid = (NFS_ATTR_FATTR_FILEID | 485 NFS_ATTR_FATTR_CHANGE | 486 NFS_ATTR_FATTR_SIZE | 487 NFS_ATTR_FATTR_ATIME | 488 NFS_ATTR_FATTR_MTIME | 489 NFS_ATTR_FATTR_CTIME | 490 NFS_ATTR_FATTR_SPACE_USED); 491 492 fattr->fileid = stat.ino; 493 fattr->size = stat.size; 494 fattr->atime = stat.atime; 495 fattr->mtime = stat.mtime; 496 fattr->ctime = stat.ctime; 497 if (version == 4) { 498 fattr->change_attr = 499 __nfsd4_change_attribute(&stat, file_inode(filp)); 500 } else 501 fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime); 502 fattr->du.nfs3.used = stat.blocks << 9; 503 } 504 505 static void 506 nfs_local_write_done(struct nfs_local_kiocb *iocb, long status) 507 { 508 struct nfs_pgio_header *hdr = iocb->hdr; 509 struct inode *inode = hdr->inode; 510 511 dprintk("%s: wrote %ld bytes.\n", __func__, status > 0 ? status : 0); 512 513 /* Handle short writes as if they are ENOSPC */ 514 if (status > 0 && status < hdr->args.count) { 515 hdr->mds_offset += status; 516 hdr->args.offset += status; 517 hdr->args.pgbase += status; 518 hdr->args.count -= status; 519 nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); 520 status = -ENOSPC; 521 } 522 if (status < 0) 523 nfs_reset_boot_verifier(inode); 524 else if (nfs_should_remove_suid(inode)) { 525 /* Deal with the suid/sgid bit corner case */ 526 spin_lock(&inode->i_lock); 527 nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); 528 spin_unlock(&inode->i_lock); 529 } 530 nfs_local_pgio_done(hdr, status); 531 } 532 533 static void nfs_local_call_write(struct work_struct *work) 534 { 535 struct nfs_local_kiocb *iocb = 536 container_of(work, struct nfs_local_kiocb, work); 537 struct file *filp = iocb->kiocb.ki_filp; 538 unsigned long old_flags = current->flags; 539 const struct cred *save_cred; 540 struct iov_iter iter; 541 ssize_t status; 542 543 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; 544 save_cred = override_creds(filp->f_cred); 545 546 nfs_local_iter_init(&iter, iocb, WRITE); 547 548 file_start_write(filp); 549 status = filp->f_op->write_iter(&iocb->kiocb, &iter); 550 file_end_write(filp); 551 WARN_ON_ONCE(status == -EIOCBQUEUED); 552 553 nfs_local_write_done(iocb, status); 554 nfs_local_vfs_getattr(iocb); 555 nfs_local_pgio_release(iocb); 556 557 revert_creds(save_cred); 558 current->flags = old_flags; 559 } 560 561 static int 562 nfs_do_local_write(struct nfs_pgio_header *hdr, 563 struct nfsd_file *localio, 564 const struct rpc_call_ops *call_ops) 565 { 566 struct nfs_local_kiocb *iocb; 567 568 dprintk("%s: vfs_write count=%u pos=%llu %s\n", 569 __func__, hdr->args.count, hdr->args.offset, 570 (hdr->args.stable == NFS_UNSTABLE) ? "unstable" : "stable"); 571 572 iocb = nfs_local_iocb_alloc(hdr, localio, GFP_NOIO); 573 if (iocb == NULL) 574 return -ENOMEM; 575 576 switch (hdr->args.stable) { 577 default: 578 break; 579 case NFS_DATA_SYNC: 580 iocb->kiocb.ki_flags |= IOCB_DSYNC; 581 break; 582 case NFS_FILE_SYNC: 583 iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC; 584 } 585 nfs_local_pgio_init(hdr, call_ops); 586 587 nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable); 588 589 INIT_WORK(&iocb->work, nfs_local_call_write); 590 queue_work(nfslocaliod_workqueue, &iocb->work); 591 592 return 0; 593 } 594 595 int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, 596 struct nfs_pgio_header *hdr, 597 const struct rpc_call_ops *call_ops) 598 { 599 int status = 0; 600 struct file *filp = nfs_to->nfsd_file_file(localio); 601 602 if (!hdr->args.count) 603 return 0; 604 /* Don't support filesystems without read_iter/write_iter */ 605 if (!filp->f_op->read_iter || !filp->f_op->write_iter) { 606 nfs_local_disable(clp); 607 status = -EAGAIN; 608 goto out; 609 } 610 611 switch (hdr->rw_mode) { 612 case FMODE_READ: 613 status = nfs_do_local_read(hdr, localio, call_ops); 614 break; 615 case FMODE_WRITE: 616 status = nfs_do_local_write(hdr, localio, call_ops); 617 break; 618 default: 619 dprintk("%s: invalid mode: %d\n", __func__, 620 hdr->rw_mode); 621 status = -EINVAL; 622 } 623 out: 624 if (status != 0) { 625 nfs_to_nfsd_file_put_local(localio); 626 hdr->task.tk_status = status; 627 nfs_local_hdr_release(hdr, call_ops); 628 } 629 return status; 630 } 631 632 static void 633 nfs_local_init_commit(struct nfs_commit_data *data, 634 const struct rpc_call_ops *call_ops) 635 { 636 data->task.tk_ops = call_ops; 637 } 638 639 static int 640 nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) 641 { 642 loff_t start = data->args.offset; 643 loff_t end = LLONG_MAX; 644 645 if (data->args.count > 0) { 646 end = start + data->args.count - 1; 647 if (end < start) 648 end = LLONG_MAX; 649 } 650 651 dprintk("%s: commit %llu - %llu\n", __func__, start, end); 652 return vfs_fsync_range(filp, start, end, 0); 653 } 654 655 static void 656 nfs_local_commit_done(struct nfs_commit_data *data, int status) 657 { 658 if (status >= 0) { 659 nfs_set_local_verifier(data->inode, 660 data->res.verf, 661 NFS_FILE_SYNC); 662 data->res.op_status = NFS4_OK; 663 data->task.tk_status = 0; 664 } else { 665 nfs_reset_boot_verifier(data->inode); 666 data->res.op_status = nfs4_stat_to_errno(status); 667 data->task.tk_status = status; 668 } 669 } 670 671 static void 672 nfs_local_release_commit_data(struct nfsd_file *localio, 673 struct nfs_commit_data *data, 674 const struct rpc_call_ops *call_ops) 675 { 676 nfs_to_nfsd_file_put_local(localio); 677 call_ops->rpc_call_done(&data->task, data); 678 call_ops->rpc_release(data); 679 } 680 681 static struct nfs_local_fsync_ctx * 682 nfs_local_fsync_ctx_alloc(struct nfs_commit_data *data, 683 struct nfsd_file *localio, gfp_t flags) 684 { 685 struct nfs_local_fsync_ctx *ctx = kmalloc(sizeof(*ctx), flags); 686 687 if (ctx != NULL) { 688 ctx->localio = localio; 689 ctx->data = data; 690 INIT_WORK(&ctx->work, nfs_local_fsync_work); 691 kref_init(&ctx->kref); 692 ctx->done = NULL; 693 } 694 return ctx; 695 } 696 697 static void 698 nfs_local_fsync_ctx_kref_free(struct kref *kref) 699 { 700 kfree(container_of(kref, struct nfs_local_fsync_ctx, kref)); 701 } 702 703 static void 704 nfs_local_fsync_ctx_put(struct nfs_local_fsync_ctx *ctx) 705 { 706 kref_put(&ctx->kref, nfs_local_fsync_ctx_kref_free); 707 } 708 709 static void 710 nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) 711 { 712 nfs_local_release_commit_data(ctx->localio, ctx->data, 713 ctx->data->task.tk_ops); 714 nfs_local_fsync_ctx_put(ctx); 715 } 716 717 static void 718 nfs_local_fsync_work(struct work_struct *work) 719 { 720 struct nfs_local_fsync_ctx *ctx; 721 int status; 722 723 ctx = container_of(work, struct nfs_local_fsync_ctx, work); 724 725 status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), 726 ctx->data); 727 nfs_local_commit_done(ctx->data, status); 728 if (ctx->done != NULL) 729 complete(ctx->done); 730 nfs_local_fsync_ctx_free(ctx); 731 } 732 733 int nfs_local_commit(struct nfsd_file *localio, 734 struct nfs_commit_data *data, 735 const struct rpc_call_ops *call_ops, int how) 736 { 737 struct nfs_local_fsync_ctx *ctx; 738 739 ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); 740 if (!ctx) { 741 nfs_local_commit_done(data, -ENOMEM); 742 nfs_local_release_commit_data(localio, data, call_ops); 743 return -ENOMEM; 744 } 745 746 nfs_local_init_commit(data, call_ops); 747 kref_get(&ctx->kref); 748 if (how & FLUSH_SYNC) { 749 DECLARE_COMPLETION_ONSTACK(done); 750 ctx->done = &done; 751 queue_work(nfsiod_workqueue, &ctx->work); 752 wait_for_completion(&done); 753 } else 754 queue_work(nfsiod_workqueue, &ctx->work); 755 nfs_local_fsync_ctx_put(ctx); 756 return 0; 757 } 758