1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem high-level (buffered) writeback. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * 7 * 8 * To support network filesystems with local caching, we manage a situation 9 * that can be envisioned like the following: 10 * 11 * +---+---+-----+-----+---+----------+ 12 * Folios: | | | | | | | 13 * +---+---+-----+-----+---+----------+ 14 * 15 * +------+------+ +----+----+ 16 * Upload: | | |.....| | | 17 * (Stream 0) +------+------+ +----+----+ 18 * 19 * +------+------+------+------+------+ 20 * Cache: | | | | | | 21 * (Stream 1) +------+------+------+------+------+ 22 * 23 * Where we have a sequence of folios of varying sizes that we need to overlay 24 * with multiple parallel streams of I/O requests, where the I/O requests in a 25 * stream may also be of various sizes (in cifs, for example, the sizes are 26 * negotiated with the server; in something like ceph, they may represent the 27 * sizes of storage objects). 28 * 29 * The sequence in each stream may contain gaps and noncontiguous subrequests 30 * may be glued together into single vectored write RPCs. 31 */ 32 33 #include <linux/export.h> 34 #include <linux/fs.h> 35 #include <linux/mm.h> 36 #include <linux/pagemap.h> 37 #include "internal.h" 38 39 /* 40 * Kill all dirty folios in the event of an unrecoverable error, starting with 41 * a locked folio we've already obtained from writeback_iter(). 42 */ 43 static void netfs_kill_dirty_pages(struct address_space *mapping, 44 struct writeback_control *wbc, 45 struct folio *folio) 46 { 47 int error = 0; 48 49 do { 50 enum netfs_folio_trace why = netfs_folio_trace_kill; 51 struct netfs_group *group = NULL; 52 struct netfs_folio *finfo = NULL; 53 void *priv; 54 55 priv = folio_detach_private(folio); 56 if (priv) { 57 finfo = __netfs_folio_info(priv); 58 if (finfo) { 59 /* Kill folio from streaming write. */ 60 group = finfo->netfs_group; 61 why = netfs_folio_trace_kill_s; 62 } else { 63 group = priv; 64 if (group == NETFS_FOLIO_COPY_TO_CACHE) { 65 /* Kill copy-to-cache folio */ 66 why = netfs_folio_trace_kill_cc; 67 group = NULL; 68 } else { 69 /* Kill folio with group */ 70 why = netfs_folio_trace_kill_g; 71 } 72 } 73 } 74 75 trace_netfs_folio(folio, why); 76 77 folio_start_writeback(folio); 78 folio_unlock(folio); 79 folio_end_writeback(folio); 80 81 netfs_put_group(group); 82 kfree(finfo); 83 84 } while ((folio = writeback_iter(mapping, wbc, folio, &error))); 85 } 86 87 /* 88 * Create a write request and set it up appropriately for the origin type. 89 */ 90 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, 91 struct file *file, 92 loff_t start, 93 enum netfs_io_origin origin) 94 { 95 struct netfs_io_request *wreq; 96 struct netfs_inode *ictx; 97 bool is_cacheable = (origin == NETFS_WRITEBACK || 98 origin == NETFS_WRITEBACK_SINGLE || 99 origin == NETFS_WRITETHROUGH || 100 origin == NETFS_PGPRIV2_COPY_TO_CACHE); 101 102 wreq = netfs_alloc_request(mapping, file, start, 0, origin); 103 if (IS_ERR(wreq)) 104 return wreq; 105 106 _enter("R=%x", wreq->debug_id); 107 108 ictx = netfs_inode(wreq->inode); 109 if (is_cacheable) 110 fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx)); 111 if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE) < 0) 112 goto nomem; 113 114 wreq->cleaned_to = wreq->start; 115 116 wreq->io_streams[0].stream_nr = 0; 117 wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER; 118 wreq->io_streams[0].prepare_write = ictx->ops->prepare_write; 119 wreq->io_streams[0].issue_write = ictx->ops->issue_write; 120 wreq->io_streams[0].collected_to = start; 121 wreq->io_streams[0].transferred = 0; 122 123 wreq->io_streams[1].stream_nr = 1; 124 wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE; 125 wreq->io_streams[1].collected_to = start; 126 wreq->io_streams[1].transferred = 0; 127 if (fscache_resources_valid(&wreq->cache_resources)) { 128 wreq->io_streams[1].avail = true; 129 wreq->io_streams[1].active = true; 130 wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq; 131 wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write; 132 } 133 134 return wreq; 135 nomem: 136 netfs_put_failed_request(wreq); 137 return ERR_PTR(-ENOMEM); 138 } 139 140 /** 141 * netfs_prepare_write_failed - Note write preparation failed 142 * @subreq: The subrequest to mark 143 * 144 * Mark a subrequest to note that preparation for write failed. 145 */ 146 void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq) 147 { 148 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 149 trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed); 150 } 151 EXPORT_SYMBOL(netfs_prepare_write_failed); 152 153 /* 154 * Prepare a write subrequest. We need to allocate a new subrequest 155 * if we don't have one. 156 */ 157 void netfs_prepare_write(struct netfs_io_request *wreq, 158 struct netfs_io_stream *stream, 159 loff_t start) 160 { 161 struct netfs_io_subrequest *subreq; 162 struct iov_iter *wreq_iter = &wreq->buffer.iter; 163 164 /* Make sure we don't point the iterator at a used-up folio_queue 165 * struct being used as a placeholder to prevent the queue from 166 * collapsing. In such a case, extend the queue. 167 */ 168 if (iov_iter_is_folioq(wreq_iter) && 169 wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) 170 rolling_buffer_make_space(&wreq->buffer); 171 172 subreq = netfs_alloc_subrequest(wreq); 173 subreq->source = stream->source; 174 subreq->start = start; 175 subreq->stream_nr = stream->stream_nr; 176 subreq->io_iter = *wreq_iter; 177 178 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index); 179 180 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); 181 182 stream->sreq_max_len = UINT_MAX; 183 stream->sreq_max_segs = INT_MAX; 184 switch (stream->source) { 185 case NETFS_UPLOAD_TO_SERVER: 186 netfs_stat(&netfs_n_wh_upload); 187 stream->sreq_max_len = wreq->wsize; 188 break; 189 case NETFS_WRITE_TO_CACHE: 190 netfs_stat(&netfs_n_wh_write); 191 break; 192 default: 193 WARN_ON_ONCE(1); 194 break; 195 } 196 197 if (stream->prepare_write) 198 stream->prepare_write(subreq); 199 200 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 201 202 /* We add to the end of the list whilst the collector may be walking 203 * the list. The collector only goes nextwards and uses the lock to 204 * remove entries off of the front. 205 */ 206 spin_lock(&wreq->lock); 207 /* Write IN_PROGRESS before pointer to new subreq */ 208 list_add_tail_release(&subreq->rreq_link, &stream->subrequests); 209 if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { 210 if (!stream->active) { 211 stream->collected_to = subreq->start; 212 /* Write list pointers before active flag */ 213 smp_store_release(&stream->active, true); 214 } 215 } 216 217 spin_unlock(&wreq->lock); 218 219 stream->construct = subreq; 220 } 221 222 /* 223 * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O 224 * operation. The operation may be asynchronous and should call 225 * netfs_write_subrequest_terminated() when complete. 226 */ 227 static void netfs_do_issue_write(struct netfs_io_stream *stream, 228 struct netfs_io_subrequest *subreq) 229 { 230 struct netfs_io_request *wreq = subreq->rreq; 231 232 _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len); 233 234 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 235 return netfs_write_subrequest_terminated(subreq, subreq->error); 236 237 trace_netfs_sreq(subreq, netfs_sreq_trace_submit); 238 stream->issue_write(subreq); 239 } 240 241 void netfs_reissue_write(struct netfs_io_stream *stream, 242 struct netfs_io_subrequest *subreq, 243 struct iov_iter *source) 244 { 245 size_t size = subreq->len - subreq->transferred; 246 247 // TODO: Use encrypted buffer 248 subreq->io_iter = *source; 249 iov_iter_advance(source, size); 250 iov_iter_truncate(&subreq->io_iter, size); 251 252 subreq->retry_count++; 253 subreq->error = 0; 254 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 255 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 256 netfs_stat(&netfs_n_wh_retry_write_subreq); 257 netfs_do_issue_write(stream, subreq); 258 } 259 260 void netfs_issue_write(struct netfs_io_request *wreq, 261 struct netfs_io_stream *stream) 262 { 263 struct netfs_io_subrequest *subreq = stream->construct; 264 265 if (!subreq) 266 return; 267 stream->construct = NULL; 268 subreq->io_iter.count = subreq->len; 269 netfs_do_issue_write(stream, subreq); 270 } 271 272 /* 273 * Add data to the write subrequest, dispatching each as we fill it up or if it 274 * is discontiguous with the previous. We only fill one part at a time so that 275 * we can avoid overrunning the credits obtained (cifs) and try to parallelise 276 * content-crypto preparation with network writes. 277 */ 278 size_t netfs_advance_write(struct netfs_io_request *wreq, 279 struct netfs_io_stream *stream, 280 loff_t start, size_t len, bool to_eof) 281 { 282 struct netfs_io_subrequest *subreq = stream->construct; 283 size_t part; 284 285 if (!stream->avail) { 286 _leave("no write"); 287 return len; 288 } 289 290 _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0); 291 292 if (subreq && start != subreq->start + subreq->len) { 293 netfs_issue_write(wreq, stream); 294 subreq = NULL; 295 } 296 297 if (!stream->construct) 298 netfs_prepare_write(wreq, stream, start); 299 subreq = stream->construct; 300 301 part = umin(stream->sreq_max_len - subreq->len, len); 302 _debug("part %zx/%zx %zx/%zx", subreq->len, stream->sreq_max_len, part, len); 303 subreq->len += part; 304 subreq->nr_segs++; 305 stream->submit_extendable_to -= part; 306 307 if (subreq->len >= stream->sreq_max_len || 308 subreq->nr_segs >= stream->sreq_max_segs || 309 to_eof) { 310 netfs_issue_write(wreq, stream); 311 subreq = NULL; 312 } 313 314 return part; 315 } 316 317 /* 318 * Write some of a pending folio data back to the server. 319 */ 320 static int netfs_write_folio(struct netfs_io_request *wreq, 321 struct writeback_control *wbc, 322 struct folio *folio) 323 { 324 struct netfs_io_stream *upload = &wreq->io_streams[0]; 325 struct netfs_io_stream *cache = &wreq->io_streams[1]; 326 struct netfs_io_stream *stream; 327 struct netfs_group *fgroup; /* TODO: Use this with ceph */ 328 struct netfs_folio *finfo; 329 size_t iter_off = 0; 330 size_t fsize = folio_size(folio), flen = fsize, foff = 0; 331 loff_t fpos = folio_pos(folio), i_size; 332 bool to_eof = false, streamw = false; 333 bool debug = false; 334 335 _enter(""); 336 337 if (rolling_buffer_make_space(&wreq->buffer) < 0) 338 return -ENOMEM; 339 340 /* netfs_perform_write() may shift i_size around the page or from out 341 * of the page to beyond it, but cannot move i_size into or through the 342 * page since we have it locked. 343 */ 344 i_size = i_size_read(wreq->inode); 345 346 if (fpos >= i_size) { 347 /* mmap beyond eof. */ 348 _debug("beyond eof"); 349 folio_start_writeback(folio); 350 folio_unlock(folio); 351 wreq->nr_group_rel += netfs_folio_written_back(folio); 352 netfs_put_group_many(wreq->group, wreq->nr_group_rel); 353 wreq->nr_group_rel = 0; 354 return 0; 355 } 356 357 if (fpos + fsize > wreq->i_size) 358 wreq->i_size = i_size; 359 360 fgroup = netfs_folio_group(folio); 361 finfo = netfs_folio_info(folio); 362 if (finfo) { 363 foff = finfo->dirty_offset; 364 flen = foff + finfo->dirty_len; 365 streamw = true; 366 } 367 368 if (wreq->origin == NETFS_WRITETHROUGH) { 369 to_eof = false; 370 if (flen > i_size - fpos) 371 flen = i_size - fpos; 372 } else if (flen > i_size - fpos) { 373 flen = i_size - fpos; 374 if (!streamw) 375 folio_zero_segment(folio, flen, fsize); 376 to_eof = true; 377 } else if (flen == i_size - fpos) { 378 to_eof = true; 379 } 380 flen -= foff; 381 382 _debug("folio %zx %zx %zx", foff, flen, fsize); 383 384 /* Deal with discontinuities in the stream of dirty pages. These can 385 * arise from a number of sources: 386 * 387 * (1) Intervening non-dirty pages from random-access writes, multiple 388 * flushers writing back different parts simultaneously and manual 389 * syncing. 390 * 391 * (2) Partially-written pages from write-streaming. 392 * 393 * (3) Pages that belong to a different write-back group (eg. Ceph 394 * snapshots). 395 * 396 * (4) Actually-clean pages that were marked for write to the cache 397 * when they were read. Note that these appear as a special 398 * write-back group. 399 */ 400 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) { 401 netfs_issue_write(wreq, upload); 402 } else if (fgroup != wreq->group) { 403 /* We can't write this page to the server yet. */ 404 kdebug("wrong group"); 405 folio_redirty_for_writepage(wbc, folio); 406 folio_unlock(folio); 407 netfs_issue_write(wreq, upload); 408 netfs_issue_write(wreq, cache); 409 return 0; 410 } 411 412 if (foff > 0) 413 netfs_issue_write(wreq, upload); 414 if (streamw) 415 netfs_issue_write(wreq, cache); 416 417 folio_start_writeback(folio); 418 folio_unlock(folio); 419 420 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) { 421 if (!cache->avail) { 422 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); 423 netfs_issue_write(wreq, upload); 424 netfs_folio_written_back(folio); 425 return 0; 426 } 427 trace_netfs_folio(folio, netfs_folio_trace_store_copy); 428 } else if (!upload->avail && !cache->avail) { 429 trace_netfs_folio(folio, netfs_folio_trace_cancel_store); 430 netfs_folio_written_back(folio); 431 return 0; 432 } else if (!upload->construct) { 433 trace_netfs_folio(folio, netfs_folio_trace_store); 434 } else { 435 trace_netfs_folio(folio, netfs_folio_trace_store_plus); 436 } 437 438 /* Attach the folio to the rolling buffer. */ 439 rolling_buffer_append(&wreq->buffer, folio, 0); 440 441 /* Move the submission point forward to allow for write-streaming data 442 * not starting at the front of the page. We don't do write-streaming 443 * with the cache as the cache requires DIO alignment. 444 * 445 * Also skip uploading for data that's been read and just needs copying 446 * to the cache. 447 */ 448 for (int s = 0; s < NR_IO_STREAMS; s++) { 449 stream = &wreq->io_streams[s]; 450 stream->submit_off = foff; 451 stream->submit_len = flen; 452 if (!stream->avail || 453 (stream->source == NETFS_WRITE_TO_CACHE && streamw) || 454 (stream->source == NETFS_UPLOAD_TO_SERVER && 455 fgroup == NETFS_FOLIO_COPY_TO_CACHE)) { 456 stream->submit_off = UINT_MAX; 457 stream->submit_len = 0; 458 } 459 } 460 461 /* Attach the folio to one or more subrequests. For a big folio, we 462 * could end up with thousands of subrequests if the wsize is small - 463 * but we might need to wait during the creation of subrequests for 464 * network resources (eg. SMB credits). 465 */ 466 for (;;) { 467 ssize_t part; 468 size_t lowest_off = ULONG_MAX; 469 int choose_s = -1; 470 471 /* Always add to the lowest-submitted stream first. */ 472 for (int s = 0; s < NR_IO_STREAMS; s++) { 473 stream = &wreq->io_streams[s]; 474 if (stream->submit_len > 0 && 475 stream->submit_off < lowest_off) { 476 lowest_off = stream->submit_off; 477 choose_s = s; 478 } 479 } 480 481 if (choose_s < 0) 482 break; 483 stream = &wreq->io_streams[choose_s]; 484 485 /* Advance the iterator(s). */ 486 if (stream->submit_off > iter_off) { 487 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off); 488 iter_off = stream->submit_off; 489 } 490 491 atomic64_set(&wreq->issued_to, fpos + stream->submit_off); 492 stream->submit_extendable_to = fsize - stream->submit_off; 493 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off, 494 stream->submit_len, to_eof); 495 stream->submit_off += part; 496 if (part > stream->submit_len) 497 stream->submit_len = 0; 498 else 499 stream->submit_len -= part; 500 if (part > 0) 501 debug = true; 502 } 503 504 if (fsize > iter_off) 505 rolling_buffer_advance(&wreq->buffer, fsize - iter_off); 506 atomic64_set(&wreq->issued_to, fpos + fsize); 507 508 if (!debug) 509 kdebug("R=%x: No submit", wreq->debug_id); 510 511 if (foff + flen < fsize) 512 for (int s = 0; s < NR_IO_STREAMS; s++) 513 netfs_issue_write(wreq, &wreq->io_streams[s]); 514 515 _leave(" = 0"); 516 return 0; 517 } 518 519 /* 520 * End the issuing of writes, letting the collector know we're done. 521 */ 522 static void netfs_end_issue_write(struct netfs_io_request *wreq) 523 { 524 bool needs_poke = true; 525 526 smp_wmb(); /* Write subreq lists before ALL_QUEUED. */ 527 set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); 528 529 for (int s = 0; s < NR_IO_STREAMS; s++) { 530 struct netfs_io_stream *stream = &wreq->io_streams[s]; 531 532 if (!stream->active) 533 continue; 534 if (!list_empty(&stream->subrequests)) 535 needs_poke = false; 536 netfs_issue_write(wreq, stream); 537 } 538 539 if (needs_poke) 540 netfs_wake_collector(wreq); 541 } 542 543 /* 544 * Write some of the pending data back to the server 545 */ 546 int netfs_writepages(struct address_space *mapping, 547 struct writeback_control *wbc) 548 { 549 struct netfs_inode *ictx = netfs_inode(mapping->host); 550 struct netfs_io_request *wreq = NULL; 551 struct folio *folio; 552 int error = 0; 553 554 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) 555 return 0; 556 557 /* Need the first folio to be able to set up the op. */ 558 folio = writeback_iter(mapping, wbc, NULL, &error); 559 if (!folio) 560 goto out; 561 562 wreq = netfs_create_write_req(mapping, NULL, folio_pos(folio), NETFS_WRITEBACK); 563 if (IS_ERR(wreq)) { 564 error = PTR_ERR(wreq); 565 goto couldnt_start; 566 } 567 568 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); 569 trace_netfs_write(wreq, netfs_write_trace_writeback); 570 netfs_stat(&netfs_n_wh_writepages); 571 572 do { 573 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to)); 574 575 /* It appears we don't have to handle cyclic writeback wrapping. */ 576 WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to)); 577 578 if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE && 579 unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) { 580 set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); 581 wreq->netfs_ops->begin_writeback(wreq); 582 } 583 584 error = netfs_write_folio(wreq, wbc, folio); 585 if (error == -ENOMEM) { 586 folio_redirty_for_writepage(wbc, folio); 587 folio_unlock(folio); 588 } 589 } while ((folio = writeback_iter(mapping, wbc, folio, &error))); 590 591 netfs_end_issue_write(wreq); 592 netfs_wake_collector(wreq); 593 594 netfs_put_request(wreq, netfs_rreq_trace_put_return); 595 _leave(" = %d", error); 596 return error; 597 598 couldnt_start: 599 if (error == -ENOMEM) { 600 folio_redirty_for_writepage(wbc, folio); 601 folio_unlock(folio); 602 folio = writeback_iter(mapping, wbc, folio, &error); 603 WARN_ON_ONCE(folio != NULL); 604 } else { 605 netfs_kill_dirty_pages(mapping, wbc, folio); 606 } 607 out: 608 netfs_wb_end(ictx); 609 _leave(" = %d", error); 610 return error; 611 } 612 EXPORT_SYMBOL(netfs_writepages); 613 614 /* 615 * Begin a write operation for writing through the pagecache. 616 */ 617 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len) 618 { 619 struct netfs_io_request *wreq = NULL; 620 struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp)); 621 622 netfs_wb_begin(ictx, false); 623 624 wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp, 625 iocb->ki_pos, NETFS_WRITETHROUGH); 626 if (IS_ERR(wreq)) { 627 netfs_wb_end(ictx); 628 return wreq; 629 } 630 631 wreq->io_streams[0].avail = true; 632 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); 633 trace_netfs_write(wreq, netfs_write_trace_writethrough); 634 return wreq; 635 } 636 637 /* 638 * Advance the state of the write operation used when writing through the 639 * pagecache. Data has been copied into the pagecache that we need to append 640 * to the request. If we've added more than wsize then we need to create a new 641 * subrequest. 642 */ 643 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, 644 struct folio *folio, size_t copied, bool to_page_end, 645 struct folio **writethrough_cache) 646 { 647 int ret; 648 649 _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u", 650 wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end); 651 652 /* The folio is locked. */ 653 654 if (*writethrough_cache != folio) { 655 if (*writethrough_cache) { 656 /* Did the folio get moved? */ 657 folio_put(*writethrough_cache); 658 *writethrough_cache = NULL; 659 } 660 /* We can make multiple writes to the folio... */ 661 if (wreq->len == 0) 662 trace_netfs_folio(folio, netfs_folio_trace_wthru); 663 else 664 trace_netfs_folio(folio, netfs_folio_trace_wthru_plus); 665 *writethrough_cache = folio; 666 folio_get(folio); 667 } 668 669 wreq->len += copied; 670 671 if (!to_page_end) { 672 folio_mark_dirty(folio); 673 folio_unlock(folio); 674 return 0; 675 } 676 677 ret = netfs_write_folio(wreq, wbc, folio); 678 folio_put(*writethrough_cache); 679 *writethrough_cache = NULL; 680 wreq->submitted = wreq->len; 681 return ret; 682 } 683 684 /* 685 * End a write operation used when writing through the pagecache. 686 */ 687 ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, 688 struct folio *writethrough_cache) 689 { 690 ssize_t ret; 691 692 _enter("R=%x", wreq->debug_id); 693 694 if (writethrough_cache) { 695 folio_lock(writethrough_cache); 696 netfs_write_folio(wreq, wbc, writethrough_cache); 697 folio_put(writethrough_cache); 698 wreq->submitted = wreq->len; 699 } 700 701 netfs_end_issue_write(wreq); 702 703 if (wreq->iocb) 704 ret = -EIOCBQUEUED; 705 else 706 ret = netfs_wait_for_write(wreq); 707 netfs_put_request(wreq, netfs_rreq_trace_put_return); 708 return ret; 709 } 710 711 /* 712 * Write some of a pending folio data back to the server and/or the cache. 713 */ 714 static int netfs_write_folio_single(struct netfs_io_request *wreq, 715 struct folio *folio) 716 { 717 struct netfs_io_stream *upload = &wreq->io_streams[0]; 718 struct netfs_io_stream *cache = &wreq->io_streams[1]; 719 struct netfs_io_stream *stream; 720 size_t iter_off = 0; 721 size_t fsize = folio_size(folio), flen; 722 loff_t fpos = folio_pos(folio); 723 bool to_eof = false; 724 bool no_debug = false; 725 726 _enter(""); 727 728 flen = folio_size(folio); 729 if (flen > wreq->i_size - fpos) { 730 flen = wreq->i_size - fpos; 731 folio_zero_segment(folio, flen, fsize); 732 to_eof = true; 733 } else if (flen == wreq->i_size - fpos) { 734 to_eof = true; 735 } 736 737 _debug("folio %zx/%zx", flen, fsize); 738 739 if (!upload->avail && !cache->avail) { 740 trace_netfs_folio(folio, netfs_folio_trace_cancel_store); 741 return 0; 742 } 743 744 if (!upload->construct) 745 trace_netfs_folio(folio, netfs_folio_trace_store); 746 else 747 trace_netfs_folio(folio, netfs_folio_trace_store_plus); 748 749 /* Attach the folio to the rolling buffer. */ 750 folio_get(folio); 751 rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK); 752 753 /* Move the submission point forward to allow for write-streaming data 754 * not starting at the front of the page. We don't do write-streaming 755 * with the cache as the cache requires DIO alignment. 756 * 757 * Also skip uploading for data that's been read and just needs copying 758 * to the cache. 759 */ 760 for (int s = 0; s < NR_IO_STREAMS; s++) { 761 stream = &wreq->io_streams[s]; 762 stream->submit_off = 0; 763 stream->submit_len = flen; 764 if (!stream->avail) { 765 stream->submit_off = UINT_MAX; 766 stream->submit_len = 0; 767 } 768 } 769 770 /* Attach the folio to one or more subrequests. For a big folio, we 771 * could end up with thousands of subrequests if the wsize is small - 772 * but we might need to wait during the creation of subrequests for 773 * network resources (eg. SMB credits). 774 */ 775 for (;;) { 776 ssize_t part; 777 size_t lowest_off = ULONG_MAX; 778 int choose_s = -1; 779 780 /* Always add to the lowest-submitted stream first. */ 781 for (int s = 0; s < NR_IO_STREAMS; s++) { 782 stream = &wreq->io_streams[s]; 783 if (stream->submit_len > 0 && 784 stream->submit_off < lowest_off) { 785 lowest_off = stream->submit_off; 786 choose_s = s; 787 } 788 } 789 790 if (choose_s < 0) 791 break; 792 stream = &wreq->io_streams[choose_s]; 793 794 /* Advance the iterator(s). */ 795 if (stream->submit_off > iter_off) { 796 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off); 797 iter_off = stream->submit_off; 798 } 799 800 atomic64_set(&wreq->issued_to, fpos + stream->submit_off); 801 stream->submit_extendable_to = fsize - stream->submit_off; 802 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off, 803 stream->submit_len, to_eof); 804 stream->submit_off += part; 805 if (part > stream->submit_len) 806 stream->submit_len = 0; 807 else 808 stream->submit_len -= part; 809 if (part > 0) 810 no_debug = true; 811 } 812 813 wreq->buffer.iter.iov_offset = 0; 814 if (fsize > iter_off) 815 rolling_buffer_advance(&wreq->buffer, fsize - iter_off); 816 atomic64_set(&wreq->issued_to, fpos + fsize); 817 818 if (!no_debug) 819 kdebug("R=%x: No submit", wreq->debug_id); 820 _leave(" = 0"); 821 return 0; 822 } 823 824 /** 825 * netfs_writeback_single - Write back a monolithic payload 826 * @mapping: The mapping to write from 827 * @wbc: Hints from the VM 828 * @iter: Data to write, must be ITER_FOLIOQ. 829 * 830 * Write a monolithic, non-pagecache object back to the server and/or 831 * the cache. 832 * 833 * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE; 834 * or a negative error code. 835 */ 836 int netfs_writeback_single(struct address_space *mapping, 837 struct writeback_control *wbc, 838 struct iov_iter *iter) 839 { 840 struct netfs_io_request *wreq; 841 struct netfs_inode *ictx = netfs_inode(mapping->host); 842 struct folio_queue *fq; 843 size_t size = iov_iter_count(iter); 844 int ret; 845 846 if (WARN_ON_ONCE(!iov_iter_is_folioq(iter))) 847 return -EIO; 848 849 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) { 850 /* The VFS will have undirtied the inode. */ 851 netfs_single_mark_inode_dirty(&ictx->inode); 852 return 1; 853 } 854 855 wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE); 856 if (IS_ERR(wreq)) { 857 ret = PTR_ERR(wreq); 858 goto couldnt_start; 859 } 860 861 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); 862 trace_netfs_write(wreq, netfs_write_trace_writeback_single); 863 netfs_stat(&netfs_n_wh_writepages); 864 865 if (__test_and_set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags)) 866 wreq->netfs_ops->begin_writeback(wreq); 867 868 for (fq = (struct folio_queue *)iter->folioq; fq; fq = fq->next) { 869 for (int slot = 0; slot < folioq_count(fq); slot++) { 870 struct folio *folio = folioq_folio(fq, slot); 871 size_t part = umin(folioq_folio_size(fq, slot), size); 872 873 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to)); 874 875 ret = netfs_write_folio_single(wreq, folio); 876 if (ret < 0) 877 goto stop; 878 size -= part; 879 if (size <= 0) 880 goto stop; 881 } 882 } 883 884 stop: 885 for (int s = 0; s < NR_IO_STREAMS; s++) 886 netfs_issue_write(wreq, &wreq->io_streams[s]); 887 smp_wmb(); /* Write lists before ALL_QUEUED. */ 888 set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); 889 890 netfs_wake_collector(wreq); 891 892 netfs_put_request(wreq, netfs_rreq_trace_put_return); 893 _leave(" = %d", ret); 894 return ret; 895 896 couldnt_start: 897 netfs_wb_end(ictx); 898 _leave(" = %d", ret); 899 return ret; 900 } 901 EXPORT_SYMBOL(netfs_writeback_single); 902