1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem high-level (buffered) writeback.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 *
8 * To support network filesystems with local caching, we manage a situation
9 * that can be envisioned like the following:
10 *
11 * +---+---+-----+-----+---+----------+
12 * Folios: | | | | | | |
13 * +---+---+-----+-----+---+----------+
14 *
15 * +------+------+ +----+----+
16 * Upload: | | |.....| | |
17 * (Stream 0) +------+------+ +----+----+
18 *
19 * +------+------+------+------+------+
20 * Cache: | | | | | |
21 * (Stream 1) +------+------+------+------+------+
22 *
23 * Where we have a sequence of folios of varying sizes that we need to overlay
24 * with multiple parallel streams of I/O requests, where the I/O requests in a
25 * stream may also be of various sizes (in cifs, for example, the sizes are
26 * negotiated with the server; in something like ceph, they may represent the
27 * sizes of storage objects).
28 *
29 * The sequence in each stream may contain gaps and noncontiguous subrequests
30 * may be glued together into single vectored write RPCs.
31 */
32
33 #include <linux/export.h>
34 #include <linux/fs.h>
35 #include <linux/mm.h>
36 #include <linux/pagemap.h>
37 #include "internal.h"
38
39 /*
40 * Kill all dirty folios in the event of an unrecoverable error, starting with
41 * a locked folio we've already obtained from writeback_iter().
42 */
netfs_kill_dirty_pages(struct address_space * mapping,struct writeback_control * wbc,struct folio * folio)43 static void netfs_kill_dirty_pages(struct address_space *mapping,
44 struct writeback_control *wbc,
45 struct folio *folio)
46 {
47 int error = 0;
48
49 do {
50 enum netfs_folio_trace why = netfs_folio_trace_kill;
51 struct netfs_group *group = NULL;
52 struct netfs_folio *finfo = NULL;
53 void *priv;
54
55 priv = folio_detach_private(folio);
56 if (priv) {
57 finfo = __netfs_folio_info(priv);
58 if (finfo) {
59 /* Kill folio from streaming write. */
60 group = finfo->netfs_group;
61 why = netfs_folio_trace_kill_s;
62 } else {
63 group = priv;
64 if (group == NETFS_FOLIO_COPY_TO_CACHE) {
65 /* Kill copy-to-cache folio */
66 why = netfs_folio_trace_kill_cc;
67 group = NULL;
68 } else {
69 /* Kill folio with group */
70 why = netfs_folio_trace_kill_g;
71 }
72 }
73 }
74
75 trace_netfs_folio(folio, why);
76
77 folio_start_writeback(folio);
78 folio_unlock(folio);
79 folio_end_writeback(folio);
80
81 netfs_put_group(group);
82 kfree(finfo);
83
84 } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
85 }
86
87 /*
88 * Create a write request and set it up appropriately for the origin type.
89 */
netfs_create_write_req(struct address_space * mapping,struct file * file,loff_t start,enum netfs_io_origin origin)90 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
91 struct file *file,
92 loff_t start,
93 enum netfs_io_origin origin)
94 {
95 struct netfs_io_request *wreq;
96 struct netfs_inode *ictx;
97 bool is_cacheable = (origin == NETFS_WRITEBACK ||
98 origin == NETFS_WRITEBACK_SINGLE ||
99 origin == NETFS_WRITETHROUGH ||
100 origin == NETFS_PGPRIV2_COPY_TO_CACHE);
101
102 wreq = netfs_alloc_request(mapping, file, start, 0, origin);
103 if (IS_ERR(wreq))
104 return wreq;
105
106 _enter("R=%x", wreq->debug_id);
107
108 ictx = netfs_inode(wreq->inode);
109 if (is_cacheable)
110 fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx));
111 if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE, wreq->gfp) < 0)
112 goto nomem;
113
114 wreq->cleaned_to = wreq->start;
115
116 wreq->io_streams[0].stream_nr = 0;
117 wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER;
118 wreq->io_streams[0].prepare_write = ictx->ops->prepare_write;
119 wreq->io_streams[0].issue_write = ictx->ops->issue_write;
120 wreq->io_streams[0].collected_to = start;
121 wreq->io_streams[0].transferred = 0;
122
123 wreq->io_streams[1].stream_nr = 1;
124 wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE;
125 wreq->io_streams[1].collected_to = start;
126 wreq->io_streams[1].transferred = 0;
127 if (fscache_resources_valid(&wreq->cache_resources)) {
128 wreq->io_streams[1].avail = true;
129 wreq->io_streams[1].active = true;
130 wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq;
131 wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write;
132 }
133
134 return wreq;
135 nomem:
136 netfs_put_failed_request(wreq);
137 return ERR_PTR(-ENOMEM);
138 }
139
140 /**
141 * netfs_prepare_write_failed - Note write preparation failed
142 * @subreq: The subrequest to mark
143 *
144 * Mark a subrequest to note that preparation for write failed.
145 */
netfs_prepare_write_failed(struct netfs_io_subrequest * subreq)146 void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq)
147 {
148 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
149 trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed);
150 }
151 EXPORT_SYMBOL(netfs_prepare_write_failed);
152
153 /*
154 * Prepare a write subrequest. We need to allocate a new subrequest
155 * if we don't have one.
156 */
netfs_prepare_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream,loff_t start)157 void netfs_prepare_write(struct netfs_io_request *wreq,
158 struct netfs_io_stream *stream,
159 loff_t start)
160 {
161 struct netfs_io_subrequest *subreq;
162 struct iov_iter *wreq_iter = &wreq->buffer.iter;
163
164 /* Make sure we don't point the iterator at a used-up folio_queue
165 * struct being used as a placeholder to prevent the queue from
166 * collapsing. In such a case, extend the queue.
167 */
168 if (iov_iter_is_folioq(wreq_iter) &&
169 wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq))
170 rolling_buffer_make_space(&wreq->buffer, wreq->gfp);
171
172 subreq = netfs_alloc_subrequest(wreq);
173 if (!subreq)
174 return;
175 subreq->source = stream->source;
176 subreq->start = start;
177 subreq->stream_nr = stream->stream_nr;
178 subreq->io_iter = *wreq_iter;
179
180 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
181
182 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
183
184 stream->sreq_max_len = UINT_MAX;
185 stream->sreq_max_segs = INT_MAX;
186 switch (stream->source) {
187 case NETFS_UPLOAD_TO_SERVER:
188 netfs_stat(&netfs_n_wh_upload);
189 stream->sreq_max_len = wreq->wsize;
190 break;
191 case NETFS_WRITE_TO_CACHE:
192 netfs_stat(&netfs_n_wh_write);
193 break;
194 default:
195 WARN_ON_ONCE(1);
196 break;
197 }
198
199 if (stream->prepare_write)
200 stream->prepare_write(subreq);
201
202 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
203
204 /* We add to the end of the list whilst the collector may be walking
205 * the list. The collector only goes nextwards and uses the lock to
206 * remove entries off of the front.
207 */
208 spin_lock(&wreq->lock);
209 /* Write IN_PROGRESS before pointer to new subreq */
210 list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
211 if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
212 if (!stream->active) {
213 stream->collected_to = subreq->start;
214 /* Write list pointers before active flag */
215 smp_store_release(&stream->active, true);
216 }
217 }
218
219 spin_unlock(&wreq->lock);
220
221 stream->construct = subreq;
222 }
223
224 /*
225 * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O
226 * operation. The operation may be asynchronous and should call
227 * netfs_write_subrequest_terminated() when complete.
228 */
netfs_do_issue_write(struct netfs_io_stream * stream,struct netfs_io_subrequest * subreq)229 static void netfs_do_issue_write(struct netfs_io_stream *stream,
230 struct netfs_io_subrequest *subreq)
231 {
232 struct netfs_io_request *wreq = subreq->rreq;
233
234 _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
235
236 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
237 return netfs_write_subrequest_terminated(subreq, subreq->error);
238
239 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
240 stream->issue_write(subreq);
241 }
242
netfs_reissue_write(struct netfs_io_stream * stream,struct netfs_io_subrequest * subreq,struct iov_iter * source)243 void netfs_reissue_write(struct netfs_io_stream *stream,
244 struct netfs_io_subrequest *subreq,
245 struct iov_iter *source)
246 {
247 size_t size = subreq->len - subreq->transferred;
248
249 // TODO: Use encrypted buffer
250 subreq->io_iter = *source;
251 iov_iter_advance(source, size);
252 iov_iter_truncate(&subreq->io_iter, size);
253
254 subreq->retry_count++;
255 subreq->error = 0;
256 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
257 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
258 netfs_stat(&netfs_n_wh_retry_write_subreq);
259 netfs_do_issue_write(stream, subreq);
260 }
261
netfs_issue_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream)262 void netfs_issue_write(struct netfs_io_request *wreq,
263 struct netfs_io_stream *stream)
264 {
265 struct netfs_io_subrequest *subreq = stream->construct;
266
267 if (!subreq)
268 return;
269 stream->construct = NULL;
270 subreq->io_iter.count = subreq->len;
271 netfs_do_issue_write(stream, subreq);
272 }
273
274 /*
275 * Add data to the write subrequest, dispatching each as we fill it up or if it
276 * is discontiguous with the previous. We only fill one part at a time so that
277 * we can avoid overrunning the credits obtained (cifs) and try to parallelise
278 * content-crypto preparation with network writes.
279 */
netfs_advance_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream,loff_t start,size_t len,bool to_eof)280 size_t netfs_advance_write(struct netfs_io_request *wreq,
281 struct netfs_io_stream *stream,
282 loff_t start, size_t len, bool to_eof)
283 {
284 struct netfs_io_subrequest *subreq = stream->construct;
285 size_t part;
286
287 if (!stream->avail) {
288 _leave("no write");
289 return len;
290 }
291
292 _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0);
293
294 if (subreq && start != subreq->start + subreq->len) {
295 netfs_issue_write(wreq, stream);
296 subreq = NULL;
297 }
298
299 if (!stream->construct)
300 netfs_prepare_write(wreq, stream, start);
301 subreq = stream->construct;
302
303 part = umin(stream->sreq_max_len - subreq->len, len);
304 _debug("part %zx/%zx %zx/%zx", subreq->len, stream->sreq_max_len, part, len);
305 subreq->len += part;
306 subreq->nr_segs++;
307 stream->submit_extendable_to -= part;
308
309 if (subreq->len >= stream->sreq_max_len ||
310 subreq->nr_segs >= stream->sreq_max_segs ||
311 to_eof) {
312 netfs_issue_write(wreq, stream);
313 subreq = NULL;
314 }
315
316 return part;
317 }
318
319 /*
320 * Write some of a pending folio data back to the server.
321 */
netfs_write_folio(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * folio)322 static int netfs_write_folio(struct netfs_io_request *wreq,
323 struct writeback_control *wbc,
324 struct folio *folio)
325 {
326 struct netfs_io_stream *upload = &wreq->io_streams[0];
327 struct netfs_io_stream *cache = &wreq->io_streams[1];
328 struct netfs_io_stream *stream;
329 struct netfs_group *fgroup; /* TODO: Use this with ceph */
330 struct netfs_folio *finfo;
331 size_t iter_off = 0;
332 size_t fsize = folio_size(folio), flen = fsize, foff = 0;
333 loff_t fpos = folio_pos(folio), i_size;
334 bool to_eof = false, streamw = false;
335 bool debug = false;
336
337 _enter("");
338
339 if (rolling_buffer_make_space(&wreq->buffer, wreq->gfp) < 0)
340 return -ENOMEM;
341
342 /* netfs_perform_write() may shift i_size around the page or from out
343 * of the page to beyond it, but cannot move i_size into or through the
344 * page since we have it locked.
345 */
346 i_size = i_size_read(wreq->inode);
347
348 if (fpos >= i_size) {
349 /* mmap beyond eof. */
350 _debug("beyond eof");
351 folio_start_writeback(folio);
352 folio_unlock(folio);
353 wreq->nr_group_rel += netfs_folio_written_back(folio);
354 netfs_put_group_many(wreq->group, wreq->nr_group_rel);
355 wreq->nr_group_rel = 0;
356 return 0;
357 }
358
359 if (fpos + fsize > wreq->i_size)
360 wreq->i_size = i_size;
361
362 fgroup = netfs_folio_group(folio);
363 finfo = netfs_folio_info(folio);
364 if (finfo) {
365 foff = finfo->dirty_offset;
366 flen = foff + finfo->dirty_len;
367 streamw = true;
368 }
369
370 if (wreq->origin == NETFS_WRITETHROUGH) {
371 to_eof = false;
372 if (flen > i_size - fpos)
373 flen = i_size - fpos;
374 } else if (flen > i_size - fpos) {
375 flen = i_size - fpos;
376 if (!streamw)
377 folio_zero_segment(folio, flen, fsize);
378 to_eof = true;
379 } else if (flen == i_size - fpos) {
380 to_eof = true;
381 }
382 flen -= foff;
383
384 _debug("folio %zx %zx %zx", foff, flen, fsize);
385
386 /* Deal with discontinuities in the stream of dirty pages. These can
387 * arise from a number of sources:
388 *
389 * (1) Intervening non-dirty pages from random-access writes, multiple
390 * flushers writing back different parts simultaneously and manual
391 * syncing.
392 *
393 * (2) Partially-written pages from write-streaming.
394 *
395 * (3) Pages that belong to a different write-back group (eg. Ceph
396 * snapshots).
397 *
398 * (4) Actually-clean pages that were marked for write to the cache
399 * when they were read. Note that these appear as a special
400 * write-back group.
401 */
402 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
403 netfs_issue_write(wreq, upload);
404 } else if (fgroup != wreq->group) {
405 /* We can't write this page to the server yet. */
406 kdebug("wrong group");
407 folio_redirty_for_writepage(wbc, folio);
408 folio_unlock(folio);
409 netfs_issue_write(wreq, upload);
410 netfs_issue_write(wreq, cache);
411 return 0;
412 }
413
414 if (foff > 0)
415 netfs_issue_write(wreq, upload);
416 if (streamw)
417 netfs_issue_write(wreq, cache);
418
419 folio_start_writeback(folio);
420 folio_unlock(folio);
421
422 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
423 if (!cache->avail) {
424 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
425 netfs_issue_write(wreq, upload);
426 netfs_folio_written_back(folio);
427 return 0;
428 }
429 trace_netfs_folio(folio, netfs_folio_trace_store_copy);
430 } else if (!upload->avail && !cache->avail) {
431 trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
432 netfs_folio_written_back(folio);
433 return 0;
434 } else if (!upload->construct) {
435 trace_netfs_folio(folio, netfs_folio_trace_store);
436 } else {
437 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
438 }
439
440 /* Attach the folio to the rolling buffer. */
441 rolling_buffer_append(&wreq->buffer, folio, 0, wreq->gfp);
442
443 /* Move the submission point forward to allow for write-streaming data
444 * not starting at the front of the page. We don't do write-streaming
445 * with the cache as the cache requires DIO alignment.
446 *
447 * Also skip uploading for data that's been read and just needs copying
448 * to the cache.
449 */
450 for (int s = 0; s < NR_IO_STREAMS; s++) {
451 stream = &wreq->io_streams[s];
452 stream->submit_off = foff;
453 stream->submit_len = flen;
454 if (!stream->avail ||
455 (stream->source == NETFS_WRITE_TO_CACHE && streamw) ||
456 (stream->source == NETFS_UPLOAD_TO_SERVER &&
457 fgroup == NETFS_FOLIO_COPY_TO_CACHE)) {
458 stream->submit_off = UINT_MAX;
459 stream->submit_len = 0;
460 }
461 }
462
463 /* Attach the folio to one or more subrequests. For a big folio, we
464 * could end up with thousands of subrequests if the wsize is small -
465 * but we might need to wait during the creation of subrequests for
466 * network resources (eg. SMB credits).
467 */
468 for (;;) {
469 ssize_t part;
470 size_t lowest_off = ULONG_MAX;
471 int choose_s = -1;
472
473 /* Always add to the lowest-submitted stream first. */
474 for (int s = 0; s < NR_IO_STREAMS; s++) {
475 stream = &wreq->io_streams[s];
476 if (stream->submit_len > 0 &&
477 stream->submit_off < lowest_off) {
478 lowest_off = stream->submit_off;
479 choose_s = s;
480 }
481 }
482
483 if (choose_s < 0)
484 break;
485 stream = &wreq->io_streams[choose_s];
486
487 /* Advance the iterator(s). */
488 if (stream->submit_off > iter_off) {
489 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off);
490 iter_off = stream->submit_off;
491 }
492
493 atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
494 stream->submit_extendable_to = fsize - stream->submit_off;
495 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
496 stream->submit_len, to_eof);
497 stream->submit_off += part;
498 if (part > stream->submit_len)
499 stream->submit_len = 0;
500 else
501 stream->submit_len -= part;
502 if (part > 0)
503 debug = true;
504 }
505
506 if (fsize > iter_off)
507 rolling_buffer_advance(&wreq->buffer, fsize - iter_off);
508 atomic64_set(&wreq->issued_to, fpos + fsize);
509
510 if (!debug)
511 kdebug("R=%x: No submit", wreq->debug_id);
512
513 if (foff + flen < fsize)
514 for (int s = 0; s < NR_IO_STREAMS; s++)
515 netfs_issue_write(wreq, &wreq->io_streams[s]);
516
517 _leave(" = 0");
518 return 0;
519 }
520
521 /*
522 * End the issuing of writes, letting the collector know we're done.
523 */
netfs_end_issue_write(struct netfs_io_request * wreq)524 static void netfs_end_issue_write(struct netfs_io_request *wreq)
525 {
526 bool needs_poke = true;
527
528 smp_wmb(); /* Write subreq lists before ALL_QUEUED. */
529 set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
530
531 for (int s = 0; s < NR_IO_STREAMS; s++) {
532 struct netfs_io_stream *stream = &wreq->io_streams[s];
533
534 if (!stream->active)
535 continue;
536 if (!list_empty(&stream->subrequests))
537 needs_poke = false;
538 netfs_issue_write(wreq, stream);
539 }
540
541 if (needs_poke)
542 netfs_wake_collector(wreq);
543 }
544
545 /*
546 * Write some of the pending data back to the server
547 */
netfs_writepages(struct address_space * mapping,struct writeback_control * wbc)548 int netfs_writepages(struct address_space *mapping,
549 struct writeback_control *wbc)
550 {
551 struct netfs_inode *ictx = netfs_inode(mapping->host);
552 struct netfs_io_request *wreq = NULL;
553 struct folio *folio;
554 int error = 0;
555
556 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE))
557 return 0;
558
559 /* Need the first folio to be able to set up the op. */
560 folio = writeback_iter(mapping, wbc, NULL, &error);
561 if (!folio)
562 goto out;
563
564 wreq = netfs_create_write_req(mapping, NULL, folio_pos(folio), NETFS_WRITEBACK);
565 if (IS_ERR(wreq)) {
566 error = PTR_ERR(wreq);
567 goto couldnt_start;
568 }
569
570 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
571 trace_netfs_write(wreq, netfs_write_trace_writeback);
572 netfs_stat(&netfs_n_wh_writepages);
573
574 do {
575 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
576
577 /* It appears we don't have to handle cyclic writeback wrapping. */
578 WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to));
579
580 if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE &&
581 unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
582 set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
583 wreq->netfs_ops->begin_writeback(wreq);
584 }
585
586 error = netfs_write_folio(wreq, wbc, folio);
587 if (error == -ENOMEM) {
588 folio_redirty_for_writepage(wbc, folio);
589 folio_unlock(folio);
590 }
591 } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
592
593 netfs_end_issue_write(wreq);
594 netfs_wake_collector(wreq);
595
596 netfs_put_request(wreq, netfs_rreq_trace_put_return);
597 _leave(" = %d", error);
598 return error;
599
600 couldnt_start:
601 if (error == -ENOMEM) {
602 folio_redirty_for_writepage(wbc, folio);
603 folio_unlock(folio);
604 folio = writeback_iter(mapping, wbc, folio, &error);
605 WARN_ON_ONCE(folio != NULL);
606 } else {
607 netfs_kill_dirty_pages(mapping, wbc, folio);
608 }
609 out:
610 netfs_wb_end(ictx);
611 _leave(" = %d", error);
612 return error;
613 }
614 EXPORT_SYMBOL(netfs_writepages);
615
616 /*
617 * Begin a write operation for writing through the pagecache.
618 */
netfs_begin_writethrough(struct kiocb * iocb,size_t len)619 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len)
620 {
621 struct netfs_io_request *wreq = NULL;
622 struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp));
623
624 netfs_wb_begin(ictx, false);
625
626 wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp,
627 iocb->ki_pos, NETFS_WRITETHROUGH);
628 if (IS_ERR(wreq)) {
629 netfs_wb_end(ictx);
630 return wreq;
631 }
632
633 wreq->io_streams[0].avail = true;
634 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
635 trace_netfs_write(wreq, netfs_write_trace_writethrough);
636 return wreq;
637 }
638
639 /*
640 * Advance the state of the write operation used when writing through the
641 * pagecache. Data has been copied into the pagecache that we need to append
642 * to the request. If we've added more than wsize then we need to create a new
643 * subrequest.
644 */
netfs_advance_writethrough(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * folio,size_t copied,bool to_page_end,struct folio ** writethrough_cache)645 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
646 struct folio *folio, size_t copied, bool to_page_end,
647 struct folio **writethrough_cache)
648 {
649 int ret;
650
651 _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u",
652 wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end);
653
654 /* The folio is locked. */
655
656 if (*writethrough_cache != folio) {
657 if (*writethrough_cache) {
658 /* Did the folio get moved? */
659 folio_put(*writethrough_cache);
660 *writethrough_cache = NULL;
661 }
662 /* We can make multiple writes to the folio... */
663 if (wreq->len == 0)
664 trace_netfs_folio(folio, netfs_folio_trace_wthru);
665 else
666 trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
667 *writethrough_cache = folio;
668 folio_get(folio);
669 }
670
671 wreq->len += copied;
672
673 if (!to_page_end) {
674 folio_mark_dirty(folio);
675 folio_unlock(folio);
676 return 0;
677 }
678
679 ret = netfs_write_folio(wreq, wbc, folio);
680 folio_put(*writethrough_cache);
681 *writethrough_cache = NULL;
682 wreq->submitted = wreq->len;
683 return ret;
684 }
685
686 /*
687 * End a write operation used when writing through the pagecache.
688 */
netfs_end_writethrough(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * writethrough_cache)689 ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
690 struct folio *writethrough_cache)
691 {
692 ssize_t ret;
693
694 _enter("R=%x", wreq->debug_id);
695
696 if (writethrough_cache) {
697 folio_lock(writethrough_cache);
698 netfs_write_folio(wreq, wbc, writethrough_cache);
699 folio_put(writethrough_cache);
700 wreq->submitted = wreq->len;
701 }
702
703 netfs_end_issue_write(wreq);
704
705 if (wreq->iocb)
706 ret = -EIOCBQUEUED;
707 else
708 ret = netfs_wait_for_write(wreq);
709 netfs_put_request(wreq, netfs_rreq_trace_put_return);
710 return ret;
711 }
712
713 /*
714 * Write some of a pending folio data back to the server and/or the cache.
715 */
netfs_write_folio_single(struct netfs_io_request * wreq,struct folio * folio)716 static int netfs_write_folio_single(struct netfs_io_request *wreq,
717 struct folio *folio)
718 {
719 struct netfs_io_stream *upload = &wreq->io_streams[0];
720 struct netfs_io_stream *cache = &wreq->io_streams[1];
721 struct netfs_io_stream *stream;
722 size_t iter_off = 0;
723 size_t fsize = folio_size(folio), flen;
724 loff_t fpos = folio_pos(folio);
725 ssize_t ret;
726 bool to_eof = false;
727 bool no_debug = false;
728
729 _enter("");
730
731 flen = folio_size(folio);
732 if (flen > wreq->i_size - fpos) {
733 flen = wreq->i_size - fpos;
734 folio_zero_segment(folio, flen, fsize);
735 to_eof = true;
736 } else if (flen == wreq->i_size - fpos) {
737 to_eof = true;
738 }
739
740 _debug("folio %zx/%zx", flen, fsize);
741
742 if (!upload->avail && !cache->avail) {
743 trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
744 return 0;
745 }
746
747 if (!upload->construct)
748 trace_netfs_folio(folio, netfs_folio_trace_store);
749 else
750 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
751
752 /* Attach the folio to the rolling buffer. */
753 folio_get(folio);
754 ret = rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK, wreq->gfp);
755 if (ret < 0) {
756 folio_put(folio);
757 return ret;
758 }
759
760 /* Move the submission point forward to allow for write-streaming data
761 * not starting at the front of the page. We don't do write-streaming
762 * with the cache as the cache requires DIO alignment.
763 *
764 * Also skip uploading for data that's been read and just needs copying
765 * to the cache.
766 */
767 for (int s = 0; s < NR_IO_STREAMS; s++) {
768 stream = &wreq->io_streams[s];
769 stream->submit_off = 0;
770 stream->submit_len = flen;
771 if (!stream->avail) {
772 stream->submit_off = UINT_MAX;
773 stream->submit_len = 0;
774 }
775 }
776
777 /* Attach the folio to one or more subrequests. For a big folio, we
778 * could end up with thousands of subrequests if the wsize is small -
779 * but we might need to wait during the creation of subrequests for
780 * network resources (eg. SMB credits).
781 */
782 for (;;) {
783 ssize_t part;
784 size_t lowest_off = ULONG_MAX;
785 int choose_s = -1;
786
787 /* Always add to the lowest-submitted stream first. */
788 for (int s = 0; s < NR_IO_STREAMS; s++) {
789 stream = &wreq->io_streams[s];
790 if (stream->submit_len > 0 &&
791 stream->submit_off < lowest_off) {
792 lowest_off = stream->submit_off;
793 choose_s = s;
794 }
795 }
796
797 if (choose_s < 0)
798 break;
799 stream = &wreq->io_streams[choose_s];
800
801 /* Advance the iterator(s). */
802 if (stream->submit_off > iter_off) {
803 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off);
804 iter_off = stream->submit_off;
805 }
806
807 atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
808 stream->submit_extendable_to = fsize - stream->submit_off;
809 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
810 stream->submit_len, to_eof);
811 stream->submit_off += part;
812 if (part > stream->submit_len)
813 stream->submit_len = 0;
814 else
815 stream->submit_len -= part;
816 if (part > 0)
817 no_debug = true;
818 }
819
820 wreq->buffer.iter.iov_offset = 0;
821 if (fsize > iter_off)
822 rolling_buffer_advance(&wreq->buffer, fsize - iter_off);
823 atomic64_set(&wreq->issued_to, fpos + fsize);
824
825 if (!no_debug)
826 kdebug("R=%x: No submit", wreq->debug_id);
827 _leave(" = 0");
828 return 0;
829 }
830
831 /**
832 * netfs_writeback_single - Write back a monolithic payload
833 * @mapping: The mapping to write from
834 * @wbc: Hints from the VM
835 * @iter: Data to write, must be ITER_FOLIOQ.
836 *
837 * Write a monolithic, non-pagecache object back to the server and/or
838 * the cache.
839 *
840 * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE;
841 * or a negative error code.
842 */
netfs_writeback_single(struct address_space * mapping,struct writeback_control * wbc,struct iov_iter * iter)843 int netfs_writeback_single(struct address_space *mapping,
844 struct writeback_control *wbc,
845 struct iov_iter *iter)
846 {
847 struct netfs_io_request *wreq;
848 struct netfs_inode *ictx = netfs_inode(mapping->host);
849 struct folio_queue *fq;
850 size_t size = iov_iter_count(iter);
851 int ret;
852
853 if (WARN_ON_ONCE(!iov_iter_is_folioq(iter)))
854 return -EIO;
855
856 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) {
857 /* The VFS will have undirtied the inode. */
858 netfs_single_mark_inode_dirty(&ictx->inode);
859 return 1;
860 }
861
862 wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE);
863 if (IS_ERR(wreq)) {
864 ret = PTR_ERR(wreq);
865 goto couldnt_start;
866 }
867
868 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
869 trace_netfs_write(wreq, netfs_write_trace_writeback_single);
870 netfs_stat(&netfs_n_wh_writepages);
871
872 if (__test_and_set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))
873 wreq->netfs_ops->begin_writeback(wreq);
874
875 for (fq = (struct folio_queue *)iter->folioq; fq; fq = fq->next) {
876 for (int slot = 0; slot < folioq_count(fq); slot++) {
877 struct folio *folio = folioq_folio(fq, slot);
878 size_t part = umin(folioq_folio_size(fq, slot), size);
879
880 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
881
882 ret = netfs_write_folio_single(wreq, folio);
883 if (ret < 0)
884 goto stop;
885 size -= part;
886 if (size <= 0)
887 goto stop;
888 }
889 }
890
891 stop:
892 for (int s = 0; s < NR_IO_STREAMS; s++)
893 netfs_issue_write(wreq, &wreq->io_streams[s]);
894 smp_wmb(); /* Write lists before ALL_QUEUED. */
895 set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
896
897 netfs_wake_collector(wreq);
898
899 netfs_put_request(wreq, netfs_rreq_trace_put_return);
900 _leave(" = %d", ret);
901 return ret;
902
903 couldnt_start:
904 netfs_wb_end(ictx);
905 _leave(" = %d", ret);
906 return ret;
907 }
908 EXPORT_SYMBOL(netfs_writeback_single);
909