1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem high-level buffered write support.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/export.h>
9 #include <linux/fs.h>
10 #include <linux/mm.h>
11 #include <linux/pagemap.h>
12 #include <linux/slab.h>
13 #include "internal.h"
14
15 /*
16 * Grab a folio for writing and lock it. Attempt to allocate as large a folio
17 * as possible to hold as much of the remaining length as possible in one go.
18 */
netfs_grab_folio_for_write(struct address_space * mapping,uoff_t pos,size_t part)19 static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
20 uoff_t pos, size_t part)
21 {
22 pgoff_t index = pos / PAGE_SIZE;
23 fgf_t fgp_flags = FGP_WRITEBEGIN;
24
25 if (mapping_large_folio_support(mapping))
26 fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part);
27
28 return __filemap_get_folio(mapping, index, fgp_flags,
29 mapping_gfp_mask(mapping));
30 }
31
32 /*
33 * Update i_size and estimate the update to i_blocks to reflect the additional
34 * data written into the pagecache until we can find out from the server what
35 * the values actually are.
36 */
netfs_update_i_size(struct netfs_inode * ctx,struct inode * inode,uoff_t pos,size_t copied)37 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
38 uoff_t pos, size_t copied)
39 {
40 uoff_t i_size, end = pos + copied;
41 blkcnt_t add;
42 size_t gap;
43
44 if (end <= i_size_read(inode))
45 return;
46
47 if (ctx->ops->update_i_size) {
48 ctx->ops->update_i_size(inode, end);
49 return;
50 }
51
52 spin_lock(&inode->i_lock);
53
54 i_size = i_size_read(inode);
55 if (end > i_size) {
56 i_size_write(inode, end);
57
58 gap = SECTOR_SIZE - (i_size & (SECTOR_SIZE - 1));
59 if (copied > gap) {
60 add = DIV_ROUND_UP(copied - gap, SECTOR_SIZE);
61
62 inode->i_blocks = min_t(blkcnt_t,
63 DIV_ROUND_UP(end, SECTOR_SIZE),
64 inode->i_blocks + add);
65 }
66 }
67 spin_unlock(&inode->i_lock);
68 }
69
70 /**
71 * netfs_perform_write - Copy data into the pagecache.
72 * @iocb: The operation parameters
73 * @iter: The source buffer
74 * @netfs_group: Grouping for dirty folios (eg. ceph snaps).
75 *
76 * Copy data into pagecache folios attached to the inode specified by @iocb.
77 * The caller must hold appropriate inode locks.
78 *
79 * Dirty folios are tagged with a netfs_folio struct if they're not up to date
80 * to indicate the range modified. Dirty folios may also be tagged with a
81 * netfs-specific grouping such that data from an old group gets flushed before
82 * a new one is started.
83 */
netfs_perform_write(struct kiocb * iocb,struct iov_iter * iter,struct netfs_group * netfs_group)84 ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
85 struct netfs_group *netfs_group)
86 {
87 struct file *file = iocb->ki_filp;
88 struct inode *inode = file_inode(file);
89 struct address_space *mapping = inode->i_mapping;
90 struct netfs_inode *ctx = netfs_inode(inode);
91 struct folio *folio = NULL;
92 unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0;
93 ssize_t written = 0, ret;
94 uoff_t pos = iocb->ki_pos;
95 size_t max_chunk = mapping_max_folio_size(mapping);
96 bool maybe_trouble = false;
97
98 netfs_stat(&netfs_n_wh_buffered_write);
99
100 do {
101 enum netfs_folio_trace trace;
102 struct netfs_folio *finfo;
103 struct netfs_group *group;
104 uoff_t fpos;
105 size_t flen;
106 size_t offset; /* Offset into pagecache folio */
107 size_t part; /* Bytes to write to folio */
108 size_t copied; /* Bytes copied from user */
109 void *priv;
110
111 offset = pos & (max_chunk - 1);
112 part = min(max_chunk - offset, iov_iter_count(iter));
113
114 /* Bring in the user pages that we will copy from _first_ lest
115 * we hit a nasty deadlock on copying from the same page as
116 * we're writing to, without it being marked uptodate.
117 *
118 * Not only is this an optimisation, but it is also required to
119 * check that the address is actually valid, when atomic
120 * usercopies are used below.
121 *
122 * We rely on the page being held onto long enough by the LRU
123 * that we can grab it below if this causes it to be read.
124 */
125 ret = -EFAULT;
126 if (unlikely(fault_in_iov_iter_readable(iter, part) == part))
127 break;
128
129 folio = netfs_grab_folio_for_write(mapping, pos, part);
130 if (IS_ERR(folio)) {
131 ret = PTR_ERR(folio);
132 break;
133 }
134
135 flen = folio_size(folio);
136 fpos = folio_pos(folio);
137 offset = pos - fpos;
138 part = min_t(size_t, flen - offset, part);
139
140 /* Wait for writeback to complete. The writeback engine owns
141 * the info in folio->private and may change it until it
142 * removes the WB mark.
143 */
144 if (folio_get_private(folio) &&
145 folio_wait_writeback_killable(folio)) {
146 ret = written ? -EINTR : -ERESTARTSYS;
147 goto error_folio_unlock;
148 }
149
150 if (signal_pending(current)) {
151 ret = written ? -EINTR : -ERESTARTSYS;
152 goto error_folio_unlock;
153 }
154
155 finfo = netfs_folio_info(folio);
156 group = netfs_folio_group(folio);
157
158 /* If the requested group differs from the group set on the
159 * page, then we need to flush out the folio if it has a group
160 * set (ie. is non-NULL). Note that COPY_TO_CACHE is a special
161 * case, being a netfs annotation rather than an actual group.
162 *
163 * The filesystem isn't permitted to mix writes with groups and
164 * writes without groups as the NULL group is used to indicate
165 * that no group is set.
166 */
167 if (unlikely(group != netfs_group) &&
168 group != NETFS_FOLIO_COPY_TO_CACHE &&
169 group) {
170 WARN_ON_ONCE(!netfs_group);
171 goto flush_content;
172 }
173
174 /* Decide how we should modify a folio. We might be attempting
175 * to do write-streaming, as we don't want to a local RMW cycle
176 * if we can avoid it. If we're doing local caching or content
177 * crypto, we award that priority over avoiding RMW. If the
178 * file is open readably, then we let ->read_folio() fill in
179 * the gaps.
180 */
181 if (folio_test_uptodate(folio)) {
182 if (mapping_writably_mapped(mapping))
183 flush_dcache_folio(folio);
184 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
185 if (unlikely(copied == 0))
186 goto copy_failed;
187 trace = netfs_folio_is_uptodate;
188 goto copied_uptodate;
189 }
190
191 /* If the page is above the zero-point then we assume that the
192 * server would just return a block of zeros or a short read if
193 * we try to read it.
194 */
195 if (fpos >= netfs_read_zero_point(inode)) {
196 folio_zero_segment(folio, 0, offset);
197 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
198 if (unlikely(copied == 0))
199 goto copy_failed;
200 folio_zero_segment(folio, offset + copied, flen);
201 if (finfo)
202 trace = netfs_modify_and_clear_rm_finfo;
203 else
204 trace = netfs_modify_and_clear;
205 goto mark_uptodate;
206 }
207
208 /* See if we can write a whole folio in one go. */
209 if (!maybe_trouble && offset == 0 && part >= flen) {
210 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
211 if (likely(copied == part)) {
212 if (finfo)
213 trace = netfs_whole_folio_modify_filled;
214 else
215 trace = netfs_whole_folio_modify;
216 goto mark_uptodate;
217 }
218 if (copied == 0)
219 goto copy_failed;
220 if (!finfo || copied <= finfo->dirty_offset) {
221 maybe_trouble = true;
222 iov_iter_revert(iter, copied);
223 copied = 0;
224 folio_unlock(folio);
225 goto retry;
226 }
227
228 /* We overwrote some existing dirty data, so we have to
229 * accept the partial write.
230 */
231 finfo->dirty_len += finfo->dirty_offset;
232 if (finfo->dirty_len == flen) {
233 trace = netfs_whole_folio_modify_filled_efault;
234 goto mark_uptodate;
235 }
236 if (copied > finfo->dirty_len)
237 finfo->dirty_len = copied;
238 finfo->dirty_offset = 0;
239 trace = netfs_whole_folio_modify_efault;
240 goto copied;
241 }
242
243 /* We don't want to do a streaming write on a file that loses
244 * caching service temporarily because the backing store got
245 * culled.
246 */
247 if (netfs_is_cache_maybe_enabled(ctx)) {
248 if (finfo) {
249 netfs_stat(&netfs_n_wh_wstream_conflict);
250 goto flush_content;
251 }
252 ret = netfs_prefetch_for_write(file, folio, offset, part);
253 if (ret < 0) {
254 _debug("prefetch = %zd", ret);
255 goto error_folio_unlock;
256 }
257 /* Note that copy-to-cache may have been set. */
258
259 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
260 if (unlikely(copied == 0))
261 goto copy_failed;
262 trace = netfs_just_prefetch;
263 goto copied_uptodate;
264 }
265
266 /* Do a streaming write on a folio that has nothing in it yet. */
267 if (!finfo) {
268 ret = -EIO;
269 if (WARN_ON(folio_get_private(folio)))
270 goto error_folio_unlock;
271 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
272 if (unlikely(copied == 0))
273 goto copy_failed;
274 if (offset == 0 && copied == flen) {
275 trace = netfs_streaming_filled_page;
276 goto mark_uptodate;
277 }
278
279 finfo = kzalloc_obj(*finfo);
280 if (!finfo) {
281 iov_iter_revert(iter, copied);
282 ret = -ENOMEM;
283 goto error_folio_unlock;
284 }
285 finfo->netfs_group = netfs_get_group(netfs_group);
286 finfo->dirty_offset = offset;
287 finfo->dirty_len = copied;
288 folio_attach_private(folio, (void *)((unsigned long)finfo |
289 NETFS_FOLIO_INFO));
290 trace = netfs_streaming_write;
291 goto copied;
292 }
293
294 /* We can continue a streaming write only if it continues on
295 * from the previous. If it overlaps, we must flush lest we
296 * suffer a partial copy and disjoint dirty regions.
297 */
298 if (offset == finfo->dirty_offset + finfo->dirty_len) {
299 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
300 if (unlikely(copied == 0))
301 goto copy_failed;
302 finfo->dirty_len += copied;
303 if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
304 trace = netfs_streaming_cont_filled_page;
305 goto mark_uptodate;
306 }
307 trace = netfs_streaming_write_cont;
308 goto copied;
309 }
310
311 /* Incompatible write; flush the folio and try again. */
312 flush_content:
313 trace_netfs_folio(folio, netfs_flush_content);
314 folio_unlock(folio);
315 folio_put(folio);
316 ret = filemap_write_and_wait_range(mapping, fpos, fpos + flen - 1);
317 if (ret < 0)
318 goto out;
319 continue;
320
321 /* Mark a folio as being up to data when we've filled it
322 * completely. If the folio has a group attached, then it must
323 * be the same group, otherwise we should have flushed it out
324 * above. We have to get rid of the netfs_folio struct if
325 * there was one.
326 */
327 mark_uptodate:
328 folio_mark_uptodate(folio);
329
330 copied_uptodate:
331 priv = folio_get_private(folio);
332 if (likely(priv == netfs_group)) {
333 /* Already set correctly; no change required. */
334 } else if (priv == NETFS_FOLIO_COPY_TO_CACHE) {
335 if (!netfs_group)
336 folio_detach_private(folio);
337 else
338 folio_change_private(folio, netfs_get_group(netfs_group));
339 } else if (!priv) {
340 folio_attach_private(folio, netfs_get_group(netfs_group));
341 } else {
342 WARN_ON_ONCE(!finfo);
343 if (netfs_group)
344 /* finfo->netfs_group has a ref */
345 folio_change_private(folio, netfs_group);
346 else
347 folio_detach_private(folio);
348 kfree(finfo);
349 }
350
351 copied:
352 trace_netfs_folio(folio, trace);
353 flush_dcache_folio(folio);
354
355 /* Update the inode size if we moved the EOF marker */
356 netfs_update_i_size(ctx, inode, pos, copied);
357 pos += copied;
358 written += copied;
359
360 folio_mark_dirty(folio);
361 folio_unlock(folio);
362 retry:
363 folio_put(folio);
364 folio = NULL;
365
366 ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags);
367 if (unlikely(ret < 0))
368 break;
369
370 cond_resched();
371 } while (iov_iter_count(iter));
372
373 out:
374 if (likely(written)) {
375 /* Set indication that ctime and mtime got updated in case
376 * close is deferred.
377 */
378 set_bit(NETFS_ICTX_MODIFIED_ATTR, &ctx->flags);
379 if (unlikely(ctx->ops->post_modify))
380 ctx->ops->post_modify(inode);
381 }
382
383 iocb->ki_pos += written;
384 _leave(" = %zd [%zd]", written, ret);
385 return written ? written : ret;
386
387 copy_failed:
388 ret = -EFAULT;
389 error_folio_unlock:
390 folio_unlock(folio);
391 folio_put(folio);
392 goto out;
393 }
394 EXPORT_SYMBOL(netfs_perform_write);
395
396 /**
397 * netfs_buffered_write_iter_locked - write data to a file
398 * @iocb: IO state structure (file, offset, etc.)
399 * @from: iov_iter with data to write
400 * @netfs_group: Grouping for dirty folios (eg. ceph snaps).
401 *
402 * This function does all the work needed for actually writing data to a
403 * file. It does all basic checks, removes SUID from the file, updates
404 * modification times and calls proper subroutines depending on whether we
405 * do direct IO or a standard buffered write.
406 *
407 * The caller must hold appropriate locks around this function and have called
408 * generic_write_checks() already. The caller is also responsible for doing
409 * any necessary syncing afterwards.
410 *
411 * This function does *not* take care of syncing data in case of O_SYNC write.
412 * A caller has to handle it. This is mainly due to the fact that we want to
413 * avoid syncing under i_rwsem.
414 *
415 * Return:
416 * * number of bytes written, even for truncated writes
417 * * negative error code if no data has been written at all
418 */
netfs_buffered_write_iter_locked(struct kiocb * iocb,struct iov_iter * from,struct netfs_group * netfs_group)419 ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
420 struct netfs_group *netfs_group)
421 {
422 struct file *file = iocb->ki_filp;
423 struct inode *inode = file_inode(file);
424 ssize_t ret;
425
426 trace_netfs_write_iter(iocb, from);
427
428 ret = file_remove_privs(file);
429 if (ret)
430 return ret;
431
432 ret = file_update_time(file);
433 if (ret)
434 return ret;
435
436 if (iocb->ki_pos > i_size_read(inode)) {
437 ret = netfs_clear_stale_pre_isize(inode, i_size_read(inode),
438 iocb->ki_pos,
439 iocb->ki_flags & IOCB_NOWAIT);
440 if (ret)
441 return ret;
442 }
443
444 return netfs_perform_write(iocb, from, netfs_group);
445 }
446 EXPORT_SYMBOL(netfs_buffered_write_iter_locked);
447
448 /**
449 * netfs_file_write_iter - write data to a file
450 * @iocb: IO state structure
451 * @from: iov_iter with data to write
452 *
453 * Perform a write to a file, writing into the pagecache if possible and doing
454 * an unbuffered write instead if not.
455 *
456 * Return:
457 * * Negative error code if no data has been written at all of
458 * vfs_fsync_range() failed for a synchronous write
459 * * Number of bytes written, even for truncated writes
460 */
netfs_file_write_iter(struct kiocb * iocb,struct iov_iter * from)461 ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
462 {
463 struct file *file = iocb->ki_filp;
464 struct inode *inode = file->f_mapping->host;
465 struct netfs_inode *ictx = netfs_inode(inode);
466 ssize_t ret;
467
468 _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode));
469
470 if (!iov_iter_count(from))
471 return 0;
472
473 if ((iocb->ki_flags & IOCB_DIRECT) ||
474 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
475 return netfs_unbuffered_write_iter(iocb, from);
476
477 ret = netfs_start_io_write(inode);
478 if (ret < 0)
479 return ret;
480
481 ret = generic_write_checks(iocb, from);
482 if (ret > 0)
483 ret = netfs_buffered_write_iter_locked(iocb, from, NULL);
484 netfs_end_io_write(inode);
485 if (ret > 0)
486 ret = generic_write_sync(iocb, ret);
487 return ret;
488 }
489 EXPORT_SYMBOL(netfs_file_write_iter);
490
491 /*
492 * Notification that a previously read-only page is about to become writable.
493 * The caller indicates the precise page that needs to be written to, but
494 * we only track group on a per-folio basis, so we block more often than
495 * we might otherwise.
496 */
netfs_page_mkwrite(struct vm_fault * vmf,struct netfs_group * netfs_group)497 vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group)
498 {
499 struct netfs_group *group;
500 struct folio *folio = page_folio(vmf->page);
501 struct file *file = vmf->vma->vm_file;
502 struct address_space *mapping = file->f_mapping;
503 struct inode *inode = file_inode(file);
504 struct netfs_inode *ictx = netfs_inode(inode);
505 vm_fault_t ret = VM_FAULT_NOPAGE;
506 void *priv;
507 int err;
508
509 _enter("%lx", folio->index);
510
511 sb_start_pagefault(inode->i_sb);
512
513 if (folio_lock_killable(folio) < 0)
514 goto out;
515 if (folio->mapping != mapping)
516 goto unlock;
517 if (folio_wait_writeback_killable(folio) < 0)
518 goto unlock;
519
520 /* Can we see a streaming write here? */
521 if (WARN_ON(!folio_test_uptodate(folio))) {
522 ret = VM_FAULT_SIGBUS;
523 goto unlock;
524 }
525
526 group = netfs_folio_group(folio);
527 if (group &&
528 group != netfs_group &&
529 group != NETFS_FOLIO_COPY_TO_CACHE) {
530 folio_unlock(folio);
531 err = filemap_fdatawrite_range(mapping,
532 folio_pos(folio),
533 folio_next_pos(folio));
534 switch (err) {
535 case 0:
536 ret = VM_FAULT_RETRY;
537 goto out;
538 case -ENOMEM:
539 ret = VM_FAULT_OOM;
540 goto out;
541 default:
542 ret = VM_FAULT_SIGBUS;
543 goto out;
544 }
545 }
546
547 if (folio_test_dirty(folio))
548 trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
549 else
550 trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
551
552 priv = folio_get_private(folio);
553 if (priv != netfs_group) {
554 if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
555 folio_detach_private(folio);
556 else if (netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
557 folio_change_private(folio, netfs_get_group(netfs_group));
558 else if (netfs_group && !priv)
559 folio_attach_private(folio, netfs_get_group(netfs_group));
560 else
561 WARN_ON_ONCE(1);
562 }
563
564 file_update_time(file);
565 set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags);
566 if (ictx->ops->post_modify)
567 ictx->ops->post_modify(inode);
568 ret = VM_FAULT_LOCKED;
569 out:
570 sb_end_pagefault(inode->i_sb);
571 return ret;
572 unlock:
573 folio_unlock(folio);
574 goto out;
575 }
576 EXPORT_SYMBOL(netfs_page_mkwrite);
577