xref: /linux/fs/ceph/addr.c (revision 8fe8e62f432f0a7a0292a328a73c6912b9a2b73b)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/ceph/ceph_debug.h>
3 
4 #include <linux/backing-dev.h>
5 #include <linux/fs.h>
6 #include <linux/mm.h>
7 #include <linux/swap.h>
8 #include <linux/pagemap.h>
9 #include <linux/slab.h>
10 #include <linux/folio_batch.h>
11 #include <linux/task_io_accounting_ops.h>
12 #include <linux/signal.h>
13 #include <linux/iversion.h>
14 #include <linux/ktime.h>
15 #include <linux/netfs.h>
16 #include <trace/events/netfs.h>
17 
18 #include "super.h"
19 #include "mds_client.h"
20 #include "cache.h"
21 #include "metric.h"
22 #include "subvolume_metrics.h"
23 #include "crypto.h"
24 #include <linux/ceph/osd_client.h>
25 #include <linux/ceph/striper.h>
26 
27 /*
28  * Ceph address space ops.
29  *
30  * There are a few funny things going on here.
31  *
32  * The folio->private field is used to reference a struct
33  * ceph_snap_context for _every_ dirty folio.  This indicates which
34  * snapshot the folio was logically dirtied in, and thus which snap
35  * context needs to be associated with the osd write during writeback.
36  *
37  * Similarly, struct ceph_inode_info maintains a set of counters to
38  * count dirty pages on the inode.  In the absence of snapshots,
39  * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count.
40  *
41  * When a snapshot is taken (that is, when the client receives
42  * notification that a snapshot was taken), each inode with caps and
43  * with dirty pages (dirty pages implies there is a cap) gets a new
44  * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending
45  * order, new snaps go to the tail).  The i_wrbuffer_ref_head count is
46  * moved to capsnap->dirty. (Unless a sync write is currently in
47  * progress.  In that case, the capsnap is said to be "pending", new
48  * writes cannot start, and the capsnap isn't "finalized" until the
49  * write completes (or fails) and a final size/mtime for the inode for
50  * that snap can be settled upon.)  i_wrbuffer_ref_head is reset to 0.
51  *
52  * On writeback, we must submit writes to the osd IN SNAP ORDER.  So,
53  * we look for the first capsnap in i_cap_snaps and write out pages in
54  * that snap context _only_.  Then we move on to the next capsnap,
55  * eventually reaching the "live" or "head" context (i.e., pages that
56  * are not yet snapped) and are writing the most recently dirtied
57  * pages.
58  *
59  * Invalidate and so forth must take care to ensure the dirty page
60  * accounting is preserved.
61  */
62 
63 #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10))
64 #define CONGESTION_OFF_THRESH(congestion_kb)				\
65 	(CONGESTION_ON_THRESH(congestion_kb) -				\
66 	 (CONGESTION_ON_THRESH(congestion_kb) >> 2))
67 
68 static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len,
69 					struct folio **foliop, void **_fsdata);
70 
71 static inline
72 struct ceph_snap_context *ceph_folio_snap_context(const struct folio *folio)
73 {
74 	return folio->private;
75 }
76 
77 /*
78  * Dirty a page.  Optimistically adjust accounting, on the assumption
79  * that we won't race with invalidate.  If we do, readjust.
80  */
81 static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio)
82 {
83 	struct inode *inode = mapping->host;
84 	struct ceph_client *cl = ceph_inode_to_client(inode);
85 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
86 	struct ceph_inode_info *ci;
87 	struct ceph_snap_context *snapc;
88 
89 	if (folio_test_dirty(folio)) {
90 		doutc(cl, "%llx.%llx %p idx %lu -- already dirty\n",
91 		      ceph_vinop(inode), folio, folio->index);
92 		VM_BUG_ON_FOLIO(!folio_test_private(folio), folio);
93 		return false;
94 	}
95 
96 	atomic64_inc(&mdsc->dirty_folios);
97 
98 	ci = ceph_inode(inode);
99 
100 	/* dirty the head */
101 	spin_lock(&ci->i_ceph_lock);
102 	if (__ceph_have_pending_cap_snap(ci)) {
103 		struct ceph_cap_snap *capsnap =
104 				list_last_entry(&ci->i_cap_snaps,
105 						struct ceph_cap_snap,
106 						ci_item);
107 		snapc = ceph_get_snap_context(capsnap->context);
108 		capsnap->dirty_pages++;
109 	} else {
110 		BUG_ON(!ci->i_head_snapc);
111 		snapc = ceph_get_snap_context(ci->i_head_snapc);
112 		++ci->i_wrbuffer_ref_head;
113 	}
114 	if (ci->i_wrbuffer_ref == 0)
115 		ihold(inode);
116 	++ci->i_wrbuffer_ref;
117 	doutc(cl, "%llx.%llx %p idx %lu head %d/%d -> %d/%d "
118 	      "snapc %p seq %lld (%d snaps)\n",
119 	      ceph_vinop(inode), folio, folio->index,
120 	      ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1,
121 	      ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head,
122 	      snapc, snapc->seq, snapc->num_snaps);
123 	spin_unlock(&ci->i_ceph_lock);
124 
125 	/*
126 	 * Reference snap context in folio->private. Setting folio->private is
127 	 * what gets us the invalidate_folio callback.
128 	 */
129 	VM_WARN_ON_FOLIO(folio->private, folio);
130 	folio_attach_private(folio, snapc);
131 
132 	return ceph_fscache_dirty_folio(mapping, folio);
133 }
134 
135 /*
136  * If we are truncating the full folio (i.e. offset == 0), adjust the
137  * dirty folio counters appropriately.  Only called if there is private
138  * data on the folio.
139  */
140 static void ceph_invalidate_folio(struct folio *folio, size_t offset,
141 				size_t length)
142 {
143 	struct inode *inode = folio->mapping->host;
144 	struct ceph_client *cl = ceph_inode_to_client(inode);
145 	struct ceph_inode_info *ci = ceph_inode(inode);
146 	struct ceph_snap_context *snapc;
147 
148 
149 	if (offset != 0 || length != folio_size(folio)) {
150 		doutc(cl, "%llx.%llx idx %lu partial dirty page %zu~%zu\n",
151 		      ceph_vinop(inode), folio->index, offset, length);
152 		return;
153 	}
154 
155 	WARN_ON(!folio_test_locked(folio));
156 	if (folio_test_private(folio)) {
157 		doutc(cl, "%llx.%llx idx %lu full dirty page\n",
158 		      ceph_vinop(inode), folio->index);
159 
160 		snapc = folio_detach_private(folio);
161 		ceph_put_wrbuffer_cap_refs(ci, 1, snapc);
162 		ceph_put_snap_context(snapc);
163 	}
164 
165 	netfs_invalidate_folio(folio, offset, length);
166 }
167 
168 static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)
169 {
170 	struct inode *inode = rreq->inode;
171 	struct ceph_inode_info *ci = ceph_inode(inode);
172 	struct ceph_file_layout *lo = &ci->i_layout;
173 	unsigned long max_pages = inode->i_sb->s_bdi->ra_pages;
174 	loff_t end = rreq->start + rreq->len, new_end;
175 	struct ceph_netfs_request_data *priv = rreq->netfs_priv;
176 	unsigned long max_len;
177 	u32 blockoff;
178 
179 	if (priv) {
180 		/* Readahead is disabled by posix_fadvise POSIX_FADV_RANDOM */
181 		if (priv->file_ra_disabled)
182 			max_pages = 0;
183 		else
184 			max_pages = priv->file_ra_pages;
185 
186 	}
187 
188 	/* Readahead is disabled */
189 	if (!max_pages)
190 		return;
191 
192 	max_len = max_pages << PAGE_SHIFT;
193 
194 	/*
195 	 * Try to expand the length forward by rounding up it to the next
196 	 * block, but do not exceed the file size, unless the original
197 	 * request already exceeds it.
198 	 */
199 	new_end = umin(round_up(end, lo->stripe_unit), rreq->i_size);
200 	if (new_end > end && new_end <= rreq->start + max_len)
201 		rreq->len = new_end - rreq->start;
202 
203 	/* Try to expand the start downward */
204 	div_u64_rem(rreq->start, lo->stripe_unit, &blockoff);
205 	if (rreq->len + blockoff <= max_len) {
206 		rreq->start -= blockoff;
207 		rreq->len += blockoff;
208 	}
209 }
210 
211 static void finish_netfs_read(struct ceph_osd_request *req)
212 {
213 	struct inode *inode = req->r_inode;
214 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
215 	struct ceph_client *cl = fsc->client;
216 	struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0);
217 	struct netfs_io_subrequest *subreq = req->r_priv;
218 	struct ceph_osd_req_op *op = &req->r_ops[0];
219 	int err = req->r_result;
220 	bool sparse = (op->op == CEPH_OSD_OP_SPARSE_READ);
221 
222 	ceph_update_read_metrics(&fsc->mdsc->metric, req->r_start_latency,
223 				 req->r_end_latency, osd_data->length, err);
224 
225 	doutc(cl, "result %d subreq->len=%zu i_size=%lld\n", req->r_result,
226 	      subreq->len, i_size_read(req->r_inode));
227 
228 	/* no object means success but no data */
229 	if (err == -ENOENT) {
230 		__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
231 		__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
232 		err = 0;
233 	} else if (err == -EBLOCKLISTED) {
234 		fsc->blocklisted = true;
235 	}
236 
237 	if (err >= 0) {
238 		if (sparse && err > 0)
239 			err = ceph_sparse_ext_map_end(op);
240 		if (err < subreq->len &&
241 		    subreq->rreq->origin != NETFS_UNBUFFERED_READ &&
242 		    subreq->rreq->origin != NETFS_DIO_READ)
243 			__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
244 		if (IS_ENCRYPTED(inode) && err > 0) {
245 			err = ceph_fscrypt_decrypt_extents(inode,
246 					osd_data->pages, subreq->start,
247 					op->extent.sparse_ext,
248 					op->extent.sparse_ext_cnt);
249 			if (err > subreq->len)
250 				err = subreq->len;
251 		}
252 		if (err > 0)
253 			__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
254 	}
255 
256 	if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) {
257 		int num_pages = calc_pages_for(osd_data->alignment,
258 					       osd_data->length);
259 
260 		for (int i = 0; i < num_pages; i++)
261 			put_page(osd_data->pages[i]);
262 		kvfree(osd_data->pages);
263 	}
264 	if (err > 0) {
265 		ceph_subvolume_metrics_record_io(fsc->mdsc, ceph_inode(inode),
266 						 false, err,
267 						 req->r_start_latency,
268 						 req->r_end_latency);
269 		subreq->transferred = err;
270 		err = 0;
271 	}
272 	subreq->error = err;
273 	trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress);
274 	netfs_read_subreq_terminated(subreq);
275 	iput(req->r_inode);
276 	ceph_dec_osd_stopping_blocker(fsc->mdsc);
277 }
278 
279 #ifdef CONFIG_CEPH_FS_INLINE_DATA
280 static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
281 {
282 	struct netfs_io_request *rreq = subreq->rreq;
283 	struct inode *inode = rreq->inode;
284 	struct ceph_mds_reply_info_parsed *rinfo;
285 	struct ceph_mds_reply_info_in *iinfo;
286 	struct ceph_mds_request *req;
287 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
288 	ssize_t err = 0;
289 	size_t len;
290 	int mode;
291 
292 	if (rreq->origin != NETFS_UNBUFFERED_READ &&
293 	    rreq->origin != NETFS_DIO_READ)
294 		__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
295 	__clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
296 
297 	if (subreq->start >= inode->i_size)
298 		goto out;
299 
300 	/* We need to fetch the inline data. */
301 	mode = ceph_try_to_choose_auth_mds(inode, CEPH_STAT_CAP_INLINE_DATA);
302 	req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, mode);
303 	if (IS_ERR(req)) {
304 		err = PTR_ERR(req);
305 		goto out;
306 	}
307 	req->r_ino1 = ceph_vino(inode);
308 	req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA);
309 	req->r_num_caps = 2;
310 
311 	trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
312 	err = ceph_mdsc_do_request(mdsc, NULL, req);
313 	if (err < 0)
314 		goto out;
315 
316 	rinfo = &req->r_reply_info;
317 	iinfo = &rinfo->targeti;
318 	if (iinfo->inline_version == CEPH_INLINE_NONE) {
319 		/* The data got uninlined */
320 		ceph_mdsc_put_request(req);
321 		return false;
322 	}
323 
324 	if (subreq->start >= iinfo->inline_len) {
325 		ceph_mdsc_put_request(req);
326 		err = -ENODATA;
327 		goto out;
328 	}
329 
330 	len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);
331 	err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter);
332 	if (err == 0) {
333 		err = -EFAULT;
334 	} else {
335 		subreq->transferred += err;
336 		err = 0;
337 	}
338 
339 	ceph_mdsc_put_request(req);
340 out:
341 	subreq->error = err;
342 	trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress);
343 	netfs_read_subreq_terminated(subreq);
344 	return true;
345 }
346 #else
347 static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
348 {
349 	return false;
350 }
351 #endif /* CONFIG_CEPH_FS_INLINE_DATA */
352 
353 static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq)
354 {
355 	struct netfs_io_request *rreq = subreq->rreq;
356 	struct inode *inode = rreq->inode;
357 	struct ceph_inode_info *ci = ceph_inode(inode);
358 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
359 	u64 objno, objoff;
360 	u32 xlen;
361 
362 	/* Truncate the extent at the end of the current block */
363 	ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
364 				      &objno, &objoff, &xlen);
365 	rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize);
366 	return 0;
367 }
368 
369 static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
370 {
371 	struct netfs_io_request *rreq = subreq->rreq;
372 	struct inode *inode = rreq->inode;
373 	struct ceph_inode_info *ci = ceph_inode(inode);
374 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
375 	struct ceph_client *cl = fsc->client;
376 	struct ceph_osd_request *req = NULL;
377 	struct ceph_vino vino = ceph_vino(inode);
378 	int err = 0;
379 	u64 len;
380 	bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);
381 	u64 off = subreq->start;
382 	int extent_cnt;
383 
384 	if (ceph_inode_is_shutdown(inode)) {
385 		err = -EIO;
386 		goto out;
387 	}
388 
389 	if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq))
390 		return;
391 
392 	// TODO: This rounding here is slightly dodgy.  It *should* work, for
393 	// now, as the cache only deals in blocks that are a multiple of
394 	// PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE.  What needs to
395 	// happen is for the fscrypt driving to be moved into netfslib and the
396 	// data in the cache also to be stored encrypted.
397 	len = subreq->len;
398 	ceph_fscrypt_adjust_off_and_len(inode, &off, &len);
399 
400 	req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino,
401 			off, &len, 0, 1, sparse ? CEPH_OSD_OP_SPARSE_READ : CEPH_OSD_OP_READ,
402 			CEPH_OSD_FLAG_READ, NULL, ci->i_truncate_seq,
403 			ci->i_truncate_size, false);
404 	if (IS_ERR(req)) {
405 		err = PTR_ERR(req);
406 		req = NULL;
407 		goto out;
408 	}
409 
410 	if (sparse) {
411 		extent_cnt = __ceph_sparse_read_ext_count(inode, len);
412 		err = ceph_alloc_sparse_ext_map(&req->r_ops[0], extent_cnt);
413 		if (err)
414 			goto out;
415 	}
416 
417 	doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n",
418 	      ceph_vinop(inode), subreq->start, subreq->len, len);
419 
420 	/*
421 	 * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for
422 	 * encrypted inodes. We'd need infrastructure that handles an iov_iter
423 	 * instead of page arrays, and we don't have that as of yet. Once the
424 	 * dust settles on the write helpers and encrypt/decrypt routines for
425 	 * netfs, we should be able to rework this.
426 	 */
427 	if (IS_ENCRYPTED(inode)) {
428 		struct page **pages;
429 		size_t page_off;
430 
431 		/*
432 		 * FIXME: io_iter.count needs to be corrected to aligned
433 		 * length. Otherwise, iov_iter_get_pages_alloc2() operates
434 		 * with the initial unaligned length value. As a result,
435 		 * ceph_msg_data_cursor_init() triggers BUG_ON() in the case
436 		 * if msg->sparse_read_total > msg->data_length.
437 		 */
438 		subreq->io_iter.count = len;
439 
440 		err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off);
441 		if (err < 0) {
442 			doutc(cl, "%llx.%llx failed to allocate pages, %d\n",
443 			      ceph_vinop(inode), err);
444 			goto out;
445 		}
446 
447 		/* should always give us a page-aligned read */
448 		WARN_ON_ONCE(page_off);
449 		len = err;
450 		err = 0;
451 
452 		osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false,
453 						 false);
454 	} else {
455 		osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter);
456 	}
457 	if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
458 		struct ceph_osd_data *osd_data =
459 			osd_req_op_extent_osd_data(req, 0);
460 
461 		if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) {
462 			int num_pages = calc_pages_for(osd_data->alignment,
463 						       osd_data->length);
464 
465 			for (int i = 0; i < num_pages; i++)
466 				put_page(osd_data->pages[i]);
467 			kvfree(osd_data->pages);
468 		}
469 		err = -EIO;
470 		goto out;
471 	}
472 	req->r_callback = finish_netfs_read;
473 	req->r_priv = subreq;
474 	req->r_inode = inode;
475 	ihold(inode);
476 
477 	trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
478 	ceph_osdc_start_request(req->r_osdc, req);
479 out:
480 	ceph_osdc_put_request(req);
481 	if (err) {
482 		subreq->error = err;
483 		netfs_read_subreq_terminated(subreq);
484 	}
485 	doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err);
486 }
487 
488 static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)
489 {
490 	struct inode *inode = rreq->inode;
491 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
492 	struct ceph_client *cl = ceph_inode_to_client(inode);
493 	int got = 0, want = CEPH_CAP_FILE_CACHE;
494 	struct ceph_netfs_request_data *priv;
495 	int ret = 0;
496 
497 	/* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */
498 	__set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);
499 
500 	if (rreq->origin != NETFS_READAHEAD)
501 		return 0;
502 
503 	priv = kzalloc_obj(*priv, GFP_NOFS);
504 	if (!priv)
505 		return -ENOMEM;
506 
507 	if (file) {
508 		struct ceph_rw_context *rw_ctx;
509 		struct ceph_file_info *fi = file->private_data;
510 
511 		priv->file_ra_pages = file->f_ra.ra_pages;
512 		priv->file_ra_disabled = file->f_mode & FMODE_RANDOM;
513 
514 		rw_ctx = ceph_find_rw_context(fi);
515 		if (rw_ctx) {
516 			rreq->netfs_priv = priv;
517 			return 0;
518 		}
519 
520 		/* If this is a lazy fd, also try to get LAZYIO caps */
521 		if (fi->fmode & CEPH_FILE_MODE_LAZY)
522 			want |= CEPH_CAP_FILE_LAZYIO;
523 	}
524 
525 	/*
526 	 * readahead callers do not necessarily hold Fcb caps
527 	 * (e.g. fadvise, madvise).
528 	 */
529 	ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got);
530 	if (ret < 0) {
531 		doutc(cl, "%llx.%llx, error getting cap\n", ceph_vinop(inode));
532 		goto out;
533 	}
534 
535 	if (!(got & want)) {
536 		doutc(cl, "%llx.%llx, no cache cap\n", ceph_vinop(inode));
537 		ret = -EACCES;
538 		goto out;
539 	}
540 	if (ret == 0) {
541 		ret = -EACCES;
542 		goto out;
543 	}
544 
545 	priv->caps = got;
546 	rreq->netfs_priv = priv;
547 	rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize;
548 
549 out:
550 	if (ret < 0) {
551 		if (got)
552 			ceph_put_cap_refs(ceph_inode(inode), got);
553 		kfree(priv);
554 	}
555 
556 	return ret;
557 }
558 
559 static void ceph_netfs_free_request(struct netfs_io_request *rreq)
560 {
561 	struct ceph_netfs_request_data *priv = rreq->netfs_priv;
562 
563 	if (!priv)
564 		return;
565 
566 	if (priv->caps)
567 		ceph_put_cap_refs(ceph_inode(rreq->inode), priv->caps);
568 	kfree(priv);
569 	rreq->netfs_priv = NULL;
570 }
571 
572 const struct netfs_request_ops ceph_netfs_ops = {
573 	.init_request		= ceph_init_request,
574 	.free_request		= ceph_netfs_free_request,
575 	.prepare_read		= ceph_netfs_prepare_read,
576 	.issue_read		= ceph_netfs_issue_read,
577 	.expand_readahead	= ceph_netfs_expand_readahead,
578 	.check_write_begin	= ceph_netfs_check_write_begin,
579 };
580 
581 #ifdef CONFIG_CEPH_FSCACHE
582 static void ceph_folio_start_fscache(struct folio *folio)
583 {
584 	folio_start_private_2(folio); /* [DEPRECATED] */
585 }
586 
587 static void ceph_folio_wait_fscache(struct folio *folio)
588 {
589 	folio_wait_private_2(folio); /* [DEPRECATED] */
590 }
591 
592 static void ceph_fscache_write_terminated(void *priv, ssize_t error)
593 {
594 	struct inode *inode = priv;
595 
596 	if (IS_ERR_VALUE(error) && error != -ENOBUFS)
597 		ceph_fscache_invalidate(inode, false);
598 }
599 
600 static void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching)
601 {
602 	struct ceph_inode_info *ci = ceph_inode(inode);
603 	struct fscache_cookie *cookie = ceph_fscache_cookie(ci);
604 
605 	fscache_write_to_cache(cookie, inode->i_mapping, off, len, i_size_read(inode),
606 			       ceph_fscache_write_terminated, inode, true, caching);
607 }
608 #else
609 static inline void ceph_folio_start_fscache(struct folio *folio)
610 {
611 }
612 
613 static inline void ceph_folio_wait_fscache(struct folio *folio)
614 {
615 }
616 
617 static inline void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching)
618 {
619 }
620 #endif /* CONFIG_CEPH_FSCACHE */
621 
622 struct ceph_writeback_ctl
623 {
624 	loff_t i_size;
625 	u64 truncate_size;
626 	u32 truncate_seq;
627 	bool size_stable;
628 
629 	bool head_snapc;
630 	struct ceph_snap_context *snapc;
631 	struct ceph_snap_context *last_snapc;
632 
633 	bool done;
634 	bool should_loop;
635 	bool range_whole;
636 	pgoff_t start_index;
637 	pgoff_t index;
638 	pgoff_t end;
639 	xa_mark_t tag;
640 
641 	pgoff_t strip_unit_end;
642 	unsigned int wsize;
643 	unsigned int nr_folios;
644 	unsigned int max_pages;
645 	unsigned int locked_pages;
646 
647 	int op_idx;
648 	int num_ops;
649 	u64 offset;
650 	u64 len;
651 
652 	struct folio_batch fbatch;
653 	unsigned int processed_in_fbatch;
654 
655 	bool from_pool;
656 	struct page **pages;
657 	struct page **data_pages;
658 };
659 
660 /*
661  * Get ref for the oldest snapc for an inode with dirty data... that is, the
662  * only snap context we are allowed to write back.
663  */
664 static struct ceph_snap_context *
665 get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl,
666 		   struct ceph_snap_context *page_snapc)
667 {
668 	struct ceph_inode_info *ci = ceph_inode(inode);
669 	struct ceph_client *cl = ceph_inode_to_client(inode);
670 	struct ceph_snap_context *snapc = NULL;
671 	struct ceph_cap_snap *capsnap = NULL;
672 
673 	spin_lock(&ci->i_ceph_lock);
674 	list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) {
675 		doutc(cl, " capsnap %p snapc %p has %d dirty pages\n",
676 		      capsnap, capsnap->context, capsnap->dirty_pages);
677 		if (!capsnap->dirty_pages)
678 			continue;
679 
680 		/* get i_size, truncate_{seq,size} for page_snapc? */
681 		if (snapc && capsnap->context != page_snapc)
682 			continue;
683 
684 		if (ctl) {
685 			if (capsnap->writing) {
686 				ctl->i_size = i_size_read(inode);
687 				ctl->size_stable = false;
688 			} else {
689 				ctl->i_size = capsnap->size;
690 				ctl->size_stable = true;
691 			}
692 			ctl->truncate_size = capsnap->truncate_size;
693 			ctl->truncate_seq = capsnap->truncate_seq;
694 			ctl->head_snapc = false;
695 		}
696 
697 		if (snapc)
698 			break;
699 
700 		snapc = ceph_get_snap_context(capsnap->context);
701 		if (!page_snapc ||
702 		    page_snapc == snapc ||
703 		    page_snapc->seq > snapc->seq)
704 			break;
705 	}
706 	if (!snapc && ci->i_wrbuffer_ref_head) {
707 		snapc = ceph_get_snap_context(ci->i_head_snapc);
708 		doutc(cl, " head snapc %p has %d dirty pages\n", snapc,
709 		      ci->i_wrbuffer_ref_head);
710 		if (ctl) {
711 			ctl->i_size = i_size_read(inode);
712 			ctl->truncate_size = ci->i_truncate_size;
713 			ctl->truncate_seq = ci->i_truncate_seq;
714 			ctl->size_stable = false;
715 			ctl->head_snapc = true;
716 		}
717 	}
718 	spin_unlock(&ci->i_ceph_lock);
719 	return snapc;
720 }
721 
722 static u64 get_writepages_data_length(struct inode *inode,
723 				      struct folio *folio, u64 start)
724 {
725 	struct ceph_inode_info *ci = ceph_inode(inode);
726 	struct ceph_snap_context *snapc;
727 	struct ceph_cap_snap *capsnap = NULL;
728 	u64 end = i_size_read(inode);
729 	u64 ret;
730 
731 	snapc = ceph_folio_snap_context(ceph_fscrypt_pagecache_folio(folio));
732 	if (snapc != ci->i_head_snapc) {
733 		bool found = false;
734 		spin_lock(&ci->i_ceph_lock);
735 		list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) {
736 			if (capsnap->context == snapc) {
737 				if (!capsnap->writing)
738 					end = capsnap->size;
739 				found = true;
740 				break;
741 			}
742 		}
743 		spin_unlock(&ci->i_ceph_lock);
744 		WARN_ON(!found);
745 	}
746 	if (end > ceph_fscrypt_folio_offset(folio) + folio_size(folio))
747 		end = ceph_fscrypt_folio_offset(folio) + folio_size(folio);
748 	ret = end > start ? end - start : 0;
749 	if (ret && fscrypt_is_bounce_folio(folio))
750 		ret = round_up(ret, CEPH_FSCRYPT_BLOCK_SIZE);
751 	return ret;
752 }
753 
754 /*
755  * Write a folio, but leave it locked.
756  *
757  * If we get a write error, mark the mapping for error, but still adjust the
758  * dirty page accounting (i.e., folio is no longer dirty).
759  */
760 static int write_folio_nounlock(struct folio *folio,
761 		struct writeback_control *wbc)
762 {
763 	struct page *page;
764 	struct inode *inode = folio->mapping->host;
765 	struct ceph_inode_info *ci = ceph_inode(inode);
766 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
767 	struct ceph_client *cl = fsc->client;
768 	struct ceph_snap_context *snapc, *oldest;
769 	loff_t page_off = folio_pos(folio);
770 	int err;
771 	loff_t len = folio_size(folio);
772 	loff_t wlen;
773 	struct ceph_writeback_ctl ceph_wbc;
774 	struct ceph_osd_client *osdc = &fsc->client->osdc;
775 	struct ceph_osd_request *req;
776 	bool caching = ceph_is_cache_enabled(inode);
777 	struct page *bounce_page = NULL;
778 
779 	doutc(cl, "%llx.%llx folio %p idx %lu\n", ceph_vinop(inode), folio,
780 	      folio->index);
781 
782 	if (ceph_inode_is_shutdown(inode))
783 		return -EIO;
784 
785 	/* verify this is a writeable snap context */
786 	snapc = ceph_folio_snap_context(folio);
787 	if (!snapc) {
788 		doutc(cl, "%llx.%llx folio %p not dirty?\n", ceph_vinop(inode),
789 		      folio);
790 		return 0;
791 	}
792 	oldest = get_oldest_context(inode, &ceph_wbc, snapc);
793 	if (snapc->seq > oldest->seq) {
794 		doutc(cl, "%llx.%llx folio %p snapc %p not writeable - noop\n",
795 		      ceph_vinop(inode), folio, snapc);
796 		/* we should only noop if called by kswapd */
797 		WARN_ON(!(current->flags & PF_MEMALLOC));
798 		ceph_put_snap_context(oldest);
799 		folio_redirty_for_writepage(wbc, folio);
800 		return 0;
801 	}
802 	ceph_put_snap_context(oldest);
803 
804 	/* is this a partial page at end of file? */
805 	if (page_off >= ceph_wbc.i_size) {
806 		doutc(cl, "%llx.%llx folio at %lu beyond eof %llu\n",
807 		      ceph_vinop(inode), folio->index, ceph_wbc.i_size);
808 		folio_invalidate(folio, 0, folio_size(folio));
809 		return 0;
810 	}
811 
812 	if (ceph_wbc.i_size < page_off + len)
813 		len = ceph_wbc.i_size - page_off;
814 
815 	wlen = IS_ENCRYPTED(inode) ? round_up(len, CEPH_FSCRYPT_BLOCK_SIZE) : len;
816 	doutc(cl, "%llx.%llx folio %p index %lu on %llu~%llu snapc %p seq %lld\n",
817 	      ceph_vinop(inode), folio, folio->index, page_off, wlen, snapc,
818 	      snapc->seq);
819 
820 	/* wait for a cache write left pending by a previously failed attempt */
821 	ceph_folio_wait_fscache(folio);
822 
823 	if (atomic_long_inc_return(&fsc->writeback_count) >
824 	    CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb))
825 		fsc->write_congested = true;
826 
827 	req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode),
828 				    page_off, &wlen, 0, 1, CEPH_OSD_OP_WRITE,
829 				    CEPH_OSD_FLAG_WRITE, snapc,
830 				    ceph_wbc.truncate_seq,
831 				    ceph_wbc.truncate_size, true);
832 	if (IS_ERR(req)) {
833 		folio_redirty_for_writepage(wbc, folio);
834 		if (atomic_long_dec_return(&fsc->writeback_count) <
835 				CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb))
836 			fsc->write_congested = false;
837 		return PTR_ERR(req);
838 	}
839 
840 	if (wlen < len)
841 		len = wlen;
842 
843 	folio_start_writeback(folio);
844 	if (caching)
845 		ceph_folio_start_fscache(folio);
846 	ceph_fscache_write_to_cache(inode, page_off, len, caching);
847 
848 	if (IS_ENCRYPTED(inode)) {
849 		bounce_page = fscrypt_encrypt_pagecache_blocks(folio,
850 						    CEPH_FSCRYPT_BLOCK_SIZE, 0,
851 						    GFP_NOFS);
852 		if (IS_ERR(bounce_page)) {
853 			folio_redirty_for_writepage(wbc, folio);
854 			folio_end_writeback(folio);
855 			ceph_osdc_put_request(req);
856 			if (atomic_long_dec_return(&fsc->writeback_count) <
857 					CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb))
858 				fsc->write_congested = false;
859 			return PTR_ERR(bounce_page);
860 		}
861 	}
862 
863 	/* it may be a short write due to an object boundary */
864 	WARN_ON_ONCE(len > folio_size(folio));
865 	page = bounce_page ? bounce_page : &folio->page;
866 	osd_req_op_extent_osd_data_pages(req, 0, &page, wlen, 0, false, false);
867 	doutc(cl, "%llx.%llx %llu~%llu (%llu bytes, %sencrypted)\n",
868 	      ceph_vinop(inode), page_off, len, wlen,
869 	      IS_ENCRYPTED(inode) ? "" : "not ");
870 
871 	req->r_mtime = inode_get_mtime(inode);
872 	ceph_osdc_start_request(osdc, req);
873 	err = ceph_osdc_wait_request(osdc, req);
874 
875 	ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency,
876 				  req->r_end_latency, len, err);
877 	if (err >= 0 && len > 0)
878 		ceph_subvolume_metrics_record_io(fsc->mdsc, ci, true, len,
879 						 req->r_start_latency,
880 						 req->r_end_latency);
881 	fscrypt_free_bounce_page(bounce_page);
882 	ceph_osdc_put_request(req);
883 	if (err == 0)
884 		err = len;
885 
886 	if (err < 0) {
887 		struct writeback_control tmp_wbc;
888 		if (!wbc)
889 			wbc = &tmp_wbc;
890 		if (err == -ERESTARTSYS) {
891 			/* killed by SIGKILL */
892 			doutc(cl, "%llx.%llx interrupted page %p\n",
893 			      ceph_vinop(inode), folio);
894 			folio_redirty_for_writepage(wbc, folio);
895 			folio_end_writeback(folio);
896 			if (atomic_long_dec_return(&fsc->writeback_count) <
897 					CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb))
898 				fsc->write_congested = false;
899 			return err;
900 		}
901 		if (err == -EBLOCKLISTED)
902 			fsc->blocklisted = true;
903 		doutc(cl, "%llx.%llx setting mapping error %d %p\n",
904 		      ceph_vinop(inode), err, folio);
905 		mapping_set_error(&inode->i_data, err);
906 		wbc->pages_skipped++;
907 	} else {
908 		doutc(cl, "%llx.%llx cleaned page %p\n",
909 		      ceph_vinop(inode), folio);
910 		err = 0;  /* vfs expects us to return 0 */
911 	}
912 	oldest = folio_detach_private(folio);
913 	WARN_ON_ONCE(oldest != snapc);
914 	folio_end_writeback(folio);
915 	ceph_put_wrbuffer_cap_refs(ci, 1, snapc);
916 	ceph_put_snap_context(snapc);  /* page's reference */
917 
918 	if (atomic_long_dec_return(&fsc->writeback_count) <
919 	    CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb))
920 		fsc->write_congested = false;
921 
922 	return err;
923 }
924 
925 /*
926  * async writeback completion handler.
927  *
928  * If we get an error, set the mapping error bit, but not the individual
929  * page error bits.
930  */
931 static void writepages_finish(struct ceph_osd_request *req)
932 {
933 	struct inode *inode = req->r_inode;
934 	struct ceph_inode_info *ci = ceph_inode(inode);
935 	struct ceph_client *cl = ceph_inode_to_client(inode);
936 	struct ceph_osd_data *osd_data;
937 	int num_pages, total_pages = 0;
938 	int i, j;
939 	int rc = req->r_result;
940 	struct ceph_snap_context *snapc = req->r_snapc;
941 	struct address_space *mapping = inode->i_mapping;
942 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
943 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
944 	unsigned int len = 0;
945 	bool remove_page;
946 
947 	doutc(cl, "%llx.%llx rc %d\n", ceph_vinop(inode), rc);
948 	if (rc < 0) {
949 		mapping_set_error(mapping, rc);
950 		ceph_set_error_write(ci);
951 		if (rc == -EBLOCKLISTED)
952 			fsc->blocklisted = true;
953 	} else {
954 		ceph_clear_error_write(ci);
955 	}
956 
957 	/*
958 	 * We lost the cache cap, need to truncate the page before
959 	 * it is unlocked, otherwise we'd truncate it later in the
960 	 * page truncation thread, possibly losing some data that
961 	 * raced its way in
962 	 */
963 	remove_page = !(ceph_caps_issued(ci) &
964 			(CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO));
965 
966 	/* clean all pages */
967 	for (i = 0; i < req->r_num_ops; i++) {
968 		if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) {
969 			pr_warn_client(cl,
970 				"%llx.%llx incorrect op %d req %p index %d tid %llu\n",
971 				ceph_vinop(inode), req->r_ops[i].op, req, i,
972 				req->r_tid);
973 			break;
974 		}
975 
976 		osd_data = osd_req_op_extent_osd_data(req, i);
977 		BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES);
978 		len += osd_data->length;
979 		num_pages = calc_pages_for((u64)osd_data->alignment,
980 					   (u64)osd_data->length);
981 		total_pages += num_pages;
982 		for (j = 0; j < num_pages; j++) {
983 			struct folio *folio = page_folio(osd_data->pages[j]);
984 
985 			if (fscrypt_is_bounce_folio(folio)) {
986 				folio = fscrypt_pagecache_folio(folio);
987 				fscrypt_free_bounce_page(osd_data->pages[j]);
988 				osd_data->pages[j] = &folio->page;
989 			}
990 			WARN_ON(!folio_test_uptodate(folio));
991 
992 			if (atomic_long_dec_return(&fsc->writeback_count) <
993 			     CONGESTION_OFF_THRESH(
994 					fsc->mount_options->congestion_kb))
995 				fsc->write_congested = false;
996 
997 			ceph_put_snap_context(folio_detach_private(folio));
998 			folio_end_writeback(folio);
999 
1000 			if (atomic64_dec_return(&mdsc->dirty_folios) <= 0) {
1001 				wake_up_all(&mdsc->flush_end_wq);
1002 				WARN_ON(atomic64_read(&mdsc->dirty_folios) < 0);
1003 			}
1004 
1005 			doutc(cl, "unlocking %p\n", folio);
1006 
1007 			if (remove_page)
1008 				generic_error_remove_folio(inode->i_mapping,
1009 							  folio);
1010 
1011 			folio_unlock(folio);
1012 		}
1013 		doutc(cl, "%llx.%llx wrote %llu bytes cleaned %d pages\n",
1014 		      ceph_vinop(inode), osd_data->length,
1015 		      rc >= 0 ? num_pages : 0);
1016 
1017 		release_pages(osd_data->pages, num_pages);
1018 	}
1019 
1020 	ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency,
1021 				  req->r_end_latency, len, rc);
1022 
1023 	if (rc >= 0 && len > 0)
1024 		ceph_subvolume_metrics_record_io(mdsc, ci, true, len,
1025 						 req->r_start_latency,
1026 						 req->r_end_latency);
1027 
1028 	ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc);
1029 
1030 	osd_data = osd_req_op_extent_osd_data(req, 0);
1031 	if (osd_data->pages_from_pool)
1032 		mempool_free(osd_data->pages, ceph_wb_pagevec_pool);
1033 	else
1034 		kfree(osd_data->pages);
1035 	ceph_osdc_put_request(req);
1036 	ceph_dec_osd_stopping_blocker(fsc->mdsc);
1037 }
1038 
1039 static inline
1040 bool is_forced_umount(struct address_space *mapping)
1041 {
1042 	struct inode *inode = mapping->host;
1043 	struct ceph_inode_info *ci = ceph_inode(inode);
1044 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1045 	struct ceph_client *cl = fsc->client;
1046 
1047 	if (ceph_inode_is_shutdown(inode)) {
1048 		if (ci->i_wrbuffer_ref > 0) {
1049 			pr_warn_ratelimited_client(cl,
1050 				"%llx.%llx %lld forced umount\n",
1051 				ceph_vinop(inode), ceph_ino(inode));
1052 		}
1053 		mapping_set_error(mapping, -EIO);
1054 		return true;
1055 	}
1056 
1057 	return false;
1058 }
1059 
1060 static inline
1061 unsigned int ceph_define_write_size(struct address_space *mapping)
1062 {
1063 	struct inode *inode = mapping->host;
1064 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1065 	struct ceph_inode_info *ci = ceph_inode(inode);
1066 	unsigned int wsize = ci->i_layout.stripe_unit;
1067 
1068 	if (fsc->mount_options->wsize < wsize)
1069 		wsize = fsc->mount_options->wsize;
1070 
1071 	return wsize;
1072 }
1073 
1074 static inline
1075 void ceph_folio_batch_init(struct ceph_writeback_ctl *ceph_wbc)
1076 {
1077 	folio_batch_init(&ceph_wbc->fbatch);
1078 	ceph_wbc->processed_in_fbatch = 0;
1079 }
1080 
1081 static inline
1082 void ceph_folio_batch_reinit(struct ceph_writeback_ctl *ceph_wbc)
1083 {
1084 	folio_batch_release(&ceph_wbc->fbatch);
1085 	ceph_folio_batch_init(ceph_wbc);
1086 }
1087 
1088 static inline
1089 void ceph_init_writeback_ctl(struct address_space *mapping,
1090 			     struct writeback_control *wbc,
1091 			     struct ceph_writeback_ctl *ceph_wbc)
1092 {
1093 	ceph_wbc->snapc = NULL;
1094 	ceph_wbc->last_snapc = NULL;
1095 
1096 	ceph_wbc->strip_unit_end = 0;
1097 	ceph_wbc->wsize = ceph_define_write_size(mapping);
1098 
1099 	ceph_wbc->nr_folios = 0;
1100 	ceph_wbc->max_pages = 0;
1101 	ceph_wbc->locked_pages = 0;
1102 
1103 	ceph_wbc->done = false;
1104 	ceph_wbc->should_loop = false;
1105 	ceph_wbc->range_whole = false;
1106 
1107 	ceph_wbc->start_index = wbc->range_cyclic ? mapping->writeback_index : 0;
1108 	ceph_wbc->index = ceph_wbc->start_index;
1109 	ceph_wbc->end = -1;
1110 
1111 	ceph_wbc->tag = wbc_to_tag(wbc);
1112 
1113 	ceph_wbc->op_idx = -1;
1114 	ceph_wbc->num_ops = 0;
1115 	ceph_wbc->offset = 0;
1116 	ceph_wbc->len = 0;
1117 	ceph_wbc->from_pool = false;
1118 
1119 	ceph_folio_batch_init(ceph_wbc);
1120 
1121 	ceph_wbc->pages = NULL;
1122 	ceph_wbc->data_pages = NULL;
1123 }
1124 
1125 static inline
1126 int ceph_define_writeback_range(struct address_space *mapping,
1127 				struct writeback_control *wbc,
1128 				struct ceph_writeback_ctl *ceph_wbc)
1129 {
1130 	struct inode *inode = mapping->host;
1131 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1132 	struct ceph_client *cl = fsc->client;
1133 
1134 	/* find oldest snap context with dirty data */
1135 	ceph_wbc->snapc = get_oldest_context(inode, ceph_wbc, NULL);
1136 	if (!ceph_wbc->snapc) {
1137 		/* hmm, why does writepages get called when there
1138 		   is no dirty data? */
1139 		doutc(cl, " no snap context with dirty data?\n");
1140 		return -ENODATA;
1141 	}
1142 
1143 	doutc(cl, " oldest snapc is %p seq %lld (%d snaps)\n",
1144 	      ceph_wbc->snapc, ceph_wbc->snapc->seq,
1145 	      ceph_wbc->snapc->num_snaps);
1146 
1147 	ceph_wbc->should_loop = false;
1148 
1149 	if (ceph_wbc->head_snapc && ceph_wbc->snapc != ceph_wbc->last_snapc) {
1150 		/* where to start/end? */
1151 		if (wbc->range_cyclic) {
1152 			ceph_wbc->index = ceph_wbc->start_index;
1153 			ceph_wbc->end = -1;
1154 			if (ceph_wbc->index > 0)
1155 				ceph_wbc->should_loop = true;
1156 			doutc(cl, " cyclic, start at %lu\n", ceph_wbc->index);
1157 		} else {
1158 			ceph_wbc->index = wbc->range_start >> PAGE_SHIFT;
1159 			ceph_wbc->end = wbc->range_end >> PAGE_SHIFT;
1160 			if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1161 				ceph_wbc->range_whole = true;
1162 			doutc(cl, " not cyclic, %lu to %lu\n",
1163 				ceph_wbc->index, ceph_wbc->end);
1164 		}
1165 	} else if (!ceph_wbc->head_snapc) {
1166 		/* Do not respect wbc->range_{start,end}. Dirty pages
1167 		 * in that range can be associated with newer snapc.
1168 		 * They are not writeable until we write all dirty pages
1169 		 * associated with 'snapc' get written */
1170 		if (ceph_wbc->index > 0)
1171 			ceph_wbc->should_loop = true;
1172 		doutc(cl, " non-head snapc, range whole\n");
1173 	}
1174 
1175 	ceph_put_snap_context(ceph_wbc->last_snapc);
1176 	ceph_wbc->last_snapc = ceph_wbc->snapc;
1177 
1178 	return 0;
1179 }
1180 
1181 static inline
1182 bool has_writeback_done(struct ceph_writeback_ctl *ceph_wbc)
1183 {
1184 	return ceph_wbc->done && ceph_wbc->index > ceph_wbc->end;
1185 }
1186 
1187 static inline
1188 bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc,
1189 				unsigned index)
1190 {
1191 	return index < ceph_wbc->nr_folios &&
1192 		ceph_wbc->locked_pages < ceph_wbc->max_pages;
1193 }
1194 
1195 static
1196 int ceph_check_folio_before_write(struct address_space *mapping,
1197 				  struct writeback_control *wbc,
1198 				  struct ceph_writeback_ctl *ceph_wbc,
1199 				  struct folio *folio)
1200 {
1201 	struct inode *inode = mapping->host;
1202 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1203 	struct ceph_client *cl = fsc->client;
1204 	struct ceph_snap_context *pgsnapc;
1205 
1206 	/* only dirty folios, or our accounting breaks */
1207 	if (unlikely(!folio_test_dirty(folio) || folio->mapping != mapping)) {
1208 		doutc(cl, "!dirty or !mapping %p\n", folio);
1209 		return -ENODATA;
1210 	}
1211 
1212 	/* only if matching snap context */
1213 	pgsnapc = ceph_folio_snap_context(folio);
1214 	if (pgsnapc != ceph_wbc->snapc) {
1215 		doutc(cl, "folio snapc %p %lld != oldest %p %lld\n",
1216 		      pgsnapc, pgsnapc->seq,
1217 		      ceph_wbc->snapc, ceph_wbc->snapc->seq);
1218 
1219 		if (!ceph_wbc->should_loop && !ceph_wbc->head_snapc &&
1220 		    wbc->sync_mode != WB_SYNC_NONE)
1221 			ceph_wbc->should_loop = true;
1222 
1223 		return -ENODATA;
1224 	}
1225 
1226 	if (folio_pos(folio) >= ceph_wbc->i_size) {
1227 		doutc(cl, "folio at %lu beyond eof %llu\n",
1228 		      folio->index, ceph_wbc->i_size);
1229 
1230 		if ((ceph_wbc->size_stable ||
1231 		    folio_pos(folio) >= i_size_read(inode)) &&
1232 		    folio_clear_dirty_for_io(folio))
1233 			folio_invalidate(folio, 0, folio_size(folio));
1234 
1235 		return -ENODATA;
1236 	}
1237 
1238 	if (ceph_wbc->strip_unit_end &&
1239 	    (folio->index > ceph_wbc->strip_unit_end)) {
1240 		doutc(cl, "end of strip unit %p\n", folio);
1241 		return -E2BIG;
1242 	}
1243 
1244 	return 0;
1245 }
1246 
1247 static inline
1248 void __ceph_allocate_page_array(struct ceph_writeback_ctl *ceph_wbc,
1249 				unsigned int max_pages)
1250 {
1251 	ceph_wbc->pages = kmalloc_objs(*ceph_wbc->pages, max_pages, GFP_NOFS);
1252 	if (!ceph_wbc->pages) {
1253 		ceph_wbc->from_pool = true;
1254 		ceph_wbc->pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS);
1255 		BUG_ON(!ceph_wbc->pages);
1256 	}
1257 }
1258 
1259 static inline
1260 void ceph_allocate_page_array(struct address_space *mapping,
1261 			      struct ceph_writeback_ctl *ceph_wbc,
1262 			      struct folio *folio)
1263 {
1264 	struct inode *inode = mapping->host;
1265 	struct ceph_inode_info *ci = ceph_inode(inode);
1266 	u64 objnum;
1267 	u64 objoff;
1268 	u32 xlen;
1269 
1270 	/* prepare async write request */
1271 	ceph_wbc->offset = (u64)folio_pos(folio);
1272 	ceph_calc_file_object_mapping(&ci->i_layout,
1273 					ceph_wbc->offset, ceph_wbc->wsize,
1274 					&objnum, &objoff, &xlen);
1275 
1276 	ceph_wbc->num_ops = 1;
1277 	ceph_wbc->strip_unit_end = folio->index + ((xlen - 1) >> PAGE_SHIFT);
1278 
1279 	BUG_ON(ceph_wbc->pages);
1280 	ceph_wbc->max_pages = calc_pages_for(0, (u64)xlen);
1281 	__ceph_allocate_page_array(ceph_wbc, ceph_wbc->max_pages);
1282 
1283 	ceph_wbc->len = 0;
1284 }
1285 
1286 static inline
1287 bool is_folio_index_contiguous(const struct ceph_writeback_ctl *ceph_wbc,
1288 			      const struct folio *folio)
1289 {
1290 	return folio->index == (ceph_wbc->offset + ceph_wbc->len) >> PAGE_SHIFT;
1291 }
1292 
1293 static inline
1294 bool is_num_ops_too_big(struct ceph_writeback_ctl *ceph_wbc)
1295 {
1296 	return ceph_wbc->num_ops >=
1297 		(ceph_wbc->from_pool ?  CEPH_OSD_SLAB_OPS : CEPH_OSD_MAX_OPS);
1298 }
1299 
1300 static inline
1301 bool is_write_congestion_happened(struct ceph_fs_client *fsc)
1302 {
1303 	return atomic_long_inc_return(&fsc->writeback_count) >
1304 		CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb);
1305 }
1306 
1307 static inline int move_dirty_folio_in_page_array(struct address_space *mapping,
1308 		struct writeback_control *wbc,
1309 		struct ceph_writeback_ctl *ceph_wbc, struct folio *folio)
1310 {
1311 	struct inode *inode = mapping->host;
1312 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1313 	struct ceph_client *cl = fsc->client;
1314 	struct page **pages = ceph_wbc->pages;
1315 	unsigned int index = ceph_wbc->locked_pages;
1316 	gfp_t gfp_flags = ceph_wbc->locked_pages ? GFP_NOWAIT : GFP_NOFS;
1317 
1318 	if (IS_ENCRYPTED(inode)) {
1319 		pages[index] = fscrypt_encrypt_pagecache_blocks(folio,
1320 								PAGE_SIZE,
1321 								0,
1322 								gfp_flags);
1323 		if (IS_ERR(pages[index])) {
1324 			int err = PTR_ERR(pages[index]);
1325 
1326 			if (err == -EINVAL) {
1327 				pr_err_client(cl, "inode->i_blkbits=%hhu\n",
1328 						inode->i_blkbits);
1329 			}
1330 
1331 			/* better not fail on first page! */
1332 			BUG_ON(ceph_wbc->locked_pages == 0);
1333 
1334 			pages[index] = NULL;
1335 			return err;
1336 		}
1337 	} else {
1338 		pages[index] = &folio->page;
1339 	}
1340 
1341 	ceph_wbc->locked_pages++;
1342 
1343 	return 0;
1344 }
1345 
1346 static
1347 void ceph_process_folio_batch(struct address_space *mapping,
1348 			      struct writeback_control *wbc,
1349 			      struct ceph_writeback_ctl *ceph_wbc)
1350 {
1351 	struct inode *inode = mapping->host;
1352 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1353 	struct ceph_client *cl = fsc->client;
1354 	struct folio *folio = NULL;
1355 	unsigned i;
1356 	int rc;
1357 
1358 	for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) {
1359 		folio = ceph_wbc->fbatch.folios[i];
1360 
1361 		if (!folio)
1362 			continue;
1363 
1364 		doutc(cl, "? %p idx %lu, folio_test_writeback %#x, "
1365 			"folio_test_dirty %#x, folio_test_locked %#x\n",
1366 			folio, folio->index, folio_test_writeback(folio),
1367 			folio_test_dirty(folio),
1368 			folio_test_locked(folio));
1369 
1370 		if (folio_test_writeback(folio) ||
1371 		    folio_test_private_2(folio) /* [DEPRECATED] */) {
1372 			doutc(cl, "waiting on writeback %p\n", folio);
1373 			folio_wait_writeback(folio);
1374 			folio_wait_private_2(folio); /* [DEPRECATED] */
1375 			continue;
1376 		}
1377 
1378 		if (ceph_wbc->locked_pages == 0)
1379 			folio_lock(folio);
1380 		else if (!folio_trylock(folio))
1381 			break;
1382 
1383 		rc = ceph_check_folio_before_write(mapping, wbc,
1384 						   ceph_wbc, folio);
1385 		if (rc == -ENODATA) {
1386 			folio_unlock(folio);
1387 			folio_put(folio);
1388 			ceph_wbc->fbatch.folios[i] = NULL;
1389 			continue;
1390 		} else if (rc == -E2BIG) {
1391 			folio_unlock(folio);
1392 			break;
1393 		}
1394 
1395 		if (!folio_clear_dirty_for_io(folio)) {
1396 			doutc(cl, "%p !folio_clear_dirty_for_io\n", folio);
1397 			folio_unlock(folio);
1398 			folio_put(folio);
1399 			ceph_wbc->fbatch.folios[i] = NULL;
1400 			continue;
1401 		}
1402 
1403 		/*
1404 		 * We have something to write.  If this is
1405 		 * the first locked page this time through,
1406 		 * calculate max possible write size and
1407 		 * allocate a page array
1408 		 */
1409 		if (ceph_wbc->locked_pages == 0) {
1410 			ceph_allocate_page_array(mapping, ceph_wbc, folio);
1411 		} else if (!is_folio_index_contiguous(ceph_wbc, folio)) {
1412 			if (is_num_ops_too_big(ceph_wbc)) {
1413 				folio_redirty_for_writepage(wbc, folio);
1414 				folio_unlock(folio);
1415 				break;
1416 			}
1417 
1418 			ceph_wbc->num_ops++;
1419 			ceph_wbc->offset = (u64)folio_pos(folio);
1420 			ceph_wbc->len = 0;
1421 		}
1422 
1423 		/* note position of first page in fbatch */
1424 		doutc(cl, "%llx.%llx will write folio %p idx %lu\n",
1425 		      ceph_vinop(inode), folio, folio->index);
1426 
1427 		rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc,
1428 				folio);
1429 		if (rc) {
1430 			/* Did we just begin a new contiguous op? Nevermind! */
1431 			if (ceph_wbc->len == 0)
1432 				ceph_wbc->num_ops--;
1433 
1434 			folio_redirty_for_writepage(wbc, folio);
1435 			folio_unlock(folio);
1436 			break;
1437 		}
1438 
1439 		if (is_write_congestion_happened(fsc))
1440 			fsc->write_congested = true;
1441 
1442 		ceph_wbc->fbatch.folios[i] = NULL;
1443 		ceph_wbc->len += folio_size(folio);
1444 	}
1445 
1446 	ceph_wbc->processed_in_fbatch = i;
1447 }
1448 
1449 static inline
1450 void ceph_shift_unused_folios_left(struct folio_batch *fbatch)
1451 {
1452 	unsigned j, n = 0;
1453 
1454 	/* shift unused page to beginning of fbatch */
1455 	for (j = 0; j < folio_batch_count(fbatch); j++) {
1456 		if (!fbatch->folios[j])
1457 			continue;
1458 
1459 		if (n < j) {
1460 			fbatch->folios[n] = fbatch->folios[j];
1461 		}
1462 
1463 		n++;
1464 	}
1465 
1466 	fbatch->nr = n;
1467 }
1468 
1469 static void ceph_undo_wrbuffer_claim(struct inode *inode, struct folio *folio)
1470 {
1471 	struct ceph_snap_context *snapc = folio_detach_private(folio);
1472 
1473 	if (!snapc)
1474 		return;
1475 	ceph_put_wrbuffer_cap_refs(ceph_inode(inode), 1, snapc);
1476 	ceph_put_snap_context(snapc);
1477 }
1478 
1479 static
1480 int ceph_submit_write(struct address_space *mapping,
1481 			struct writeback_control *wbc,
1482 			struct ceph_writeback_ctl *ceph_wbc)
1483 {
1484 	struct inode *inode = mapping->host;
1485 	struct ceph_inode_info *ci = ceph_inode(inode);
1486 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1487 	struct ceph_client *cl = fsc->client;
1488 	struct ceph_vino vino = ceph_vino(inode);
1489 	struct ceph_osd_request *req = NULL;
1490 	struct folio *folio = NULL;
1491 	bool caching = ceph_is_cache_enabled(inode);
1492 	u64 offset;
1493 	u64 len;
1494 	unsigned i;
1495 
1496 new_request:
1497 	offset = ceph_fscrypt_folio_offset(page_folio(ceph_wbc->pages[0]));
1498 	len = ceph_wbc->wsize;
1499 
1500 	req = ceph_osdc_new_request(&fsc->client->osdc,
1501 				    &ci->i_layout, vino,
1502 				    offset, &len, 0, ceph_wbc->num_ops,
1503 				    CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE,
1504 				    ceph_wbc->snapc, ceph_wbc->truncate_seq,
1505 				    ceph_wbc->truncate_size, false);
1506 	if (IS_ERR(req)) {
1507 		req = ceph_osdc_new_request(&fsc->client->osdc,
1508 					    &ci->i_layout, vino,
1509 					    offset, &len, 0,
1510 					    min(ceph_wbc->num_ops,
1511 						CEPH_OSD_SLAB_OPS),
1512 					    CEPH_OSD_OP_WRITE,
1513 					    CEPH_OSD_FLAG_WRITE,
1514 					    ceph_wbc->snapc,
1515 					    ceph_wbc->truncate_seq,
1516 					    ceph_wbc->truncate_size,
1517 					    true);
1518 		BUG_ON(IS_ERR(req));
1519 	}
1520 
1521 	folio = page_folio(ceph_wbc->pages[ceph_wbc->locked_pages - 1]);
1522 	BUG_ON(len < ceph_fscrypt_folio_offset(folio) + folio_size(folio) - offset);
1523 
1524 	if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
1525 		for (i = 0; i < ceph_wbc->locked_pages; i++) {
1526 			folio = page_folio(ceph_wbc->pages[i]);
1527 			if (fscrypt_is_bounce_folio(folio)) {
1528 				folio = fscrypt_pagecache_folio(folio);
1529 				fscrypt_free_bounce_page(ceph_wbc->pages[i]);
1530 			}
1531 
1532 			if (atomic_long_dec_return(&fsc->writeback_count) <
1533 			    CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb))
1534 				fsc->write_congested = false;
1535 
1536 			ceph_undo_wrbuffer_claim(inode, folio);
1537 			folio_redirty_for_writepage(wbc, folio);
1538 			folio_unlock(folio);
1539 			folio_put(folio);
1540 		}
1541 
1542 		if (ceph_wbc->from_pool) {
1543 			mempool_free(ceph_wbc->pages, ceph_wb_pagevec_pool);
1544 			ceph_wbc->from_pool = false;
1545 		} else {
1546 			kfree(ceph_wbc->pages);
1547 		}
1548 		ceph_wbc->pages = NULL;
1549 		ceph_wbc->locked_pages = 0;
1550 
1551 		ceph_osdc_put_request(req);
1552 		return -EIO;
1553 	}
1554 
1555 	req->r_callback = writepages_finish;
1556 	req->r_inode = inode;
1557 
1558 	/* Format the osd request message and submit the write */
1559 	len = 0;
1560 	ceph_wbc->data_pages = ceph_wbc->pages;
1561 	ceph_wbc->op_idx = 0;
1562 	for (i = 0; i < ceph_wbc->locked_pages; i++) {
1563 		u64 cur_offset;
1564 
1565 		folio = ceph_fscrypt_pagecache_folio(page_folio(ceph_wbc->pages[i]));
1566 		cur_offset = folio_pos(folio);
1567 
1568 		/*
1569 		 * Discontinuity in page range? Ceph can handle that by just passing
1570 		 * multiple extents in the write op.
1571 		 */
1572 		if (offset + len != cur_offset) {
1573 			/* If it's full, stop here */
1574 			if (ceph_wbc->op_idx + 1 == req->r_num_ops)
1575 				break;
1576 
1577 			/* Kick off an fscache write with what we have so far. */
1578 			ceph_fscache_write_to_cache(inode, offset, len, caching);
1579 
1580 			/* Start a new extent */
1581 			osd_req_op_extent_dup_last(req, ceph_wbc->op_idx,
1582 						   cur_offset - offset);
1583 
1584 			doutc(cl, "got pages at %llu~%llu\n", offset, len);
1585 
1586 			osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx,
1587 							 ceph_wbc->data_pages,
1588 							 len, 0,
1589 							 ceph_wbc->from_pool,
1590 							 false);
1591 			osd_req_op_extent_update(req, ceph_wbc->op_idx, len);
1592 
1593 			len = 0;
1594 			offset = cur_offset;
1595 			ceph_wbc->data_pages = ceph_wbc->pages + i;
1596 			ceph_wbc->op_idx++;
1597 		}
1598 
1599 		folio_start_writeback(folio);
1600 
1601 		if (caching)
1602 			ceph_folio_start_fscache(folio);
1603 
1604 		len += folio_size(folio);
1605 	}
1606 
1607 	ceph_fscache_write_to_cache(inode, offset, len, caching);
1608 
1609 	if (ceph_wbc->size_stable) {
1610 		len = min(len, ceph_wbc->i_size - offset);
1611 	} else if (i == ceph_wbc->locked_pages) {
1612 		/* writepages_finish() clears writeback pages
1613 		 * according to the data length, so make sure
1614 		 * data length covers all locked pages */
1615 		u64 min_len = len + 1 - folio_size(folio);
1616 		len = get_writepages_data_length(inode,
1617 						 page_folio(ceph_wbc->pages[i - 1]),
1618 						 offset);
1619 		len = max(len, min_len);
1620 	}
1621 
1622 	if (IS_ENCRYPTED(inode))
1623 		len = round_up(len, CEPH_FSCRYPT_BLOCK_SIZE);
1624 
1625 	doutc(cl, "got pages at %llu~%llu\n", offset, len);
1626 
1627 	if (IS_ENCRYPTED(inode) &&
1628 	    ((offset | len) & ~CEPH_FSCRYPT_BLOCK_MASK)) {
1629 		pr_warn_client(cl,
1630 			"bad encrypted write offset=%lld len=%llu\n",
1631 			offset, len);
1632 	}
1633 
1634 	osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx,
1635 					 ceph_wbc->data_pages, len,
1636 					 0, ceph_wbc->from_pool, false);
1637 	osd_req_op_extent_update(req, ceph_wbc->op_idx, len);
1638 
1639 	BUG_ON(ceph_wbc->op_idx + 1 != req->r_num_ops);
1640 
1641 	ceph_wbc->from_pool = false;
1642 	if (i < ceph_wbc->locked_pages) {
1643 		BUG_ON(ceph_wbc->num_ops <= req->r_num_ops);
1644 		ceph_wbc->num_ops -= req->r_num_ops;
1645 		ceph_wbc->locked_pages -= i;
1646 
1647 		/* allocate new pages array for next request */
1648 		ceph_wbc->data_pages = ceph_wbc->pages;
1649 		__ceph_allocate_page_array(ceph_wbc, ceph_wbc->locked_pages);
1650 		memcpy(ceph_wbc->pages, ceph_wbc->data_pages + i,
1651 			ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages));
1652 		memset(ceph_wbc->data_pages + i, 0,
1653 			ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages));
1654 	} else {
1655 		BUG_ON(ceph_wbc->num_ops != req->r_num_ops);
1656 		/* request message now owns the pages array */
1657 		ceph_wbc->pages = NULL;
1658 	}
1659 
1660 	req->r_mtime = inode_get_mtime(inode);
1661 	ceph_osdc_start_request(&fsc->client->osdc, req);
1662 	req = NULL;
1663 
1664 	wbc->nr_to_write -= i;
1665 	if (ceph_wbc->pages)
1666 		goto new_request;
1667 
1668 	return 0;
1669 }
1670 
1671 static
1672 void ceph_wait_until_current_writes_complete(struct address_space *mapping,
1673 					     struct writeback_control *wbc,
1674 					     struct ceph_writeback_ctl *ceph_wbc)
1675 {
1676 	struct folio *folio;
1677 	unsigned i, nr;
1678 
1679 	if (wbc->sync_mode != WB_SYNC_NONE &&
1680 	    ceph_wbc->start_index == 0 && /* all dirty pages were checked */
1681 	    !ceph_wbc->head_snapc) {
1682 		ceph_wbc->index = 0;
1683 
1684 		while ((ceph_wbc->index <= ceph_wbc->end) &&
1685 			(nr = filemap_get_folios_tag(mapping,
1686 						     &ceph_wbc->index,
1687 						     (pgoff_t)-1,
1688 						     PAGECACHE_TAG_WRITEBACK,
1689 						     &ceph_wbc->fbatch))) {
1690 			for (i = 0; i < nr; i++) {
1691 				folio = ceph_wbc->fbatch.folios[i];
1692 				if (ceph_folio_snap_context(folio) != ceph_wbc->snapc)
1693 					continue;
1694 				folio_wait_writeback(folio);
1695 			}
1696 
1697 			folio_batch_release(&ceph_wbc->fbatch);
1698 			cond_resched();
1699 		}
1700 	}
1701 }
1702 
1703 /*
1704  * initiate async writeback
1705  */
1706 static int ceph_writepages_start(struct address_space *mapping,
1707 				 struct writeback_control *wbc)
1708 {
1709 	struct inode *inode = mapping->host;
1710 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
1711 	struct ceph_client *cl = fsc->client;
1712 	struct ceph_writeback_ctl ceph_wbc;
1713 	int rc = 0;
1714 
1715 	if (wbc->sync_mode == WB_SYNC_NONE && fsc->write_congested)
1716 		return 0;
1717 
1718 	doutc(cl, "%llx.%llx (mode=%s)\n", ceph_vinop(inode),
1719 	      wbc->sync_mode == WB_SYNC_NONE ? "NONE" :
1720 	      (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD"));
1721 
1722 	if (is_forced_umount(mapping)) {
1723 		/* we're in a forced umount, don't write! */
1724 		return -EIO;
1725 	}
1726 
1727 	ceph_init_writeback_ctl(mapping, wbc, &ceph_wbc);
1728 
1729 	if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
1730 		rc = -EIO;
1731 		goto out;
1732 	}
1733 
1734 retry:
1735 	rc = ceph_define_writeback_range(mapping, wbc, &ceph_wbc);
1736 	if (rc == -ENODATA) {
1737 		/* hmm, why does writepages get called when there
1738 		   is no dirty data? */
1739 		rc = 0;
1740 		goto dec_osd_stopping_blocker;
1741 	}
1742 
1743 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1744 		tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end);
1745 
1746 	while (!has_writeback_done(&ceph_wbc)) {
1747 		BUG_ON(ceph_wbc.locked_pages);
1748 		BUG_ON(ceph_wbc.pages);
1749 
1750 		ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT;
1751 
1752 get_more_pages:
1753 		ceph_folio_batch_reinit(&ceph_wbc);
1754 
1755 		ceph_wbc.nr_folios = filemap_get_folios_tag(mapping,
1756 							    &ceph_wbc.index,
1757 							    ceph_wbc.end,
1758 							    ceph_wbc.tag,
1759 							    &ceph_wbc.fbatch);
1760 		doutc(cl, "pagevec_lookup_range_tag for tag %#x got %d\n",
1761 			ceph_wbc.tag, ceph_wbc.nr_folios);
1762 
1763 		if (!ceph_wbc.nr_folios && !ceph_wbc.locked_pages)
1764 			break;
1765 
1766 process_folio_batch:
1767 		ceph_process_folio_batch(mapping, wbc, &ceph_wbc);
1768 		ceph_shift_unused_folios_left(&ceph_wbc.fbatch);
1769 
1770 		/* did we get anything? */
1771 		if (!ceph_wbc.locked_pages)
1772 			goto release_folios;
1773 
1774 		if (ceph_wbc.processed_in_fbatch) {
1775 			if (folio_batch_count(&ceph_wbc.fbatch) == 0 &&
1776 			    ceph_wbc.locked_pages < ceph_wbc.max_pages) {
1777 				doutc(cl, "reached end fbatch, trying for more\n");
1778 				goto get_more_pages;
1779 			}
1780 		}
1781 
1782 		rc = ceph_submit_write(mapping, wbc, &ceph_wbc);
1783 		if (rc) {
1784 			/* the OSD client is being torn down, don't retry */
1785 			folio_batch_release(&ceph_wbc.fbatch);
1786 			goto dec_osd_stopping_blocker;
1787 		}
1788 
1789 		ceph_wbc.locked_pages = 0;
1790 		ceph_wbc.strip_unit_end = 0;
1791 
1792 		if (folio_batch_count(&ceph_wbc.fbatch) > 0) {
1793 			ceph_wbc.nr_folios =
1794 				folio_batch_count(&ceph_wbc.fbatch);
1795 			goto process_folio_batch;
1796 		}
1797 
1798 		/*
1799 		 * We stop writing back only if we are not doing
1800 		 * integrity sync. In case of integrity sync we have to
1801 		 * keep going until we have written all the pages
1802 		 * we tagged for writeback prior to entering this loop.
1803 		 */
1804 		if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE)
1805 			ceph_wbc.done = true;
1806 
1807 release_folios:
1808 		doutc(cl, "folio_batch release on %d folios (%p)\n",
1809 		      (int)ceph_wbc.fbatch.nr,
1810 		      ceph_wbc.fbatch.nr ? ceph_wbc.fbatch.folios[0] : NULL);
1811 		folio_batch_release(&ceph_wbc.fbatch);
1812 	}
1813 
1814 	if (ceph_wbc.should_loop && !ceph_wbc.done) {
1815 		/* more to do; loop back to beginning of file */
1816 		doutc(cl, "looping back to beginning of file\n");
1817 		/* OK even when start_index == 0 */
1818 		ceph_wbc.end = ceph_wbc.start_index - 1;
1819 
1820 		/* to write dirty pages associated with next snapc,
1821 		 * we need to wait until current writes complete */
1822 		ceph_wait_until_current_writes_complete(mapping, wbc, &ceph_wbc);
1823 
1824 		ceph_wbc.start_index = 0;
1825 		ceph_wbc.index = 0;
1826 		goto retry;
1827 	}
1828 
1829 	if (wbc->range_cyclic || (ceph_wbc.range_whole && wbc->nr_to_write > 0))
1830 		mapping->writeback_index = ceph_wbc.index;
1831 
1832 dec_osd_stopping_blocker:
1833 	ceph_dec_osd_stopping_blocker(fsc->mdsc);
1834 
1835 out:
1836 	ceph_put_snap_context(ceph_wbc.last_snapc);
1837 	doutc(cl, "%llx.%llx dend - startone, rc = %d\n", ceph_vinop(inode),
1838 	      rc);
1839 
1840 	return rc;
1841 }
1842 
1843 /*
1844  * See if a given @snapc is either writeable, or already written.
1845  */
1846 static int context_is_writeable_or_written(struct inode *inode,
1847 					   struct ceph_snap_context *snapc)
1848 {
1849 	struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL);
1850 	int ret = !oldest || snapc->seq <= oldest->seq;
1851 
1852 	ceph_put_snap_context(oldest);
1853 	return ret;
1854 }
1855 
1856 /**
1857  * ceph_find_incompatible - find an incompatible context and return it
1858  * @folio: folio being dirtied
1859  *
1860  * We are only allowed to write into/dirty a folio if the folio is
1861  * clean, or already dirty within the same snap context. Returns a
1862  * conflicting context if there is one, NULL if there isn't, or a
1863  * negative error code on other errors.
1864  *
1865  * Must be called with folio lock held.
1866  */
1867 static struct ceph_snap_context *
1868 ceph_find_incompatible(struct folio *folio)
1869 {
1870 	struct inode *inode = folio->mapping->host;
1871 	struct ceph_client *cl = ceph_inode_to_client(inode);
1872 	struct ceph_inode_info *ci = ceph_inode(inode);
1873 
1874 	if (ceph_inode_is_shutdown(inode)) {
1875 		doutc(cl, " %llx.%llx folio %p is shutdown\n",
1876 		      ceph_vinop(inode), folio);
1877 		return ERR_PTR(-ESTALE);
1878 	}
1879 
1880 	for (;;) {
1881 		struct ceph_snap_context *snapc, *oldest;
1882 
1883 		folio_wait_writeback(folio);
1884 
1885 		snapc = ceph_folio_snap_context(folio);
1886 		if (!snapc || snapc == ci->i_head_snapc)
1887 			break;
1888 
1889 		/*
1890 		 * this folio is already dirty in another (older) snap
1891 		 * context!  is it writeable now?
1892 		 */
1893 		oldest = get_oldest_context(inode, NULL, NULL);
1894 		if (snapc->seq > oldest->seq) {
1895 			/* not writeable -- return it for the caller to deal with */
1896 			ceph_put_snap_context(oldest);
1897 			doutc(cl, " %llx.%llx folio %p snapc %p not current or oldest\n",
1898 			      ceph_vinop(inode), folio, snapc);
1899 			return ceph_get_snap_context(snapc);
1900 		}
1901 		ceph_put_snap_context(oldest);
1902 
1903 		/* yay, writeable, do it now (without dropping folio lock) */
1904 		doutc(cl, " %llx.%llx folio %p snapc %p not current, but oldest\n",
1905 		      ceph_vinop(inode), folio, snapc);
1906 		if (folio_clear_dirty_for_io(folio)) {
1907 			int r = write_folio_nounlock(folio, NULL);
1908 			if (r < 0)
1909 				return ERR_PTR(r);
1910 		}
1911 	}
1912 	return NULL;
1913 }
1914 
1915 static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len,
1916 					struct folio **foliop, void **_fsdata)
1917 {
1918 	struct inode *inode = file_inode(file);
1919 	struct ceph_inode_info *ci = ceph_inode(inode);
1920 	struct ceph_snap_context *snapc;
1921 
1922 	snapc = ceph_find_incompatible(*foliop);
1923 	if (snapc) {
1924 		int r;
1925 
1926 		folio_unlock(*foliop);
1927 		folio_put(*foliop);
1928 		*foliop = NULL;
1929 		if (IS_ERR(snapc))
1930 			return PTR_ERR(snapc);
1931 
1932 		ceph_queue_writeback(inode);
1933 		r = wait_event_killable(ci->i_cap_wq,
1934 					context_is_writeable_or_written(inode, snapc));
1935 		ceph_put_snap_context(snapc);
1936 		/*
1937 		 * *foliop is cleared, so a return of 0 makes
1938 		 * netfs_write_begin() grab the folio again and call us again
1939 		 * to recheck it. Any error (fatal signal) fails the write.
1940 		 */
1941 		return r;
1942 	}
1943 	return 0;
1944 }
1945 
1946 /*
1947  * We are only allowed to write into/dirty the page if the page is
1948  * clean, or already dirty within the same snap context.
1949  */
1950 static int ceph_write_begin(const struct kiocb *iocb,
1951 			    struct address_space *mapping,
1952 			    loff_t pos, unsigned len,
1953 			    struct folio **foliop, void **fsdata)
1954 {
1955 	struct file *file = iocb->ki_filp;
1956 	struct inode *inode = file_inode(file);
1957 	struct ceph_inode_info *ci = ceph_inode(inode);
1958 	int r;
1959 
1960 	r = netfs_write_begin(&ci->netfs, file, inode->i_mapping, pos, len, foliop, NULL);
1961 	if (r < 0)
1962 		return r;
1963 
1964 	folio_wait_private_2(*foliop); /* [DEPRECATED] */
1965 	WARN_ON_ONCE(!folio_test_locked(*foliop));
1966 	return 0;
1967 }
1968 
1969 /*
1970  * we don't do anything in here that simple_write_end doesn't do
1971  * except adjust dirty page accounting
1972  */
1973 static int ceph_write_end(const struct kiocb *iocb,
1974 			  struct address_space *mapping, loff_t pos,
1975 			  unsigned len, unsigned copied,
1976 			  struct folio *folio, void *fsdata)
1977 {
1978 	struct file *file = iocb->ki_filp;
1979 	struct inode *inode = file_inode(file);
1980 	struct ceph_client *cl = ceph_inode_to_client(inode);
1981 	bool check_cap = false;
1982 
1983 	doutc(cl, "%llx.%llx file %p folio %p %d~%d (%d)\n", ceph_vinop(inode),
1984 	      file, folio, (int)pos, (int)copied, (int)len);
1985 
1986 	if (!folio_test_uptodate(folio)) {
1987 		/* just return that nothing was copied on a short copy */
1988 		if (copied < len) {
1989 			copied = 0;
1990 			goto out;
1991 		}
1992 		folio_mark_uptodate(folio);
1993 	}
1994 
1995 	/* did file size increase? */
1996 	if (pos+copied > i_size_read(inode))
1997 		check_cap = ceph_inode_set_size(inode, pos+copied);
1998 
1999 	folio_mark_dirty(folio);
2000 
2001 out:
2002 	folio_unlock(folio);
2003 	folio_put(folio);
2004 
2005 	if (check_cap)
2006 		ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY);
2007 
2008 	return copied;
2009 }
2010 
2011 const struct address_space_operations ceph_aops = {
2012 	.read_folio = netfs_read_folio,
2013 	.readahead = netfs_readahead,
2014 	.writepages = ceph_writepages_start,
2015 	.write_begin = ceph_write_begin,
2016 	.write_end = ceph_write_end,
2017 	.dirty_folio = ceph_dirty_folio,
2018 	.invalidate_folio = ceph_invalidate_folio,
2019 	.release_folio = netfs_release_folio,
2020 	.direct_IO = noop_direct_IO,
2021 	.migrate_folio = filemap_migrate_folio,
2022 };
2023 
2024 static void ceph_block_sigs(sigset_t *oldset)
2025 {
2026 	sigset_t mask;
2027 	siginitsetinv(&mask, sigmask(SIGKILL));
2028 	sigprocmask(SIG_BLOCK, &mask, oldset);
2029 }
2030 
2031 static void ceph_restore_sigs(sigset_t *oldset)
2032 {
2033 	sigprocmask(SIG_SETMASK, oldset, NULL);
2034 }
2035 
2036 /*
2037  * True if the page cache is only being kept alive by LAZYIO: CACHE is no
2038  * longer present in the implemented cap set, while LAZYIO still is, so
2039  * handle_cap_grant() left the pages in place for the lazy openers.
2040  *
2041  * Such pages are only valid under the lazy semantics, and a non-lazy reader
2042  * must not be served from them.  try_get_cap_refs() already refuses to
2043  * satisfy a non-lazy CACHE want with LAZYIO for read_iter()/write_iter();
2044  * faults need the same gate.
2045  *
2046  * Unlike __prep_cap(), which has to reason about a single cap, the question
2047  * here is whether the cached data can have gone stale, so the aggregate of
2048  * all caps is the right granularity: as long as any one of them still has
2049  * CACHE implemented, its revocation has not been ACKed and the MDS cannot
2050  * have handed CACHE to another client.  Note that __ceph_caps_issued()
2051  * reports i_snap_caps in "have" only and not in "implemented", hence the
2052  * OR; snap caps never include LAZYIO, so folding them in can only make this
2053  * more conservative.
2054  */
2055 static bool ceph_pages_retained_for_lazyio(struct inode *inode)
2056 {
2057 	struct ceph_inode_info *ci = ceph_inode(inode);
2058 	int have, implemented;
2059 
2060 	if (!inode->i_data.nrpages)
2061 		return false;
2062 
2063 	spin_lock(&ci->i_ceph_lock);
2064 	have = __ceph_caps_issued(ci, &implemented);
2065 	implemented |= have;
2066 	spin_unlock(&ci->i_ceph_lock);
2067 
2068 	return !(implemented & CEPH_CAP_FILE_CACHE) &&
2069 	       (implemented & CEPH_CAP_FILE_LAZYIO);
2070 }
2071 
2072 /*
2073  * vm ops
2074  */
2075 static vm_fault_t ceph_filemap_fault(struct vm_fault *vmf)
2076 {
2077 	struct vm_area_struct *vma = vmf->vma;
2078 	struct inode *inode = file_inode(vma->vm_file);
2079 	struct ceph_inode_info *ci = ceph_inode(inode);
2080 	struct ceph_client *cl = ceph_inode_to_client(inode);
2081 	struct ceph_file_info *fi = vma->vm_file->private_data;
2082 	loff_t off = (loff_t)vmf->pgoff << PAGE_SHIFT;
2083 	int want, got, err;
2084 	sigset_t oldset;
2085 	vm_fault_t ret = VM_FAULT_SIGBUS;
2086 
2087 	if (ceph_inode_is_shutdown(inode))
2088 		return ret;
2089 
2090 	ceph_block_sigs(&oldset);
2091 
2092 	doutc(cl, "%llx.%llx %llu trying to get caps\n",
2093 	      ceph_vinop(inode), off);
2094 	if (fi->fmode & CEPH_FILE_MODE_LAZY)
2095 		want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO;
2096 	else
2097 		want = CEPH_CAP_FILE_CACHE;
2098 
2099 	got = 0;
2100 	err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_RD, want, -1, &got);
2101 	if (err < 0)
2102 		goto out_restore;
2103 
2104 	doutc(cl, "%llx.%llx %llu got cap refs on %s\n", ceph_vinop(inode),
2105 	      off, ceph_cap_string(got));
2106 
2107 	/*
2108 	 * Don't fault a non-lazy mapping in from page cache that only LAZYIO
2109 	 * covers.  The "every opener is lazy" gate in handle_cap_grant() can't
2110 	 * catch this on its own: an open that happens after the cache was
2111 	 * retained sends no cap message, so nothing re-evaluates the decision.
2112 	 * Drop the range here instead and let the fault below re-read it.
2113 	 */
2114 	if (!(fi->fmode & CEPH_FILE_MODE_LAZY) &&
2115 	    !(got & CEPH_CAP_FILE_CACHE) &&
2116 	    ceph_pages_retained_for_lazyio(inode)) {
2117 		struct address_space *mapping = inode->i_mapping;
2118 
2119 		doutc(cl, "%llx.%llx %llu dropping LAZYIO page cache\n",
2120 		      ceph_vinop(inode), off);
2121 		err = invalidate_inode_pages2_range(mapping, vmf->pgoff,
2122 						    vmf->pgoff);
2123 		if (err == -EBUSY) {
2124 			/*
2125 			 * The folio is dirty and ceph has no launder_folio(),
2126 			 * so invalidate_inode_pages2_range() cannot drop it.
2127 			 * Write it back and retry -- handing a folio that only
2128 			 * LAZYIO covers to a non-lazy vma is not an option.
2129 			 */
2130 			err = filemap_write_and_wait_range(mapping, off,
2131 							   off + PAGE_SIZE - 1);
2132 			if (!err)
2133 				err = invalidate_inode_pages2_range(mapping,
2134 						vmf->pgoff, vmf->pgoff);
2135 		}
2136 		if (err) {
2137 			doutc(cl, "%llx.%llx %llu failed to drop LAZYIO page cache: %d\n",
2138 			      ceph_vinop(inode), off, err);
2139 			ceph_put_cap_refs(ci, got);
2140 			goto out_restore;
2141 		}
2142 	}
2143 
2144 	if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) ||
2145 	    !ceph_has_inline_data(ci)) {
2146 		CEPH_DEFINE_RW_CONTEXT(rw_ctx, got);
2147 		ceph_add_rw_context(fi, &rw_ctx);
2148 		ret = filemap_fault(vmf);
2149 		ceph_del_rw_context(fi, &rw_ctx);
2150 		doutc(cl, "%llx.%llx %llu drop cap refs %s ret %x\n",
2151 		      ceph_vinop(inode), off, ceph_cap_string(got), ret);
2152 	} else
2153 		err = -EAGAIN;
2154 
2155 	ceph_put_cap_refs(ci, got);
2156 
2157 	if (err != -EAGAIN)
2158 		goto out_restore;
2159 
2160 	/* read inline data */
2161 	if (off >= PAGE_SIZE) {
2162 		/* does not support inline data > PAGE_SIZE */
2163 		ret = VM_FAULT_SIGBUS;
2164 	} else {
2165 		struct address_space *mapping = inode->i_mapping;
2166 		struct folio *folio;
2167 
2168 		filemap_invalidate_lock_shared(mapping);
2169 		folio = __filemap_get_folio(mapping, 0,
2170 				FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
2171 				mapping_gfp_constraint(mapping, ~__GFP_FS));
2172 		if (IS_ERR(folio)) {
2173 			ret = VM_FAULT_OOM;
2174 			goto out_inline;
2175 		}
2176 		err = __ceph_do_getattr(inode, folio,
2177 					 CEPH_STAT_CAP_INLINE_DATA, true);
2178 		if (err < 0 || off >= i_size_read(inode)) {
2179 			folio_unlock(folio);
2180 			folio_put(folio);
2181 			ret = vmf_error(err);
2182 			goto out_inline;
2183 		}
2184 		if (err < folio_size(folio))
2185 			folio_zero_segment(folio, err, folio_size(folio));
2186 		else
2187 			flush_dcache_folio(folio);
2188 		folio_mark_uptodate(folio);
2189 		vmf->page = folio_page(folio, 0);
2190 		ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED;
2191 out_inline:
2192 		filemap_invalidate_unlock_shared(mapping);
2193 		doutc(cl, "%llx.%llx %llu read inline data ret %x\n",
2194 		      ceph_vinop(inode), off, ret);
2195 	}
2196 out_restore:
2197 	ceph_restore_sigs(&oldset);
2198 	if (err < 0)
2199 		ret = vmf_error(err);
2200 
2201 	return ret;
2202 }
2203 
2204 /*
2205  * Return true if the MDS has issued us a cap that covers buffered
2206  * dirtying: either BUFFER, or LAZYIO (as long as LAZYIO itself is not
2207  * being revoked).  This mirrors the conditions under which
2208  * try_get_cap_refs() will hand out a BUFFER or LAZYIO ref.
2209  */
2210 static bool ceph_have_dirtyable_caps(struct inode *inode)
2211 {
2212 	struct ceph_inode_info *ci = ceph_inode(inode);
2213 	int have, implemented;
2214 	bool ret = false;
2215 
2216 	spin_lock(&ci->i_ceph_lock);
2217 	have = __ceph_caps_issued(ci, &implemented);
2218 	if (have & CEPH_CAP_FILE_BUFFER) {
2219 		ret = true;
2220 	} else if ((have & CEPH_CAP_FILE_LAZYIO) &&
2221 		   !((implemented & ~have) & CEPH_CAP_FILE_LAZYIO)) {
2222 		ret = true;
2223 	}
2224 	spin_unlock(&ci->i_ceph_lock);
2225 	return ret;
2226 }
2227 
2228 static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf)
2229 {
2230 	struct vm_area_struct *vma = vmf->vma;
2231 	struct inode *inode = file_inode(vma->vm_file);
2232 	struct ceph_client *cl = ceph_inode_to_client(inode);
2233 	struct ceph_inode_info *ci = ceph_inode(inode);
2234 	struct ceph_file_info *fi = vma->vm_file->private_data;
2235 	struct ceph_cap_flush *prealloc_cf;
2236 	struct folio *folio = page_folio(vmf->page);
2237 	loff_t off = folio_pos(folio);
2238 	loff_t size = i_size_read(inode);
2239 	size_t len;
2240 	int want, got, err;
2241 	sigset_t oldset;
2242 	vm_fault_t ret = VM_FAULT_SIGBUS;
2243 
2244 	if (ceph_inode_is_shutdown(inode))
2245 		return ret;
2246 
2247 	prealloc_cf = ceph_alloc_cap_flush();
2248 	if (!prealloc_cf)
2249 		return VM_FAULT_OOM;
2250 
2251 	sb_start_pagefault(inode->i_sb);
2252 	ceph_block_sigs(&oldset);
2253 
2254 	if (off + folio_size(folio) <= size)
2255 		len = folio_size(folio);
2256 	else
2257 		len = offset_in_folio(folio, size);
2258 
2259 	doutc(cl, "%llx.%llx %llu~%zd getting caps i_size %llu\n",
2260 	      ceph_vinop(inode), off, len, size);
2261 	if (fi->fmode & CEPH_FILE_MODE_LAZY)
2262 		want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO;
2263 	else
2264 		want = CEPH_CAP_FILE_BUFFER;
2265 
2266 retry_caps:
2267 	got = 0;
2268 	err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_WR, want, off + len, &got);
2269 	if (err < 0)
2270 		goto out_free;
2271 
2272 	doutc(cl, "%llx.%llx %llu~%zd got cap refs on %s\n", ceph_vinop(inode),
2273 	      off, len, ceph_cap_string(got));
2274 
2275 	/*
2276 	 * ceph_write_iter() makes the same check and falls back to
2277 	 * synchronous writes, but a page fault has no such fallback:
2278 	 * dirtying the folio without BUFFER or LAZYIO would leave dirty
2279 	 * data uncovered by any issued cap (e.g. while LAZYIO is being
2280 	 * revoked, or after it has been released).  Wait for the MDS to
2281 	 * (re)grant a covering cap, matching how the exclude gate in
2282 	 * try_get_cap_refs() blocks buffered writes while BUFFER is
2283 	 * revoking.
2284 	 */
2285 	if ((fi->fmode & CEPH_FILE_MODE_LAZY) &&
2286 	    (got & (CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO)) == 0) {
2287 		ceph_put_cap_refs(ci, got);
2288 		got = 0;
2289 		doutc(cl, "%llx.%llx %llu~%zd waiting for BUFFER or LAZYIO\n",
2290 		      ceph_vinop(inode), off, len);
2291 		/*
2292 		 * Bail out if the inode is being shut down: the caps are gone
2293 		 * for good, so ceph_have_dirtyable_caps() would never become
2294 		 * true again.  Note that we cannot rely on ceph_get_caps()
2295 		 * failing with -ESTALE on the retry either -- if WR is still
2296 		 * held it would succeed without BUFFER or LAZYIO and send us
2297 		 * straight back here.
2298 		 */
2299 		err = wait_event_killable(ci->i_cap_wq,
2300 					  ceph_have_dirtyable_caps(inode) ||
2301 					  ceph_inode_is_shutdown(inode));
2302 		if (err)
2303 			goto out_free;
2304 		if (ceph_inode_is_shutdown(inode)) {
2305 			err = -ESTALE;
2306 			goto out_free;
2307 		}
2308 		goto retry_caps;
2309 	}
2310 
2311 	/* Update time before taking folio lock */
2312 	file_update_time(vma->vm_file);
2313 	inode_inc_iversion_raw(inode);
2314 
2315 	do {
2316 		struct ceph_snap_context *snapc;
2317 
2318 		folio_lock(folio);
2319 
2320 		if (folio_mkwrite_check_truncate(folio, inode) < 0) {
2321 			folio_unlock(folio);
2322 			ret = VM_FAULT_NOPAGE;
2323 			break;
2324 		}
2325 
2326 		snapc = ceph_find_incompatible(folio);
2327 		if (!snapc) {
2328 			/* success.  we'll keep the folio locked. */
2329 			folio_mark_dirty(folio);
2330 			ret = VM_FAULT_LOCKED;
2331 			break;
2332 		}
2333 
2334 		folio_unlock(folio);
2335 
2336 		if (IS_ERR(snapc)) {
2337 			ret = VM_FAULT_SIGBUS;
2338 			break;
2339 		}
2340 
2341 		ceph_queue_writeback(inode);
2342 		err = wait_event_killable(ci->i_cap_wq,
2343 				context_is_writeable_or_written(inode, snapc));
2344 		ceph_put_snap_context(snapc);
2345 	} while (err == 0);
2346 
2347 	if (ret == VM_FAULT_LOCKED) {
2348 		int dirty;
2349 		spin_lock(&ci->i_ceph_lock);
2350 		dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
2351 					       &prealloc_cf);
2352 		spin_unlock(&ci->i_ceph_lock);
2353 		if (dirty)
2354 			__mark_inode_dirty(inode, dirty);
2355 	}
2356 
2357 	doutc(cl, "%llx.%llx %llu~%zd dropping cap refs on %s ret %x\n",
2358 	      ceph_vinop(inode), off, len, ceph_cap_string(got), ret);
2359 	ceph_put_cap_refs_async(ci, got);
2360 out_free:
2361 	ceph_restore_sigs(&oldset);
2362 	sb_end_pagefault(inode->i_sb);
2363 	ceph_free_cap_flush(prealloc_cf);
2364 	if (err < 0)
2365 		ret = vmf_error(err);
2366 	return ret;
2367 }
2368 
2369 #ifdef CONFIG_CEPH_FS_INLINE_DATA
2370 void ceph_fill_inline_data(struct inode *inode, struct folio *locked_folio,
2371 			   char	*data, size_t len)
2372 {
2373 	struct ceph_client *cl = ceph_inode_to_client(inode);
2374 	struct address_space *mapping = inode->i_mapping;
2375 	struct folio *folio;
2376 
2377 	if (locked_folio) {
2378 		folio = locked_folio;
2379 	} else {
2380 		if (i_size_read(inode) == 0)
2381 			return;
2382 		folio = __filemap_get_folio(mapping, 0,
2383 				FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
2384 				mapping_gfp_constraint(mapping, ~__GFP_FS));
2385 		if (IS_ERR(folio))
2386 			return;
2387 		if (folio_test_uptodate(folio)) {
2388 			folio_unlock(folio);
2389 			folio_put(folio);
2390 			return;
2391 		}
2392 	}
2393 
2394 	doutc(cl, "%p %llx.%llx len %zu locked_folio %p\n", inode,
2395 	      ceph_vinop(inode), len, locked_folio);
2396 
2397 	if (len > folio_size(folio)) {
2398 		pr_warn_ratelimited_client(cl, "oversized inline data %zu\n",
2399 					   len);
2400 		len = folio_size(folio);
2401 	}
2402 
2403 	if (len > 0)
2404 		memcpy_to_folio(folio, 0, data, len);
2405 
2406 	if (folio != locked_folio) {
2407 		if (len < folio_size(folio))
2408 			folio_zero_segment(folio, len, folio_size(folio));
2409 
2410 		folio_mark_uptodate(folio);
2411 		folio_unlock(folio);
2412 		folio_put(folio);
2413 	}
2414 }
2415 
2416 int ceph_uninline_data(struct file *file)
2417 {
2418 	struct inode *inode = file_inode(file);
2419 	struct ceph_inode_info *ci = ceph_inode(inode);
2420 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
2421 	struct ceph_client *cl = fsc->client;
2422 	struct ceph_osd_request *req = NULL;
2423 	struct ceph_cap_flush *prealloc_cf = NULL;
2424 	struct folio *folio = NULL;
2425 	struct ceph_snap_context *snapc = NULL;
2426 	u64 inline_version = CEPH_INLINE_NONE;
2427 	struct page *pages[1];
2428 	int err = 0;
2429 	u64 len;
2430 
2431 	spin_lock(&ci->i_ceph_lock);
2432 	inline_version = ceph_inline_version(ci);
2433 	spin_unlock(&ci->i_ceph_lock);
2434 
2435 	doutc(cl, "%llx.%llx inline_version %llu\n", ceph_vinop(inode),
2436 	      inline_version);
2437 
2438 	if (ceph_inode_is_shutdown(inode)) {
2439 		err = -EIO;
2440 		goto out;
2441 	}
2442 
2443 	if (inline_version == CEPH_INLINE_NONE)
2444 		return 0;
2445 
2446 	prealloc_cf = ceph_alloc_cap_flush();
2447 	if (!prealloc_cf)
2448 		return -ENOMEM;
2449 
2450 	if (inline_version == 1) /* initial version, no data */
2451 		goto out_uninline;
2452 
2453 	down_read(&fsc->mdsc->snap_rwsem);
2454 	spin_lock(&ci->i_ceph_lock);
2455 	if (__ceph_have_pending_cap_snap(ci)) {
2456 		struct ceph_cap_snap *capsnap =
2457 				list_last_entry(&ci->i_cap_snaps,
2458 						struct ceph_cap_snap,
2459 						ci_item);
2460 		snapc = ceph_get_snap_context(capsnap->context);
2461 	} else {
2462 		if (!ci->i_head_snapc) {
2463 			ci->i_head_snapc = ceph_get_snap_context(
2464 				ci->i_snap_realm->cached_context);
2465 		}
2466 		snapc = ceph_get_snap_context(ci->i_head_snapc);
2467 	}
2468 	spin_unlock(&ci->i_ceph_lock);
2469 	up_read(&fsc->mdsc->snap_rwsem);
2470 
2471 	folio = read_mapping_folio(inode->i_mapping, 0, file);
2472 	if (IS_ERR(folio)) {
2473 		err = PTR_ERR(folio);
2474 		goto out;
2475 	}
2476 
2477 	folio_lock(folio);
2478 
2479 	len = i_size_read(inode);
2480 	if (len > folio_size(folio))
2481 		len = folio_size(folio);
2482 
2483 	req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
2484 				    ceph_vino(inode), 0, &len, 0, 1,
2485 				    CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE,
2486 				    snapc, 0, 0, false);
2487 	if (IS_ERR(req)) {
2488 		err = PTR_ERR(req);
2489 		goto out_unlock;
2490 	}
2491 
2492 	req->r_mtime = inode_get_mtime(inode);
2493 	ceph_osdc_start_request(&fsc->client->osdc, req);
2494 	err = ceph_osdc_wait_request(&fsc->client->osdc, req);
2495 	ceph_osdc_put_request(req);
2496 	if (err < 0)
2497 		goto out_unlock;
2498 
2499 	req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
2500 				    ceph_vino(inode), 0, &len, 1, 3,
2501 				    CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE,
2502 				    snapc, ci->i_truncate_seq,
2503 				    ci->i_truncate_size, false);
2504 	if (IS_ERR(req)) {
2505 		err = PTR_ERR(req);
2506 		goto out_unlock;
2507 	}
2508 
2509 	pages[0] = folio_page(folio, 0);
2510 	osd_req_op_extent_osd_data_pages(req, 1, pages, len, 0, false, false);
2511 
2512 	{
2513 		__le64 xattr_buf = cpu_to_le64(inline_version);
2514 		err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR,
2515 					    "inline_version", &xattr_buf,
2516 					    sizeof(xattr_buf),
2517 					    CEPH_OSD_CMPXATTR_OP_GT,
2518 					    CEPH_OSD_CMPXATTR_MODE_U64);
2519 		if (err)
2520 			goto out_put_req;
2521 	}
2522 
2523 	{
2524 		char xattr_buf[32];
2525 		int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf),
2526 					 "%llu", inline_version);
2527 		err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR,
2528 					    "inline_version",
2529 					    xattr_buf, xattr_len, 0, 0);
2530 		if (err)
2531 			goto out_put_req;
2532 	}
2533 
2534 	req->r_mtime = inode_get_mtime(inode);
2535 	ceph_osdc_start_request(&fsc->client->osdc, req);
2536 	err = ceph_osdc_wait_request(&fsc->client->osdc, req);
2537 
2538 	ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency,
2539 				  req->r_end_latency, len, err);
2540 
2541 out_uninline:
2542 	if (!err) {
2543 		int dirty;
2544 
2545 		/* Set to CAP_INLINE_NONE and dirty the caps */
2546 		down_read(&fsc->mdsc->snap_rwsem);
2547 		spin_lock(&ci->i_ceph_lock);
2548 		ceph_set_inline_version(ci, CEPH_INLINE_NONE);
2549 		dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, &prealloc_cf);
2550 		spin_unlock(&ci->i_ceph_lock);
2551 		up_read(&fsc->mdsc->snap_rwsem);
2552 		if (dirty)
2553 			__mark_inode_dirty(inode, dirty);
2554 	}
2555 out_put_req:
2556 	ceph_osdc_put_request(req);
2557 	if (err == -ECANCELED)
2558 		err = 0;
2559 out_unlock:
2560 	if (folio) {
2561 		folio_unlock(folio);
2562 		folio_put(folio);
2563 	}
2564 out:
2565 	ceph_put_snap_context(snapc);
2566 	ceph_free_cap_flush(prealloc_cf);
2567 	doutc(cl, "%llx.%llx inline_version %llu = %d\n",
2568 	      ceph_vinop(inode), inline_version, err);
2569 	return err;
2570 }
2571 #endif /* CONFIG_CEPH_FS_INLINE_DATA */
2572 
2573 static const struct vm_operations_struct ceph_vmops = {
2574 	.fault		= ceph_filemap_fault,
2575 	.page_mkwrite	= ceph_page_mkwrite,
2576 };
2577 
2578 int ceph_mmap_prepare(struct vm_area_desc *desc)
2579 {
2580 	struct address_space *mapping = desc->file->f_mapping;
2581 
2582 	if (!mapping->a_ops->read_folio)
2583 		return -ENOEXEC;
2584 	desc->vm_ops = &ceph_vmops;
2585 	return 0;
2586 }
2587 
2588 enum {
2589 	POOL_READ	= 1,
2590 	POOL_WRITE	= 2,
2591 };
2592 
2593 static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
2594 				s64 pool, struct ceph_string *pool_ns)
2595 {
2596 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(&ci->netfs.inode);
2597 	struct ceph_mds_client *mdsc = fsc->mdsc;
2598 	struct ceph_client *cl = fsc->client;
2599 	struct ceph_osd_request *rd_req = NULL, *wr_req = NULL;
2600 	struct rb_node **p, *parent;
2601 	struct ceph_pool_perm *perm;
2602 	struct page **pages;
2603 	size_t pool_ns_len;
2604 	int err = 0, err2 = 0, have = 0;
2605 
2606 	down_read(&mdsc->pool_perm_rwsem);
2607 	p = &mdsc->pool_perm_tree.rb_node;
2608 	while (*p) {
2609 		perm = rb_entry(*p, struct ceph_pool_perm, node);
2610 		if (pool < perm->pool)
2611 			p = &(*p)->rb_left;
2612 		else if (pool > perm->pool)
2613 			p = &(*p)->rb_right;
2614 		else {
2615 			int ret = ceph_compare_string(pool_ns,
2616 						perm->pool_ns,
2617 						perm->pool_ns_len);
2618 			if (ret < 0)
2619 				p = &(*p)->rb_left;
2620 			else if (ret > 0)
2621 				p = &(*p)->rb_right;
2622 			else {
2623 				have = perm->perm;
2624 				break;
2625 			}
2626 		}
2627 	}
2628 	up_read(&mdsc->pool_perm_rwsem);
2629 	if (*p)
2630 		goto out;
2631 
2632 	if (pool_ns)
2633 		doutc(cl, "pool %lld ns %.*s no perm cached\n", pool,
2634 		      (int)pool_ns->len, pool_ns->str);
2635 	else
2636 		doutc(cl, "pool %lld no perm cached\n", pool);
2637 
2638 	down_write(&mdsc->pool_perm_rwsem);
2639 	p = &mdsc->pool_perm_tree.rb_node;
2640 	parent = NULL;
2641 	while (*p) {
2642 		parent = *p;
2643 		perm = rb_entry(parent, struct ceph_pool_perm, node);
2644 		if (pool < perm->pool)
2645 			p = &(*p)->rb_left;
2646 		else if (pool > perm->pool)
2647 			p = &(*p)->rb_right;
2648 		else {
2649 			int ret = ceph_compare_string(pool_ns,
2650 						perm->pool_ns,
2651 						perm->pool_ns_len);
2652 			if (ret < 0)
2653 				p = &(*p)->rb_left;
2654 			else if (ret > 0)
2655 				p = &(*p)->rb_right;
2656 			else {
2657 				have = perm->perm;
2658 				break;
2659 			}
2660 		}
2661 	}
2662 	if (*p) {
2663 		up_write(&mdsc->pool_perm_rwsem);
2664 		goto out;
2665 	}
2666 
2667 	rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL,
2668 					 1, false, GFP_KERNEL);
2669 	if (!rd_req) {
2670 		err = -ENOMEM;
2671 		goto out_unlock;
2672 	}
2673 
2674 	rd_req->r_flags = CEPH_OSD_FLAG_READ;
2675 	osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0);
2676 	rd_req->r_base_oloc.pool = pool;
2677 	if (pool_ns)
2678 		rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns);
2679 	ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ceph_ino(&ci->netfs.inode));
2680 
2681 	err = ceph_osdc_alloc_messages(rd_req, GFP_KERNEL);
2682 	if (err)
2683 		goto out_unlock;
2684 
2685 	wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL,
2686 					 1, false, GFP_KERNEL);
2687 	if (!wr_req) {
2688 		err = -ENOMEM;
2689 		goto out_unlock;
2690 	}
2691 
2692 	wr_req->r_flags = CEPH_OSD_FLAG_WRITE;
2693 	osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL);
2694 	ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc);
2695 	ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid);
2696 
2697 	err = ceph_osdc_alloc_messages(wr_req, GFP_KERNEL);
2698 	if (err)
2699 		goto out_unlock;
2700 
2701 	/* one page should be large enough for STAT data */
2702 	pages = ceph_alloc_page_vector(1, GFP_KERNEL);
2703 	if (IS_ERR(pages)) {
2704 		err = PTR_ERR(pages);
2705 		goto out_unlock;
2706 	}
2707 
2708 	osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE,
2709 				     0, false, true);
2710 	ceph_osdc_start_request(&fsc->client->osdc, rd_req);
2711 
2712 	wr_req->r_mtime = inode_get_mtime(&ci->netfs.inode);
2713 	ceph_osdc_start_request(&fsc->client->osdc, wr_req);
2714 
2715 	err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req);
2716 	err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req);
2717 
2718 	if (err >= 0 || err == -ENOENT)
2719 		have |= POOL_READ;
2720 	else if (err != -EPERM) {
2721 		if (err == -EBLOCKLISTED)
2722 			fsc->blocklisted = true;
2723 		goto out_unlock;
2724 	}
2725 
2726 	if (err2 == 0 || err2 == -EEXIST)
2727 		have |= POOL_WRITE;
2728 	else if (err2 != -EPERM) {
2729 		if (err2 == -EBLOCKLISTED)
2730 			fsc->blocklisted = true;
2731 		err = err2;
2732 		goto out_unlock;
2733 	}
2734 
2735 	pool_ns_len = pool_ns ? pool_ns->len : 0;
2736 	perm = kmalloc_flex(*perm, pool_ns, pool_ns_len + 1);
2737 	if (!perm) {
2738 		err = -ENOMEM;
2739 		goto out_unlock;
2740 	}
2741 
2742 	perm->pool = pool;
2743 	perm->perm = have;
2744 	perm->pool_ns_len = pool_ns_len;
2745 	if (pool_ns_len > 0)
2746 		memcpy(perm->pool_ns, pool_ns->str, pool_ns_len);
2747 	perm->pool_ns[pool_ns_len] = 0;
2748 
2749 	rb_link_node(&perm->node, parent, p);
2750 	rb_insert_color(&perm->node, &mdsc->pool_perm_tree);
2751 	err = 0;
2752 out_unlock:
2753 	up_write(&mdsc->pool_perm_rwsem);
2754 
2755 	ceph_osdc_put_request(rd_req);
2756 	ceph_osdc_put_request(wr_req);
2757 out:
2758 	if (!err)
2759 		err = have;
2760 	if (pool_ns)
2761 		doutc(cl, "pool %lld ns %.*s result = %d\n", pool,
2762 		      (int)pool_ns->len, pool_ns->str, err);
2763 	else
2764 		doutc(cl, "pool %lld result = %d\n", pool, err);
2765 	return err;
2766 }
2767 
2768 int ceph_pool_perm_check(struct inode *inode, int need)
2769 {
2770 	struct ceph_client *cl = ceph_inode_to_client(inode);
2771 	struct ceph_inode_info *ci = ceph_inode(inode);
2772 	struct ceph_string *pool_ns;
2773 	s64 pool;
2774 	int ret;
2775 	unsigned long flags;
2776 
2777 	/* Only need to do this for regular files */
2778 	if (!S_ISREG(inode->i_mode))
2779 		return 0;
2780 
2781 	if (ceph_in_snap(inode)) {
2782 		/*
2783 		 * Pool permission check needs to write to the first object.
2784 		 * But for snapshot, head of the first object may have already
2785 		 * been deleted. Skip check to avoid creating orphan object.
2786 		 */
2787 		return 0;
2788 	}
2789 
2790 	if (ceph_test_mount_opt(ceph_inode_to_fs_client(inode),
2791 				NOPOOLPERM))
2792 		return 0;
2793 
2794 	spin_lock(&ci->i_ceph_lock);
2795 	flags = ci->i_ceph_flags;
2796 	pool = ci->i_layout.pool_id;
2797 	spin_unlock(&ci->i_ceph_lock);
2798 check:
2799 	if (flags & CEPH_I_POOL_PERM) {
2800 		if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) {
2801 			doutc(cl, "pool %lld no read perm\n", pool);
2802 			return -EPERM;
2803 		}
2804 		if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) {
2805 			doutc(cl, "pool %lld no write perm\n", pool);
2806 			return -EPERM;
2807 		}
2808 		return 0;
2809 	}
2810 
2811 	pool_ns = ceph_try_get_string(ci->i_layout.pool_ns);
2812 	ret = __ceph_pool_perm_get(ci, pool, pool_ns);
2813 	ceph_put_string(pool_ns);
2814 	if (ret < 0)
2815 		return ret;
2816 
2817 	spin_lock(&ci->i_ceph_lock);
2818 	if (pool == ci->i_layout.pool_id &&
2819 	    pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) {
2820 		set_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags);
2821 		if (ret & POOL_READ)
2822 			set_bit(CEPH_I_POOL_RD_BIT, &ci->i_ceph_flags);
2823 		if (ret & POOL_WRITE)
2824 			set_bit(CEPH_I_POOL_WR_BIT, &ci->i_ceph_flags);
2825 	} else {
2826 		pool = ci->i_layout.pool_id;
2827 	}
2828 	/* Re-read flags under the lock so check: sees the updated bits. */
2829 	flags = ci->i_ceph_flags;
2830 	spin_unlock(&ci->i_ceph_lock);
2831 	goto check;
2832 }
2833 
2834 void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc)
2835 {
2836 	struct ceph_pool_perm *perm;
2837 	struct rb_node *n;
2838 
2839 	while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) {
2840 		n = rb_first(&mdsc->pool_perm_tree);
2841 		perm = rb_entry(n, struct ceph_pool_perm, node);
2842 		rb_erase(n, &mdsc->pool_perm_tree);
2843 		kfree(perm);
2844 	}
2845 }
2846