1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/ceph/ceph_debug.h> 3 4 #include <linux/backing-dev.h> 5 #include <linux/fs.h> 6 #include <linux/mm.h> 7 #include <linux/swap.h> 8 #include <linux/pagemap.h> 9 #include <linux/slab.h> 10 #include <linux/folio_batch.h> 11 #include <linux/task_io_accounting_ops.h> 12 #include <linux/signal.h> 13 #include <linux/iversion.h> 14 #include <linux/ktime.h> 15 #include <linux/netfs.h> 16 #include <trace/events/netfs.h> 17 18 #include "super.h" 19 #include "mds_client.h" 20 #include "cache.h" 21 #include "metric.h" 22 #include "subvolume_metrics.h" 23 #include "crypto.h" 24 #include <linux/ceph/osd_client.h> 25 #include <linux/ceph/striper.h> 26 27 /* 28 * Ceph address space ops. 29 * 30 * There are a few funny things going on here. 31 * 32 * The folio->private field is used to reference a struct 33 * ceph_snap_context for _every_ dirty folio. This indicates which 34 * snapshot the folio was logically dirtied in, and thus which snap 35 * context needs to be associated with the osd write during writeback. 36 * 37 * Similarly, struct ceph_inode_info maintains a set of counters to 38 * count dirty pages on the inode. In the absence of snapshots, 39 * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. 40 * 41 * When a snapshot is taken (that is, when the client receives 42 * notification that a snapshot was taken), each inode with caps and 43 * with dirty pages (dirty pages implies there is a cap) gets a new 44 * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending 45 * order, new snaps go to the tail). The i_wrbuffer_ref_head count is 46 * moved to capsnap->dirty. (Unless a sync write is currently in 47 * progress. In that case, the capsnap is said to be "pending", new 48 * writes cannot start, and the capsnap isn't "finalized" until the 49 * write completes (or fails) and a final size/mtime for the inode for 50 * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. 51 * 52 * On writeback, we must submit writes to the osd IN SNAP ORDER. So, 53 * we look for the first capsnap in i_cap_snaps and write out pages in 54 * that snap context _only_. Then we move on to the next capsnap, 55 * eventually reaching the "live" or "head" context (i.e., pages that 56 * are not yet snapped) and are writing the most recently dirtied 57 * pages. 58 * 59 * Invalidate and so forth must take care to ensure the dirty page 60 * accounting is preserved. 61 */ 62 63 #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) 64 #define CONGESTION_OFF_THRESH(congestion_kb) \ 65 (CONGESTION_ON_THRESH(congestion_kb) - \ 66 (CONGESTION_ON_THRESH(congestion_kb) >> 2)) 67 68 static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len, 69 struct folio **foliop, void **_fsdata); 70 71 static inline 72 struct ceph_snap_context *ceph_folio_snap_context(const struct folio *folio) 73 { 74 return folio->private; 75 } 76 77 /* 78 * Dirty a page. Optimistically adjust accounting, on the assumption 79 * that we won't race with invalidate. If we do, readjust. 80 */ 81 static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio) 82 { 83 struct inode *inode = mapping->host; 84 struct ceph_client *cl = ceph_inode_to_client(inode); 85 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); 86 struct ceph_inode_info *ci; 87 struct ceph_snap_context *snapc; 88 89 if (folio_test_dirty(folio)) { 90 doutc(cl, "%llx.%llx %p idx %lu -- already dirty\n", 91 ceph_vinop(inode), folio, folio->index); 92 VM_BUG_ON_FOLIO(!folio_test_private(folio), folio); 93 return false; 94 } 95 96 atomic64_inc(&mdsc->dirty_folios); 97 98 ci = ceph_inode(inode); 99 100 /* dirty the head */ 101 spin_lock(&ci->i_ceph_lock); 102 if (__ceph_have_pending_cap_snap(ci)) { 103 struct ceph_cap_snap *capsnap = 104 list_last_entry(&ci->i_cap_snaps, 105 struct ceph_cap_snap, 106 ci_item); 107 snapc = ceph_get_snap_context(capsnap->context); 108 capsnap->dirty_pages++; 109 } else { 110 BUG_ON(!ci->i_head_snapc); 111 snapc = ceph_get_snap_context(ci->i_head_snapc); 112 ++ci->i_wrbuffer_ref_head; 113 } 114 if (ci->i_wrbuffer_ref == 0) 115 ihold(inode); 116 ++ci->i_wrbuffer_ref; 117 doutc(cl, "%llx.%llx %p idx %lu head %d/%d -> %d/%d " 118 "snapc %p seq %lld (%d snaps)\n", 119 ceph_vinop(inode), folio, folio->index, 120 ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, 121 ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, 122 snapc, snapc->seq, snapc->num_snaps); 123 spin_unlock(&ci->i_ceph_lock); 124 125 /* 126 * Reference snap context in folio->private. Setting folio->private is 127 * what gets us the invalidate_folio callback. 128 */ 129 VM_WARN_ON_FOLIO(folio->private, folio); 130 folio_attach_private(folio, snapc); 131 132 return ceph_fscache_dirty_folio(mapping, folio); 133 } 134 135 /* 136 * If we are truncating the full folio (i.e. offset == 0), adjust the 137 * dirty folio counters appropriately. Only called if there is private 138 * data on the folio. 139 */ 140 static void ceph_invalidate_folio(struct folio *folio, size_t offset, 141 size_t length) 142 { 143 struct inode *inode = folio->mapping->host; 144 struct ceph_client *cl = ceph_inode_to_client(inode); 145 struct ceph_inode_info *ci = ceph_inode(inode); 146 struct ceph_snap_context *snapc; 147 148 149 if (offset != 0 || length != folio_size(folio)) { 150 doutc(cl, "%llx.%llx idx %lu partial dirty page %zu~%zu\n", 151 ceph_vinop(inode), folio->index, offset, length); 152 return; 153 } 154 155 WARN_ON(!folio_test_locked(folio)); 156 if (folio_test_private(folio)) { 157 doutc(cl, "%llx.%llx idx %lu full dirty page\n", 158 ceph_vinop(inode), folio->index); 159 160 snapc = folio_detach_private(folio); 161 ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 162 ceph_put_snap_context(snapc); 163 } 164 165 netfs_invalidate_folio(folio, offset, length); 166 } 167 168 static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) 169 { 170 struct inode *inode = rreq->inode; 171 struct ceph_inode_info *ci = ceph_inode(inode); 172 struct ceph_file_layout *lo = &ci->i_layout; 173 unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; 174 loff_t end = rreq->start + rreq->len, new_end; 175 struct ceph_netfs_request_data *priv = rreq->netfs_priv; 176 unsigned long max_len; 177 u32 blockoff; 178 179 if (priv) { 180 /* Readahead is disabled by posix_fadvise POSIX_FADV_RANDOM */ 181 if (priv->file_ra_disabled) 182 max_pages = 0; 183 else 184 max_pages = priv->file_ra_pages; 185 186 } 187 188 /* Readahead is disabled */ 189 if (!max_pages) 190 return; 191 192 max_len = max_pages << PAGE_SHIFT; 193 194 /* 195 * Try to expand the length forward by rounding up it to the next 196 * block, but do not exceed the file size, unless the original 197 * request already exceeds it. 198 */ 199 new_end = umin(round_up(end, lo->stripe_unit), rreq->i_size); 200 if (new_end > end && new_end <= rreq->start + max_len) 201 rreq->len = new_end - rreq->start; 202 203 /* Try to expand the start downward */ 204 div_u64_rem(rreq->start, lo->stripe_unit, &blockoff); 205 if (rreq->len + blockoff <= max_len) { 206 rreq->start -= blockoff; 207 rreq->len += blockoff; 208 } 209 } 210 211 static void finish_netfs_read(struct ceph_osd_request *req) 212 { 213 struct inode *inode = req->r_inode; 214 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 215 struct ceph_client *cl = fsc->client; 216 struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0); 217 struct netfs_io_subrequest *subreq = req->r_priv; 218 struct ceph_osd_req_op *op = &req->r_ops[0]; 219 int err = req->r_result; 220 bool sparse = (op->op == CEPH_OSD_OP_SPARSE_READ); 221 222 ceph_update_read_metrics(&fsc->mdsc->metric, req->r_start_latency, 223 req->r_end_latency, osd_data->length, err); 224 225 doutc(cl, "result %d subreq->len=%zu i_size=%lld\n", req->r_result, 226 subreq->len, i_size_read(req->r_inode)); 227 228 /* no object means success but no data */ 229 if (err == -ENOENT) { 230 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); 231 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 232 err = 0; 233 } else if (err == -EBLOCKLISTED) { 234 fsc->blocklisted = true; 235 } 236 237 if (err >= 0) { 238 if (sparse && err > 0) 239 err = ceph_sparse_ext_map_end(op); 240 if (err < subreq->len && 241 subreq->rreq->origin != NETFS_UNBUFFERED_READ && 242 subreq->rreq->origin != NETFS_DIO_READ) 243 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); 244 if (IS_ENCRYPTED(inode) && err > 0) { 245 err = ceph_fscrypt_decrypt_extents(inode, 246 osd_data->pages, subreq->start, 247 op->extent.sparse_ext, 248 op->extent.sparse_ext_cnt); 249 if (err > subreq->len) 250 err = subreq->len; 251 } 252 if (err > 0) 253 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); 254 } 255 256 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { 257 int num_pages = calc_pages_for(osd_data->alignment, 258 osd_data->length); 259 260 for (int i = 0; i < num_pages; i++) 261 put_page(osd_data->pages[i]); 262 kvfree(osd_data->pages); 263 } 264 if (err > 0) { 265 ceph_subvolume_metrics_record_io(fsc->mdsc, ceph_inode(inode), 266 false, err, 267 req->r_start_latency, 268 req->r_end_latency); 269 subreq->transferred = err; 270 err = 0; 271 } 272 subreq->error = err; 273 trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); 274 netfs_read_subreq_terminated(subreq); 275 iput(req->r_inode); 276 ceph_dec_osd_stopping_blocker(fsc->mdsc); 277 } 278 279 #ifdef CONFIG_CEPH_FS_INLINE_DATA 280 static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) 281 { 282 struct netfs_io_request *rreq = subreq->rreq; 283 struct inode *inode = rreq->inode; 284 struct ceph_mds_reply_info_parsed *rinfo; 285 struct ceph_mds_reply_info_in *iinfo; 286 struct ceph_mds_request *req; 287 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); 288 ssize_t err = 0; 289 size_t len; 290 int mode; 291 292 if (rreq->origin != NETFS_UNBUFFERED_READ && 293 rreq->origin != NETFS_DIO_READ) 294 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); 295 __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); 296 297 if (subreq->start >= inode->i_size) 298 goto out; 299 300 /* We need to fetch the inline data. */ 301 mode = ceph_try_to_choose_auth_mds(inode, CEPH_STAT_CAP_INLINE_DATA); 302 req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, mode); 303 if (IS_ERR(req)) { 304 err = PTR_ERR(req); 305 goto out; 306 } 307 req->r_ino1 = ceph_vino(inode); 308 req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA); 309 req->r_num_caps = 2; 310 311 trace_netfs_sreq(subreq, netfs_sreq_trace_submit); 312 err = ceph_mdsc_do_request(mdsc, NULL, req); 313 if (err < 0) 314 goto out; 315 316 rinfo = &req->r_reply_info; 317 iinfo = &rinfo->targeti; 318 if (iinfo->inline_version == CEPH_INLINE_NONE) { 319 /* The data got uninlined */ 320 ceph_mdsc_put_request(req); 321 return false; 322 } 323 324 if (subreq->start >= iinfo->inline_len) { 325 ceph_mdsc_put_request(req); 326 err = -ENODATA; 327 goto out; 328 } 329 330 len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len); 331 err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter); 332 if (err == 0) { 333 err = -EFAULT; 334 } else { 335 subreq->transferred += err; 336 err = 0; 337 } 338 339 ceph_mdsc_put_request(req); 340 out: 341 subreq->error = err; 342 trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); 343 netfs_read_subreq_terminated(subreq); 344 return true; 345 } 346 #else 347 static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) 348 { 349 return false; 350 } 351 #endif /* CONFIG_CEPH_FS_INLINE_DATA */ 352 353 static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq) 354 { 355 struct netfs_io_request *rreq = subreq->rreq; 356 struct inode *inode = rreq->inode; 357 struct ceph_inode_info *ci = ceph_inode(inode); 358 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 359 u64 objno, objoff; 360 u32 xlen; 361 362 /* Truncate the extent at the end of the current block */ 363 ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, 364 &objno, &objoff, &xlen); 365 rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize); 366 return 0; 367 } 368 369 static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) 370 { 371 struct netfs_io_request *rreq = subreq->rreq; 372 struct inode *inode = rreq->inode; 373 struct ceph_inode_info *ci = ceph_inode(inode); 374 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 375 struct ceph_client *cl = fsc->client; 376 struct ceph_osd_request *req = NULL; 377 struct ceph_vino vino = ceph_vino(inode); 378 int err = 0; 379 u64 len; 380 bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); 381 u64 off = subreq->start; 382 int extent_cnt; 383 384 if (ceph_inode_is_shutdown(inode)) { 385 err = -EIO; 386 goto out; 387 } 388 389 if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq)) 390 return; 391 392 // TODO: This rounding here is slightly dodgy. It *should* work, for 393 // now, as the cache only deals in blocks that are a multiple of 394 // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to 395 // happen is for the fscrypt driving to be moved into netfslib and the 396 // data in the cache also to be stored encrypted. 397 len = subreq->len; 398 ceph_fscrypt_adjust_off_and_len(inode, &off, &len); 399 400 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, 401 off, &len, 0, 1, sparse ? CEPH_OSD_OP_SPARSE_READ : CEPH_OSD_OP_READ, 402 CEPH_OSD_FLAG_READ, NULL, ci->i_truncate_seq, 403 ci->i_truncate_size, false); 404 if (IS_ERR(req)) { 405 err = PTR_ERR(req); 406 req = NULL; 407 goto out; 408 } 409 410 if (sparse) { 411 extent_cnt = __ceph_sparse_read_ext_count(inode, len); 412 err = ceph_alloc_sparse_ext_map(&req->r_ops[0], extent_cnt); 413 if (err) 414 goto out; 415 } 416 417 doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n", 418 ceph_vinop(inode), subreq->start, subreq->len, len); 419 420 /* 421 * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for 422 * encrypted inodes. We'd need infrastructure that handles an iov_iter 423 * instead of page arrays, and we don't have that as of yet. Once the 424 * dust settles on the write helpers and encrypt/decrypt routines for 425 * netfs, we should be able to rework this. 426 */ 427 if (IS_ENCRYPTED(inode)) { 428 struct page **pages; 429 size_t page_off; 430 431 /* 432 * FIXME: io_iter.count needs to be corrected to aligned 433 * length. Otherwise, iov_iter_get_pages_alloc2() operates 434 * with the initial unaligned length value. As a result, 435 * ceph_msg_data_cursor_init() triggers BUG_ON() in the case 436 * if msg->sparse_read_total > msg->data_length. 437 */ 438 subreq->io_iter.count = len; 439 440 err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off); 441 if (err < 0) { 442 doutc(cl, "%llx.%llx failed to allocate pages, %d\n", 443 ceph_vinop(inode), err); 444 goto out; 445 } 446 447 /* should always give us a page-aligned read */ 448 WARN_ON_ONCE(page_off); 449 len = err; 450 err = 0; 451 452 osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, 453 false); 454 } else { 455 osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter); 456 } 457 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { 458 struct ceph_osd_data *osd_data = 459 osd_req_op_extent_osd_data(req, 0); 460 461 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { 462 int num_pages = calc_pages_for(osd_data->alignment, 463 osd_data->length); 464 465 for (int i = 0; i < num_pages; i++) 466 put_page(osd_data->pages[i]); 467 kvfree(osd_data->pages); 468 } 469 err = -EIO; 470 goto out; 471 } 472 req->r_callback = finish_netfs_read; 473 req->r_priv = subreq; 474 req->r_inode = inode; 475 ihold(inode); 476 477 trace_netfs_sreq(subreq, netfs_sreq_trace_submit); 478 ceph_osdc_start_request(req->r_osdc, req); 479 out: 480 ceph_osdc_put_request(req); 481 if (err) { 482 subreq->error = err; 483 netfs_read_subreq_terminated(subreq); 484 } 485 doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err); 486 } 487 488 static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) 489 { 490 struct inode *inode = rreq->inode; 491 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 492 struct ceph_client *cl = ceph_inode_to_client(inode); 493 int got = 0, want = CEPH_CAP_FILE_CACHE; 494 struct ceph_netfs_request_data *priv; 495 int ret = 0; 496 497 /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ 498 __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); 499 500 if (rreq->origin != NETFS_READAHEAD) 501 return 0; 502 503 priv = kzalloc_obj(*priv, GFP_NOFS); 504 if (!priv) 505 return -ENOMEM; 506 507 if (file) { 508 struct ceph_rw_context *rw_ctx; 509 struct ceph_file_info *fi = file->private_data; 510 511 priv->file_ra_pages = file->f_ra.ra_pages; 512 priv->file_ra_disabled = file->f_mode & FMODE_RANDOM; 513 514 rw_ctx = ceph_find_rw_context(fi); 515 if (rw_ctx) { 516 rreq->netfs_priv = priv; 517 return 0; 518 } 519 520 /* If this is a lazy fd, also try to get LAZYIO caps */ 521 if (fi->fmode & CEPH_FILE_MODE_LAZY) 522 want |= CEPH_CAP_FILE_LAZYIO; 523 } 524 525 /* 526 * readahead callers do not necessarily hold Fcb caps 527 * (e.g. fadvise, madvise). 528 */ 529 ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got); 530 if (ret < 0) { 531 doutc(cl, "%llx.%llx, error getting cap\n", ceph_vinop(inode)); 532 goto out; 533 } 534 535 if (!(got & want)) { 536 doutc(cl, "%llx.%llx, no cache cap\n", ceph_vinop(inode)); 537 ret = -EACCES; 538 goto out; 539 } 540 if (ret == 0) { 541 ret = -EACCES; 542 goto out; 543 } 544 545 priv->caps = got; 546 rreq->netfs_priv = priv; 547 rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize; 548 549 out: 550 if (ret < 0) { 551 if (got) 552 ceph_put_cap_refs(ceph_inode(inode), got); 553 kfree(priv); 554 } 555 556 return ret; 557 } 558 559 static void ceph_netfs_free_request(struct netfs_io_request *rreq) 560 { 561 struct ceph_netfs_request_data *priv = rreq->netfs_priv; 562 563 if (!priv) 564 return; 565 566 if (priv->caps) 567 ceph_put_cap_refs(ceph_inode(rreq->inode), priv->caps); 568 kfree(priv); 569 rreq->netfs_priv = NULL; 570 } 571 572 const struct netfs_request_ops ceph_netfs_ops = { 573 .init_request = ceph_init_request, 574 .free_request = ceph_netfs_free_request, 575 .prepare_read = ceph_netfs_prepare_read, 576 .issue_read = ceph_netfs_issue_read, 577 .expand_readahead = ceph_netfs_expand_readahead, 578 .check_write_begin = ceph_netfs_check_write_begin, 579 }; 580 581 #ifdef CONFIG_CEPH_FSCACHE 582 static void ceph_folio_start_fscache(struct folio *folio) 583 { 584 folio_start_private_2(folio); /* [DEPRECATED] */ 585 } 586 587 static void ceph_folio_wait_fscache(struct folio *folio) 588 { 589 folio_wait_private_2(folio); /* [DEPRECATED] */ 590 } 591 592 static void ceph_fscache_write_terminated(void *priv, ssize_t error) 593 { 594 struct inode *inode = priv; 595 596 if (IS_ERR_VALUE(error) && error != -ENOBUFS) 597 ceph_fscache_invalidate(inode, false); 598 } 599 600 static void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) 601 { 602 struct ceph_inode_info *ci = ceph_inode(inode); 603 struct fscache_cookie *cookie = ceph_fscache_cookie(ci); 604 605 fscache_write_to_cache(cookie, inode->i_mapping, off, len, i_size_read(inode), 606 ceph_fscache_write_terminated, inode, true, caching); 607 } 608 #else 609 static inline void ceph_folio_start_fscache(struct folio *folio) 610 { 611 } 612 613 static inline void ceph_folio_wait_fscache(struct folio *folio) 614 { 615 } 616 617 static inline void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) 618 { 619 } 620 #endif /* CONFIG_CEPH_FSCACHE */ 621 622 struct ceph_writeback_ctl 623 { 624 loff_t i_size; 625 u64 truncate_size; 626 u32 truncate_seq; 627 bool size_stable; 628 629 bool head_snapc; 630 struct ceph_snap_context *snapc; 631 struct ceph_snap_context *last_snapc; 632 633 bool done; 634 bool should_loop; 635 bool range_whole; 636 pgoff_t start_index; 637 pgoff_t index; 638 pgoff_t end; 639 xa_mark_t tag; 640 641 pgoff_t strip_unit_end; 642 unsigned int wsize; 643 unsigned int nr_folios; 644 unsigned int max_pages; 645 unsigned int locked_pages; 646 647 int op_idx; 648 int num_ops; 649 u64 offset; 650 u64 len; 651 652 struct folio_batch fbatch; 653 unsigned int processed_in_fbatch; 654 655 bool from_pool; 656 struct page **pages; 657 struct page **data_pages; 658 }; 659 660 /* 661 * Get ref for the oldest snapc for an inode with dirty data... that is, the 662 * only snap context we are allowed to write back. 663 */ 664 static struct ceph_snap_context * 665 get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, 666 struct ceph_snap_context *page_snapc) 667 { 668 struct ceph_inode_info *ci = ceph_inode(inode); 669 struct ceph_client *cl = ceph_inode_to_client(inode); 670 struct ceph_snap_context *snapc = NULL; 671 struct ceph_cap_snap *capsnap = NULL; 672 673 spin_lock(&ci->i_ceph_lock); 674 list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 675 doutc(cl, " capsnap %p snapc %p has %d dirty pages\n", 676 capsnap, capsnap->context, capsnap->dirty_pages); 677 if (!capsnap->dirty_pages) 678 continue; 679 680 /* get i_size, truncate_{seq,size} for page_snapc? */ 681 if (snapc && capsnap->context != page_snapc) 682 continue; 683 684 if (ctl) { 685 if (capsnap->writing) { 686 ctl->i_size = i_size_read(inode); 687 ctl->size_stable = false; 688 } else { 689 ctl->i_size = capsnap->size; 690 ctl->size_stable = true; 691 } 692 ctl->truncate_size = capsnap->truncate_size; 693 ctl->truncate_seq = capsnap->truncate_seq; 694 ctl->head_snapc = false; 695 } 696 697 if (snapc) 698 break; 699 700 snapc = ceph_get_snap_context(capsnap->context); 701 if (!page_snapc || 702 page_snapc == snapc || 703 page_snapc->seq > snapc->seq) 704 break; 705 } 706 if (!snapc && ci->i_wrbuffer_ref_head) { 707 snapc = ceph_get_snap_context(ci->i_head_snapc); 708 doutc(cl, " head snapc %p has %d dirty pages\n", snapc, 709 ci->i_wrbuffer_ref_head); 710 if (ctl) { 711 ctl->i_size = i_size_read(inode); 712 ctl->truncate_size = ci->i_truncate_size; 713 ctl->truncate_seq = ci->i_truncate_seq; 714 ctl->size_stable = false; 715 ctl->head_snapc = true; 716 } 717 } 718 spin_unlock(&ci->i_ceph_lock); 719 return snapc; 720 } 721 722 static u64 get_writepages_data_length(struct inode *inode, 723 struct folio *folio, u64 start) 724 { 725 struct ceph_inode_info *ci = ceph_inode(inode); 726 struct ceph_snap_context *snapc; 727 struct ceph_cap_snap *capsnap = NULL; 728 u64 end = i_size_read(inode); 729 u64 ret; 730 731 snapc = ceph_folio_snap_context(ceph_fscrypt_pagecache_folio(folio)); 732 if (snapc != ci->i_head_snapc) { 733 bool found = false; 734 spin_lock(&ci->i_ceph_lock); 735 list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 736 if (capsnap->context == snapc) { 737 if (!capsnap->writing) 738 end = capsnap->size; 739 found = true; 740 break; 741 } 742 } 743 spin_unlock(&ci->i_ceph_lock); 744 WARN_ON(!found); 745 } 746 if (end > ceph_fscrypt_folio_offset(folio) + folio_size(folio)) 747 end = ceph_fscrypt_folio_offset(folio) + folio_size(folio); 748 ret = end > start ? end - start : 0; 749 if (ret && fscrypt_is_bounce_folio(folio)) 750 ret = round_up(ret, CEPH_FSCRYPT_BLOCK_SIZE); 751 return ret; 752 } 753 754 /* 755 * Write a folio, but leave it locked. 756 * 757 * If we get a write error, mark the mapping for error, but still adjust the 758 * dirty page accounting (i.e., folio is no longer dirty). 759 */ 760 static int write_folio_nounlock(struct folio *folio, 761 struct writeback_control *wbc) 762 { 763 struct page *page; 764 struct inode *inode = folio->mapping->host; 765 struct ceph_inode_info *ci = ceph_inode(inode); 766 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 767 struct ceph_client *cl = fsc->client; 768 struct ceph_snap_context *snapc, *oldest; 769 loff_t page_off = folio_pos(folio); 770 int err; 771 loff_t len = folio_size(folio); 772 loff_t wlen; 773 struct ceph_writeback_ctl ceph_wbc; 774 struct ceph_osd_client *osdc = &fsc->client->osdc; 775 struct ceph_osd_request *req; 776 bool caching = ceph_is_cache_enabled(inode); 777 struct page *bounce_page = NULL; 778 779 doutc(cl, "%llx.%llx folio %p idx %lu\n", ceph_vinop(inode), folio, 780 folio->index); 781 782 if (ceph_inode_is_shutdown(inode)) 783 return -EIO; 784 785 /* verify this is a writeable snap context */ 786 snapc = ceph_folio_snap_context(folio); 787 if (!snapc) { 788 doutc(cl, "%llx.%llx folio %p not dirty?\n", ceph_vinop(inode), 789 folio); 790 return 0; 791 } 792 oldest = get_oldest_context(inode, &ceph_wbc, snapc); 793 if (snapc->seq > oldest->seq) { 794 doutc(cl, "%llx.%llx folio %p snapc %p not writeable - noop\n", 795 ceph_vinop(inode), folio, snapc); 796 /* we should only noop if called by kswapd */ 797 WARN_ON(!(current->flags & PF_MEMALLOC)); 798 ceph_put_snap_context(oldest); 799 folio_redirty_for_writepage(wbc, folio); 800 return 0; 801 } 802 ceph_put_snap_context(oldest); 803 804 /* is this a partial page at end of file? */ 805 if (page_off >= ceph_wbc.i_size) { 806 doutc(cl, "%llx.%llx folio at %lu beyond eof %llu\n", 807 ceph_vinop(inode), folio->index, ceph_wbc.i_size); 808 folio_invalidate(folio, 0, folio_size(folio)); 809 return 0; 810 } 811 812 if (ceph_wbc.i_size < page_off + len) 813 len = ceph_wbc.i_size - page_off; 814 815 wlen = IS_ENCRYPTED(inode) ? round_up(len, CEPH_FSCRYPT_BLOCK_SIZE) : len; 816 doutc(cl, "%llx.%llx folio %p index %lu on %llu~%llu snapc %p seq %lld\n", 817 ceph_vinop(inode), folio, folio->index, page_off, wlen, snapc, 818 snapc->seq); 819 820 /* wait for a cache write left pending by a previously failed attempt */ 821 ceph_folio_wait_fscache(folio); 822 823 if (atomic_long_inc_return(&fsc->writeback_count) > 824 CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) 825 fsc->write_congested = true; 826 827 req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode), 828 page_off, &wlen, 0, 1, CEPH_OSD_OP_WRITE, 829 CEPH_OSD_FLAG_WRITE, snapc, 830 ceph_wbc.truncate_seq, 831 ceph_wbc.truncate_size, true); 832 if (IS_ERR(req)) { 833 folio_redirty_for_writepage(wbc, folio); 834 if (atomic_long_dec_return(&fsc->writeback_count) < 835 CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 836 fsc->write_congested = false; 837 return PTR_ERR(req); 838 } 839 840 if (wlen < len) 841 len = wlen; 842 843 folio_start_writeback(folio); 844 if (caching) 845 ceph_folio_start_fscache(folio); 846 ceph_fscache_write_to_cache(inode, page_off, len, caching); 847 848 if (IS_ENCRYPTED(inode)) { 849 bounce_page = fscrypt_encrypt_pagecache_blocks(folio, 850 CEPH_FSCRYPT_BLOCK_SIZE, 0, 851 GFP_NOFS); 852 if (IS_ERR(bounce_page)) { 853 folio_redirty_for_writepage(wbc, folio); 854 folio_end_writeback(folio); 855 ceph_osdc_put_request(req); 856 if (atomic_long_dec_return(&fsc->writeback_count) < 857 CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 858 fsc->write_congested = false; 859 return PTR_ERR(bounce_page); 860 } 861 } 862 863 /* it may be a short write due to an object boundary */ 864 WARN_ON_ONCE(len > folio_size(folio)); 865 page = bounce_page ? bounce_page : &folio->page; 866 osd_req_op_extent_osd_data_pages(req, 0, &page, wlen, 0, false, false); 867 doutc(cl, "%llx.%llx %llu~%llu (%llu bytes, %sencrypted)\n", 868 ceph_vinop(inode), page_off, len, wlen, 869 IS_ENCRYPTED(inode) ? "" : "not "); 870 871 req->r_mtime = inode_get_mtime(inode); 872 ceph_osdc_start_request(osdc, req); 873 err = ceph_osdc_wait_request(osdc, req); 874 875 ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, 876 req->r_end_latency, len, err); 877 if (err >= 0 && len > 0) 878 ceph_subvolume_metrics_record_io(fsc->mdsc, ci, true, len, 879 req->r_start_latency, 880 req->r_end_latency); 881 fscrypt_free_bounce_page(bounce_page); 882 ceph_osdc_put_request(req); 883 if (err == 0) 884 err = len; 885 886 if (err < 0) { 887 struct writeback_control tmp_wbc; 888 if (!wbc) 889 wbc = &tmp_wbc; 890 if (err == -ERESTARTSYS) { 891 /* killed by SIGKILL */ 892 doutc(cl, "%llx.%llx interrupted page %p\n", 893 ceph_vinop(inode), folio); 894 folio_redirty_for_writepage(wbc, folio); 895 folio_end_writeback(folio); 896 if (atomic_long_dec_return(&fsc->writeback_count) < 897 CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 898 fsc->write_congested = false; 899 return err; 900 } 901 if (err == -EBLOCKLISTED) 902 fsc->blocklisted = true; 903 doutc(cl, "%llx.%llx setting mapping error %d %p\n", 904 ceph_vinop(inode), err, folio); 905 mapping_set_error(&inode->i_data, err); 906 wbc->pages_skipped++; 907 } else { 908 doutc(cl, "%llx.%llx cleaned page %p\n", 909 ceph_vinop(inode), folio); 910 err = 0; /* vfs expects us to return 0 */ 911 } 912 oldest = folio_detach_private(folio); 913 WARN_ON_ONCE(oldest != snapc); 914 folio_end_writeback(folio); 915 ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 916 ceph_put_snap_context(snapc); /* page's reference */ 917 918 if (atomic_long_dec_return(&fsc->writeback_count) < 919 CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 920 fsc->write_congested = false; 921 922 return err; 923 } 924 925 /* 926 * async writeback completion handler. 927 * 928 * If we get an error, set the mapping error bit, but not the individual 929 * page error bits. 930 */ 931 static void writepages_finish(struct ceph_osd_request *req) 932 { 933 struct inode *inode = req->r_inode; 934 struct ceph_inode_info *ci = ceph_inode(inode); 935 struct ceph_client *cl = ceph_inode_to_client(inode); 936 struct ceph_osd_data *osd_data; 937 int num_pages, total_pages = 0; 938 int i, j; 939 int rc = req->r_result; 940 struct ceph_snap_context *snapc = req->r_snapc; 941 struct address_space *mapping = inode->i_mapping; 942 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 943 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); 944 unsigned int len = 0; 945 bool remove_page; 946 947 doutc(cl, "%llx.%llx rc %d\n", ceph_vinop(inode), rc); 948 if (rc < 0) { 949 mapping_set_error(mapping, rc); 950 ceph_set_error_write(ci); 951 if (rc == -EBLOCKLISTED) 952 fsc->blocklisted = true; 953 } else { 954 ceph_clear_error_write(ci); 955 } 956 957 /* 958 * We lost the cache cap, need to truncate the page before 959 * it is unlocked, otherwise we'd truncate it later in the 960 * page truncation thread, possibly losing some data that 961 * raced its way in 962 */ 963 remove_page = !(ceph_caps_issued(ci) & 964 (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); 965 966 /* clean all pages */ 967 for (i = 0; i < req->r_num_ops; i++) { 968 if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) { 969 pr_warn_client(cl, 970 "%llx.%llx incorrect op %d req %p index %d tid %llu\n", 971 ceph_vinop(inode), req->r_ops[i].op, req, i, 972 req->r_tid); 973 break; 974 } 975 976 osd_data = osd_req_op_extent_osd_data(req, i); 977 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 978 len += osd_data->length; 979 num_pages = calc_pages_for((u64)osd_data->alignment, 980 (u64)osd_data->length); 981 total_pages += num_pages; 982 for (j = 0; j < num_pages; j++) { 983 struct folio *folio = page_folio(osd_data->pages[j]); 984 985 if (fscrypt_is_bounce_folio(folio)) { 986 folio = fscrypt_pagecache_folio(folio); 987 fscrypt_free_bounce_page(osd_data->pages[j]); 988 osd_data->pages[j] = &folio->page; 989 } 990 WARN_ON(!folio_test_uptodate(folio)); 991 992 if (atomic_long_dec_return(&fsc->writeback_count) < 993 CONGESTION_OFF_THRESH( 994 fsc->mount_options->congestion_kb)) 995 fsc->write_congested = false; 996 997 ceph_put_snap_context(folio_detach_private(folio)); 998 folio_end_writeback(folio); 999 1000 if (atomic64_dec_return(&mdsc->dirty_folios) <= 0) { 1001 wake_up_all(&mdsc->flush_end_wq); 1002 WARN_ON(atomic64_read(&mdsc->dirty_folios) < 0); 1003 } 1004 1005 doutc(cl, "unlocking %p\n", folio); 1006 1007 if (remove_page) 1008 generic_error_remove_folio(inode->i_mapping, 1009 folio); 1010 1011 folio_unlock(folio); 1012 } 1013 doutc(cl, "%llx.%llx wrote %llu bytes cleaned %d pages\n", 1014 ceph_vinop(inode), osd_data->length, 1015 rc >= 0 ? num_pages : 0); 1016 1017 release_pages(osd_data->pages, num_pages); 1018 } 1019 1020 ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, 1021 req->r_end_latency, len, rc); 1022 1023 if (rc >= 0 && len > 0) 1024 ceph_subvolume_metrics_record_io(mdsc, ci, true, len, 1025 req->r_start_latency, 1026 req->r_end_latency); 1027 1028 ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); 1029 1030 osd_data = osd_req_op_extent_osd_data(req, 0); 1031 if (osd_data->pages_from_pool) 1032 mempool_free(osd_data->pages, ceph_wb_pagevec_pool); 1033 else 1034 kfree(osd_data->pages); 1035 ceph_osdc_put_request(req); 1036 ceph_dec_osd_stopping_blocker(fsc->mdsc); 1037 } 1038 1039 static inline 1040 bool is_forced_umount(struct address_space *mapping) 1041 { 1042 struct inode *inode = mapping->host; 1043 struct ceph_inode_info *ci = ceph_inode(inode); 1044 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1045 struct ceph_client *cl = fsc->client; 1046 1047 if (ceph_inode_is_shutdown(inode)) { 1048 if (ci->i_wrbuffer_ref > 0) { 1049 pr_warn_ratelimited_client(cl, 1050 "%llx.%llx %lld forced umount\n", 1051 ceph_vinop(inode), ceph_ino(inode)); 1052 } 1053 mapping_set_error(mapping, -EIO); 1054 return true; 1055 } 1056 1057 return false; 1058 } 1059 1060 static inline 1061 unsigned int ceph_define_write_size(struct address_space *mapping) 1062 { 1063 struct inode *inode = mapping->host; 1064 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1065 struct ceph_inode_info *ci = ceph_inode(inode); 1066 unsigned int wsize = ci->i_layout.stripe_unit; 1067 1068 if (fsc->mount_options->wsize < wsize) 1069 wsize = fsc->mount_options->wsize; 1070 1071 return wsize; 1072 } 1073 1074 static inline 1075 void ceph_folio_batch_init(struct ceph_writeback_ctl *ceph_wbc) 1076 { 1077 folio_batch_init(&ceph_wbc->fbatch); 1078 ceph_wbc->processed_in_fbatch = 0; 1079 } 1080 1081 static inline 1082 void ceph_folio_batch_reinit(struct ceph_writeback_ctl *ceph_wbc) 1083 { 1084 folio_batch_release(&ceph_wbc->fbatch); 1085 ceph_folio_batch_init(ceph_wbc); 1086 } 1087 1088 static inline 1089 void ceph_init_writeback_ctl(struct address_space *mapping, 1090 struct writeback_control *wbc, 1091 struct ceph_writeback_ctl *ceph_wbc) 1092 { 1093 ceph_wbc->snapc = NULL; 1094 ceph_wbc->last_snapc = NULL; 1095 1096 ceph_wbc->strip_unit_end = 0; 1097 ceph_wbc->wsize = ceph_define_write_size(mapping); 1098 1099 ceph_wbc->nr_folios = 0; 1100 ceph_wbc->max_pages = 0; 1101 ceph_wbc->locked_pages = 0; 1102 1103 ceph_wbc->done = false; 1104 ceph_wbc->should_loop = false; 1105 ceph_wbc->range_whole = false; 1106 1107 ceph_wbc->start_index = wbc->range_cyclic ? mapping->writeback_index : 0; 1108 ceph_wbc->index = ceph_wbc->start_index; 1109 ceph_wbc->end = -1; 1110 1111 ceph_wbc->tag = wbc_to_tag(wbc); 1112 1113 ceph_wbc->op_idx = -1; 1114 ceph_wbc->num_ops = 0; 1115 ceph_wbc->offset = 0; 1116 ceph_wbc->len = 0; 1117 ceph_wbc->from_pool = false; 1118 1119 ceph_folio_batch_init(ceph_wbc); 1120 1121 ceph_wbc->pages = NULL; 1122 ceph_wbc->data_pages = NULL; 1123 } 1124 1125 static inline 1126 int ceph_define_writeback_range(struct address_space *mapping, 1127 struct writeback_control *wbc, 1128 struct ceph_writeback_ctl *ceph_wbc) 1129 { 1130 struct inode *inode = mapping->host; 1131 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1132 struct ceph_client *cl = fsc->client; 1133 1134 /* find oldest snap context with dirty data */ 1135 ceph_wbc->snapc = get_oldest_context(inode, ceph_wbc, NULL); 1136 if (!ceph_wbc->snapc) { 1137 /* hmm, why does writepages get called when there 1138 is no dirty data? */ 1139 doutc(cl, " no snap context with dirty data?\n"); 1140 return -ENODATA; 1141 } 1142 1143 doutc(cl, " oldest snapc is %p seq %lld (%d snaps)\n", 1144 ceph_wbc->snapc, ceph_wbc->snapc->seq, 1145 ceph_wbc->snapc->num_snaps); 1146 1147 ceph_wbc->should_loop = false; 1148 1149 if (ceph_wbc->head_snapc && ceph_wbc->snapc != ceph_wbc->last_snapc) { 1150 /* where to start/end? */ 1151 if (wbc->range_cyclic) { 1152 ceph_wbc->index = ceph_wbc->start_index; 1153 ceph_wbc->end = -1; 1154 if (ceph_wbc->index > 0) 1155 ceph_wbc->should_loop = true; 1156 doutc(cl, " cyclic, start at %lu\n", ceph_wbc->index); 1157 } else { 1158 ceph_wbc->index = wbc->range_start >> PAGE_SHIFT; 1159 ceph_wbc->end = wbc->range_end >> PAGE_SHIFT; 1160 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 1161 ceph_wbc->range_whole = true; 1162 doutc(cl, " not cyclic, %lu to %lu\n", 1163 ceph_wbc->index, ceph_wbc->end); 1164 } 1165 } else if (!ceph_wbc->head_snapc) { 1166 /* Do not respect wbc->range_{start,end}. Dirty pages 1167 * in that range can be associated with newer snapc. 1168 * They are not writeable until we write all dirty pages 1169 * associated with 'snapc' get written */ 1170 if (ceph_wbc->index > 0) 1171 ceph_wbc->should_loop = true; 1172 doutc(cl, " non-head snapc, range whole\n"); 1173 } 1174 1175 ceph_put_snap_context(ceph_wbc->last_snapc); 1176 ceph_wbc->last_snapc = ceph_wbc->snapc; 1177 1178 return 0; 1179 } 1180 1181 static inline 1182 bool has_writeback_done(struct ceph_writeback_ctl *ceph_wbc) 1183 { 1184 return ceph_wbc->done && ceph_wbc->index > ceph_wbc->end; 1185 } 1186 1187 static inline 1188 bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc, 1189 unsigned index) 1190 { 1191 return index < ceph_wbc->nr_folios && 1192 ceph_wbc->locked_pages < ceph_wbc->max_pages; 1193 } 1194 1195 static 1196 int ceph_check_folio_before_write(struct address_space *mapping, 1197 struct writeback_control *wbc, 1198 struct ceph_writeback_ctl *ceph_wbc, 1199 struct folio *folio) 1200 { 1201 struct inode *inode = mapping->host; 1202 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1203 struct ceph_client *cl = fsc->client; 1204 struct ceph_snap_context *pgsnapc; 1205 1206 /* only dirty folios, or our accounting breaks */ 1207 if (unlikely(!folio_test_dirty(folio) || folio->mapping != mapping)) { 1208 doutc(cl, "!dirty or !mapping %p\n", folio); 1209 return -ENODATA; 1210 } 1211 1212 /* only if matching snap context */ 1213 pgsnapc = ceph_folio_snap_context(folio); 1214 if (pgsnapc != ceph_wbc->snapc) { 1215 doutc(cl, "folio snapc %p %lld != oldest %p %lld\n", 1216 pgsnapc, pgsnapc->seq, 1217 ceph_wbc->snapc, ceph_wbc->snapc->seq); 1218 1219 if (!ceph_wbc->should_loop && !ceph_wbc->head_snapc && 1220 wbc->sync_mode != WB_SYNC_NONE) 1221 ceph_wbc->should_loop = true; 1222 1223 return -ENODATA; 1224 } 1225 1226 if (folio_pos(folio) >= ceph_wbc->i_size) { 1227 doutc(cl, "folio at %lu beyond eof %llu\n", 1228 folio->index, ceph_wbc->i_size); 1229 1230 if ((ceph_wbc->size_stable || 1231 folio_pos(folio) >= i_size_read(inode)) && 1232 folio_clear_dirty_for_io(folio)) 1233 folio_invalidate(folio, 0, folio_size(folio)); 1234 1235 return -ENODATA; 1236 } 1237 1238 if (ceph_wbc->strip_unit_end && 1239 (folio->index > ceph_wbc->strip_unit_end)) { 1240 doutc(cl, "end of strip unit %p\n", folio); 1241 return -E2BIG; 1242 } 1243 1244 return 0; 1245 } 1246 1247 static inline 1248 void __ceph_allocate_page_array(struct ceph_writeback_ctl *ceph_wbc, 1249 unsigned int max_pages) 1250 { 1251 ceph_wbc->pages = kmalloc_objs(*ceph_wbc->pages, max_pages, GFP_NOFS); 1252 if (!ceph_wbc->pages) { 1253 ceph_wbc->from_pool = true; 1254 ceph_wbc->pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); 1255 BUG_ON(!ceph_wbc->pages); 1256 } 1257 } 1258 1259 static inline 1260 void ceph_allocate_page_array(struct address_space *mapping, 1261 struct ceph_writeback_ctl *ceph_wbc, 1262 struct folio *folio) 1263 { 1264 struct inode *inode = mapping->host; 1265 struct ceph_inode_info *ci = ceph_inode(inode); 1266 u64 objnum; 1267 u64 objoff; 1268 u32 xlen; 1269 1270 /* prepare async write request */ 1271 ceph_wbc->offset = (u64)folio_pos(folio); 1272 ceph_calc_file_object_mapping(&ci->i_layout, 1273 ceph_wbc->offset, ceph_wbc->wsize, 1274 &objnum, &objoff, &xlen); 1275 1276 ceph_wbc->num_ops = 1; 1277 ceph_wbc->strip_unit_end = folio->index + ((xlen - 1) >> PAGE_SHIFT); 1278 1279 BUG_ON(ceph_wbc->pages); 1280 ceph_wbc->max_pages = calc_pages_for(0, (u64)xlen); 1281 __ceph_allocate_page_array(ceph_wbc, ceph_wbc->max_pages); 1282 1283 ceph_wbc->len = 0; 1284 } 1285 1286 static inline 1287 bool is_folio_index_contiguous(const struct ceph_writeback_ctl *ceph_wbc, 1288 const struct folio *folio) 1289 { 1290 return folio->index == (ceph_wbc->offset + ceph_wbc->len) >> PAGE_SHIFT; 1291 } 1292 1293 static inline 1294 bool is_num_ops_too_big(struct ceph_writeback_ctl *ceph_wbc) 1295 { 1296 return ceph_wbc->num_ops >= 1297 (ceph_wbc->from_pool ? CEPH_OSD_SLAB_OPS : CEPH_OSD_MAX_OPS); 1298 } 1299 1300 static inline 1301 bool is_write_congestion_happened(struct ceph_fs_client *fsc) 1302 { 1303 return atomic_long_inc_return(&fsc->writeback_count) > 1304 CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb); 1305 } 1306 1307 static inline int move_dirty_folio_in_page_array(struct address_space *mapping, 1308 struct writeback_control *wbc, 1309 struct ceph_writeback_ctl *ceph_wbc, struct folio *folio) 1310 { 1311 struct inode *inode = mapping->host; 1312 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1313 struct ceph_client *cl = fsc->client; 1314 struct page **pages = ceph_wbc->pages; 1315 unsigned int index = ceph_wbc->locked_pages; 1316 gfp_t gfp_flags = ceph_wbc->locked_pages ? GFP_NOWAIT : GFP_NOFS; 1317 1318 if (IS_ENCRYPTED(inode)) { 1319 pages[index] = fscrypt_encrypt_pagecache_blocks(folio, 1320 PAGE_SIZE, 1321 0, 1322 gfp_flags); 1323 if (IS_ERR(pages[index])) { 1324 int err = PTR_ERR(pages[index]); 1325 1326 if (err == -EINVAL) { 1327 pr_err_client(cl, "inode->i_blkbits=%hhu\n", 1328 inode->i_blkbits); 1329 } 1330 1331 /* better not fail on first page! */ 1332 BUG_ON(ceph_wbc->locked_pages == 0); 1333 1334 pages[index] = NULL; 1335 return err; 1336 } 1337 } else { 1338 pages[index] = &folio->page; 1339 } 1340 1341 ceph_wbc->locked_pages++; 1342 1343 return 0; 1344 } 1345 1346 static 1347 void ceph_process_folio_batch(struct address_space *mapping, 1348 struct writeback_control *wbc, 1349 struct ceph_writeback_ctl *ceph_wbc) 1350 { 1351 struct inode *inode = mapping->host; 1352 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1353 struct ceph_client *cl = fsc->client; 1354 struct folio *folio = NULL; 1355 unsigned i; 1356 int rc; 1357 1358 for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) { 1359 folio = ceph_wbc->fbatch.folios[i]; 1360 1361 if (!folio) 1362 continue; 1363 1364 doutc(cl, "? %p idx %lu, folio_test_writeback %#x, " 1365 "folio_test_dirty %#x, folio_test_locked %#x\n", 1366 folio, folio->index, folio_test_writeback(folio), 1367 folio_test_dirty(folio), 1368 folio_test_locked(folio)); 1369 1370 if (folio_test_writeback(folio) || 1371 folio_test_private_2(folio) /* [DEPRECATED] */) { 1372 doutc(cl, "waiting on writeback %p\n", folio); 1373 folio_wait_writeback(folio); 1374 folio_wait_private_2(folio); /* [DEPRECATED] */ 1375 continue; 1376 } 1377 1378 if (ceph_wbc->locked_pages == 0) 1379 folio_lock(folio); 1380 else if (!folio_trylock(folio)) 1381 break; 1382 1383 rc = ceph_check_folio_before_write(mapping, wbc, 1384 ceph_wbc, folio); 1385 if (rc == -ENODATA) { 1386 folio_unlock(folio); 1387 folio_put(folio); 1388 ceph_wbc->fbatch.folios[i] = NULL; 1389 continue; 1390 } else if (rc == -E2BIG) { 1391 folio_unlock(folio); 1392 break; 1393 } 1394 1395 if (!folio_clear_dirty_for_io(folio)) { 1396 doutc(cl, "%p !folio_clear_dirty_for_io\n", folio); 1397 folio_unlock(folio); 1398 folio_put(folio); 1399 ceph_wbc->fbatch.folios[i] = NULL; 1400 continue; 1401 } 1402 1403 /* 1404 * We have something to write. If this is 1405 * the first locked page this time through, 1406 * calculate max possible write size and 1407 * allocate a page array 1408 */ 1409 if (ceph_wbc->locked_pages == 0) { 1410 ceph_allocate_page_array(mapping, ceph_wbc, folio); 1411 } else if (!is_folio_index_contiguous(ceph_wbc, folio)) { 1412 if (is_num_ops_too_big(ceph_wbc)) { 1413 folio_redirty_for_writepage(wbc, folio); 1414 folio_unlock(folio); 1415 break; 1416 } 1417 1418 ceph_wbc->num_ops++; 1419 ceph_wbc->offset = (u64)folio_pos(folio); 1420 ceph_wbc->len = 0; 1421 } 1422 1423 /* note position of first page in fbatch */ 1424 doutc(cl, "%llx.%llx will write folio %p idx %lu\n", 1425 ceph_vinop(inode), folio, folio->index); 1426 1427 rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, 1428 folio); 1429 if (rc) { 1430 /* Did we just begin a new contiguous op? Nevermind! */ 1431 if (ceph_wbc->len == 0) 1432 ceph_wbc->num_ops--; 1433 1434 folio_redirty_for_writepage(wbc, folio); 1435 folio_unlock(folio); 1436 break; 1437 } 1438 1439 if (is_write_congestion_happened(fsc)) 1440 fsc->write_congested = true; 1441 1442 ceph_wbc->fbatch.folios[i] = NULL; 1443 ceph_wbc->len += folio_size(folio); 1444 } 1445 1446 ceph_wbc->processed_in_fbatch = i; 1447 } 1448 1449 static inline 1450 void ceph_shift_unused_folios_left(struct folio_batch *fbatch) 1451 { 1452 unsigned j, n = 0; 1453 1454 /* shift unused page to beginning of fbatch */ 1455 for (j = 0; j < folio_batch_count(fbatch); j++) { 1456 if (!fbatch->folios[j]) 1457 continue; 1458 1459 if (n < j) { 1460 fbatch->folios[n] = fbatch->folios[j]; 1461 } 1462 1463 n++; 1464 } 1465 1466 fbatch->nr = n; 1467 } 1468 1469 static void ceph_undo_wrbuffer_claim(struct inode *inode, struct folio *folio) 1470 { 1471 struct ceph_snap_context *snapc = folio_detach_private(folio); 1472 1473 if (!snapc) 1474 return; 1475 ceph_put_wrbuffer_cap_refs(ceph_inode(inode), 1, snapc); 1476 ceph_put_snap_context(snapc); 1477 } 1478 1479 static 1480 int ceph_submit_write(struct address_space *mapping, 1481 struct writeback_control *wbc, 1482 struct ceph_writeback_ctl *ceph_wbc) 1483 { 1484 struct inode *inode = mapping->host; 1485 struct ceph_inode_info *ci = ceph_inode(inode); 1486 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1487 struct ceph_client *cl = fsc->client; 1488 struct ceph_vino vino = ceph_vino(inode); 1489 struct ceph_osd_request *req = NULL; 1490 struct folio *folio = NULL; 1491 bool caching = ceph_is_cache_enabled(inode); 1492 u64 offset; 1493 u64 len; 1494 unsigned i; 1495 1496 new_request: 1497 offset = ceph_fscrypt_folio_offset(page_folio(ceph_wbc->pages[0])); 1498 len = ceph_wbc->wsize; 1499 1500 req = ceph_osdc_new_request(&fsc->client->osdc, 1501 &ci->i_layout, vino, 1502 offset, &len, 0, ceph_wbc->num_ops, 1503 CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 1504 ceph_wbc->snapc, ceph_wbc->truncate_seq, 1505 ceph_wbc->truncate_size, false); 1506 if (IS_ERR(req)) { 1507 req = ceph_osdc_new_request(&fsc->client->osdc, 1508 &ci->i_layout, vino, 1509 offset, &len, 0, 1510 min(ceph_wbc->num_ops, 1511 CEPH_OSD_SLAB_OPS), 1512 CEPH_OSD_OP_WRITE, 1513 CEPH_OSD_FLAG_WRITE, 1514 ceph_wbc->snapc, 1515 ceph_wbc->truncate_seq, 1516 ceph_wbc->truncate_size, 1517 true); 1518 BUG_ON(IS_ERR(req)); 1519 } 1520 1521 folio = page_folio(ceph_wbc->pages[ceph_wbc->locked_pages - 1]); 1522 BUG_ON(len < ceph_fscrypt_folio_offset(folio) + folio_size(folio) - offset); 1523 1524 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { 1525 for (i = 0; i < ceph_wbc->locked_pages; i++) { 1526 folio = page_folio(ceph_wbc->pages[i]); 1527 if (fscrypt_is_bounce_folio(folio)) { 1528 folio = fscrypt_pagecache_folio(folio); 1529 fscrypt_free_bounce_page(ceph_wbc->pages[i]); 1530 } 1531 1532 if (atomic_long_dec_return(&fsc->writeback_count) < 1533 CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 1534 fsc->write_congested = false; 1535 1536 ceph_undo_wrbuffer_claim(inode, folio); 1537 folio_redirty_for_writepage(wbc, folio); 1538 folio_unlock(folio); 1539 folio_put(folio); 1540 } 1541 1542 if (ceph_wbc->from_pool) { 1543 mempool_free(ceph_wbc->pages, ceph_wb_pagevec_pool); 1544 ceph_wbc->from_pool = false; 1545 } else { 1546 kfree(ceph_wbc->pages); 1547 } 1548 ceph_wbc->pages = NULL; 1549 ceph_wbc->locked_pages = 0; 1550 1551 ceph_osdc_put_request(req); 1552 return -EIO; 1553 } 1554 1555 req->r_callback = writepages_finish; 1556 req->r_inode = inode; 1557 1558 /* Format the osd request message and submit the write */ 1559 len = 0; 1560 ceph_wbc->data_pages = ceph_wbc->pages; 1561 ceph_wbc->op_idx = 0; 1562 for (i = 0; i < ceph_wbc->locked_pages; i++) { 1563 u64 cur_offset; 1564 1565 folio = ceph_fscrypt_pagecache_folio(page_folio(ceph_wbc->pages[i])); 1566 cur_offset = folio_pos(folio); 1567 1568 /* 1569 * Discontinuity in page range? Ceph can handle that by just passing 1570 * multiple extents in the write op. 1571 */ 1572 if (offset + len != cur_offset) { 1573 /* If it's full, stop here */ 1574 if (ceph_wbc->op_idx + 1 == req->r_num_ops) 1575 break; 1576 1577 /* Kick off an fscache write with what we have so far. */ 1578 ceph_fscache_write_to_cache(inode, offset, len, caching); 1579 1580 /* Start a new extent */ 1581 osd_req_op_extent_dup_last(req, ceph_wbc->op_idx, 1582 cur_offset - offset); 1583 1584 doutc(cl, "got pages at %llu~%llu\n", offset, len); 1585 1586 osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, 1587 ceph_wbc->data_pages, 1588 len, 0, 1589 ceph_wbc->from_pool, 1590 false); 1591 osd_req_op_extent_update(req, ceph_wbc->op_idx, len); 1592 1593 len = 0; 1594 offset = cur_offset; 1595 ceph_wbc->data_pages = ceph_wbc->pages + i; 1596 ceph_wbc->op_idx++; 1597 } 1598 1599 folio_start_writeback(folio); 1600 1601 if (caching) 1602 ceph_folio_start_fscache(folio); 1603 1604 len += folio_size(folio); 1605 } 1606 1607 ceph_fscache_write_to_cache(inode, offset, len, caching); 1608 1609 if (ceph_wbc->size_stable) { 1610 len = min(len, ceph_wbc->i_size - offset); 1611 } else if (i == ceph_wbc->locked_pages) { 1612 /* writepages_finish() clears writeback pages 1613 * according to the data length, so make sure 1614 * data length covers all locked pages */ 1615 u64 min_len = len + 1 - folio_size(folio); 1616 len = get_writepages_data_length(inode, 1617 page_folio(ceph_wbc->pages[i - 1]), 1618 offset); 1619 len = max(len, min_len); 1620 } 1621 1622 if (IS_ENCRYPTED(inode)) 1623 len = round_up(len, CEPH_FSCRYPT_BLOCK_SIZE); 1624 1625 doutc(cl, "got pages at %llu~%llu\n", offset, len); 1626 1627 if (IS_ENCRYPTED(inode) && 1628 ((offset | len) & ~CEPH_FSCRYPT_BLOCK_MASK)) { 1629 pr_warn_client(cl, 1630 "bad encrypted write offset=%lld len=%llu\n", 1631 offset, len); 1632 } 1633 1634 osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, 1635 ceph_wbc->data_pages, len, 1636 0, ceph_wbc->from_pool, false); 1637 osd_req_op_extent_update(req, ceph_wbc->op_idx, len); 1638 1639 BUG_ON(ceph_wbc->op_idx + 1 != req->r_num_ops); 1640 1641 ceph_wbc->from_pool = false; 1642 if (i < ceph_wbc->locked_pages) { 1643 BUG_ON(ceph_wbc->num_ops <= req->r_num_ops); 1644 ceph_wbc->num_ops -= req->r_num_ops; 1645 ceph_wbc->locked_pages -= i; 1646 1647 /* allocate new pages array for next request */ 1648 ceph_wbc->data_pages = ceph_wbc->pages; 1649 __ceph_allocate_page_array(ceph_wbc, ceph_wbc->locked_pages); 1650 memcpy(ceph_wbc->pages, ceph_wbc->data_pages + i, 1651 ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); 1652 memset(ceph_wbc->data_pages + i, 0, 1653 ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); 1654 } else { 1655 BUG_ON(ceph_wbc->num_ops != req->r_num_ops); 1656 /* request message now owns the pages array */ 1657 ceph_wbc->pages = NULL; 1658 } 1659 1660 req->r_mtime = inode_get_mtime(inode); 1661 ceph_osdc_start_request(&fsc->client->osdc, req); 1662 req = NULL; 1663 1664 wbc->nr_to_write -= i; 1665 if (ceph_wbc->pages) 1666 goto new_request; 1667 1668 return 0; 1669 } 1670 1671 static 1672 void ceph_wait_until_current_writes_complete(struct address_space *mapping, 1673 struct writeback_control *wbc, 1674 struct ceph_writeback_ctl *ceph_wbc) 1675 { 1676 struct folio *folio; 1677 unsigned i, nr; 1678 1679 if (wbc->sync_mode != WB_SYNC_NONE && 1680 ceph_wbc->start_index == 0 && /* all dirty pages were checked */ 1681 !ceph_wbc->head_snapc) { 1682 ceph_wbc->index = 0; 1683 1684 while ((ceph_wbc->index <= ceph_wbc->end) && 1685 (nr = filemap_get_folios_tag(mapping, 1686 &ceph_wbc->index, 1687 (pgoff_t)-1, 1688 PAGECACHE_TAG_WRITEBACK, 1689 &ceph_wbc->fbatch))) { 1690 for (i = 0; i < nr; i++) { 1691 folio = ceph_wbc->fbatch.folios[i]; 1692 if (ceph_folio_snap_context(folio) != ceph_wbc->snapc) 1693 continue; 1694 folio_wait_writeback(folio); 1695 } 1696 1697 folio_batch_release(&ceph_wbc->fbatch); 1698 cond_resched(); 1699 } 1700 } 1701 } 1702 1703 /* 1704 * initiate async writeback 1705 */ 1706 static int ceph_writepages_start(struct address_space *mapping, 1707 struct writeback_control *wbc) 1708 { 1709 struct inode *inode = mapping->host; 1710 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 1711 struct ceph_client *cl = fsc->client; 1712 struct ceph_writeback_ctl ceph_wbc; 1713 int rc = 0; 1714 1715 if (wbc->sync_mode == WB_SYNC_NONE && fsc->write_congested) 1716 return 0; 1717 1718 doutc(cl, "%llx.%llx (mode=%s)\n", ceph_vinop(inode), 1719 wbc->sync_mode == WB_SYNC_NONE ? "NONE" : 1720 (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); 1721 1722 if (is_forced_umount(mapping)) { 1723 /* we're in a forced umount, don't write! */ 1724 return -EIO; 1725 } 1726 1727 ceph_init_writeback_ctl(mapping, wbc, &ceph_wbc); 1728 1729 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { 1730 rc = -EIO; 1731 goto out; 1732 } 1733 1734 retry: 1735 rc = ceph_define_writeback_range(mapping, wbc, &ceph_wbc); 1736 if (rc == -ENODATA) { 1737 /* hmm, why does writepages get called when there 1738 is no dirty data? */ 1739 rc = 0; 1740 goto dec_osd_stopping_blocker; 1741 } 1742 1743 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1744 tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end); 1745 1746 while (!has_writeback_done(&ceph_wbc)) { 1747 BUG_ON(ceph_wbc.locked_pages); 1748 BUG_ON(ceph_wbc.pages); 1749 1750 ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT; 1751 1752 get_more_pages: 1753 ceph_folio_batch_reinit(&ceph_wbc); 1754 1755 ceph_wbc.nr_folios = filemap_get_folios_tag(mapping, 1756 &ceph_wbc.index, 1757 ceph_wbc.end, 1758 ceph_wbc.tag, 1759 &ceph_wbc.fbatch); 1760 doutc(cl, "pagevec_lookup_range_tag for tag %#x got %d\n", 1761 ceph_wbc.tag, ceph_wbc.nr_folios); 1762 1763 if (!ceph_wbc.nr_folios && !ceph_wbc.locked_pages) 1764 break; 1765 1766 process_folio_batch: 1767 ceph_process_folio_batch(mapping, wbc, &ceph_wbc); 1768 ceph_shift_unused_folios_left(&ceph_wbc.fbatch); 1769 1770 /* did we get anything? */ 1771 if (!ceph_wbc.locked_pages) 1772 goto release_folios; 1773 1774 if (ceph_wbc.processed_in_fbatch) { 1775 if (folio_batch_count(&ceph_wbc.fbatch) == 0 && 1776 ceph_wbc.locked_pages < ceph_wbc.max_pages) { 1777 doutc(cl, "reached end fbatch, trying for more\n"); 1778 goto get_more_pages; 1779 } 1780 } 1781 1782 rc = ceph_submit_write(mapping, wbc, &ceph_wbc); 1783 if (rc) { 1784 /* the OSD client is being torn down, don't retry */ 1785 folio_batch_release(&ceph_wbc.fbatch); 1786 goto dec_osd_stopping_blocker; 1787 } 1788 1789 ceph_wbc.locked_pages = 0; 1790 ceph_wbc.strip_unit_end = 0; 1791 1792 if (folio_batch_count(&ceph_wbc.fbatch) > 0) { 1793 ceph_wbc.nr_folios = 1794 folio_batch_count(&ceph_wbc.fbatch); 1795 goto process_folio_batch; 1796 } 1797 1798 /* 1799 * We stop writing back only if we are not doing 1800 * integrity sync. In case of integrity sync we have to 1801 * keep going until we have written all the pages 1802 * we tagged for writeback prior to entering this loop. 1803 */ 1804 if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) 1805 ceph_wbc.done = true; 1806 1807 release_folios: 1808 doutc(cl, "folio_batch release on %d folios (%p)\n", 1809 (int)ceph_wbc.fbatch.nr, 1810 ceph_wbc.fbatch.nr ? ceph_wbc.fbatch.folios[0] : NULL); 1811 folio_batch_release(&ceph_wbc.fbatch); 1812 } 1813 1814 if (ceph_wbc.should_loop && !ceph_wbc.done) { 1815 /* more to do; loop back to beginning of file */ 1816 doutc(cl, "looping back to beginning of file\n"); 1817 /* OK even when start_index == 0 */ 1818 ceph_wbc.end = ceph_wbc.start_index - 1; 1819 1820 /* to write dirty pages associated with next snapc, 1821 * we need to wait until current writes complete */ 1822 ceph_wait_until_current_writes_complete(mapping, wbc, &ceph_wbc); 1823 1824 ceph_wbc.start_index = 0; 1825 ceph_wbc.index = 0; 1826 goto retry; 1827 } 1828 1829 if (wbc->range_cyclic || (ceph_wbc.range_whole && wbc->nr_to_write > 0)) 1830 mapping->writeback_index = ceph_wbc.index; 1831 1832 dec_osd_stopping_blocker: 1833 ceph_dec_osd_stopping_blocker(fsc->mdsc); 1834 1835 out: 1836 ceph_put_snap_context(ceph_wbc.last_snapc); 1837 doutc(cl, "%llx.%llx dend - startone, rc = %d\n", ceph_vinop(inode), 1838 rc); 1839 1840 return rc; 1841 } 1842 1843 /* 1844 * See if a given @snapc is either writeable, or already written. 1845 */ 1846 static int context_is_writeable_or_written(struct inode *inode, 1847 struct ceph_snap_context *snapc) 1848 { 1849 struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL); 1850 int ret = !oldest || snapc->seq <= oldest->seq; 1851 1852 ceph_put_snap_context(oldest); 1853 return ret; 1854 } 1855 1856 /** 1857 * ceph_find_incompatible - find an incompatible context and return it 1858 * @folio: folio being dirtied 1859 * 1860 * We are only allowed to write into/dirty a folio if the folio is 1861 * clean, or already dirty within the same snap context. Returns a 1862 * conflicting context if there is one, NULL if there isn't, or a 1863 * negative error code on other errors. 1864 * 1865 * Must be called with folio lock held. 1866 */ 1867 static struct ceph_snap_context * 1868 ceph_find_incompatible(struct folio *folio) 1869 { 1870 struct inode *inode = folio->mapping->host; 1871 struct ceph_client *cl = ceph_inode_to_client(inode); 1872 struct ceph_inode_info *ci = ceph_inode(inode); 1873 1874 if (ceph_inode_is_shutdown(inode)) { 1875 doutc(cl, " %llx.%llx folio %p is shutdown\n", 1876 ceph_vinop(inode), folio); 1877 return ERR_PTR(-ESTALE); 1878 } 1879 1880 for (;;) { 1881 struct ceph_snap_context *snapc, *oldest; 1882 1883 folio_wait_writeback(folio); 1884 1885 snapc = ceph_folio_snap_context(folio); 1886 if (!snapc || snapc == ci->i_head_snapc) 1887 break; 1888 1889 /* 1890 * this folio is already dirty in another (older) snap 1891 * context! is it writeable now? 1892 */ 1893 oldest = get_oldest_context(inode, NULL, NULL); 1894 if (snapc->seq > oldest->seq) { 1895 /* not writeable -- return it for the caller to deal with */ 1896 ceph_put_snap_context(oldest); 1897 doutc(cl, " %llx.%llx folio %p snapc %p not current or oldest\n", 1898 ceph_vinop(inode), folio, snapc); 1899 return ceph_get_snap_context(snapc); 1900 } 1901 ceph_put_snap_context(oldest); 1902 1903 /* yay, writeable, do it now (without dropping folio lock) */ 1904 doutc(cl, " %llx.%llx folio %p snapc %p not current, but oldest\n", 1905 ceph_vinop(inode), folio, snapc); 1906 if (folio_clear_dirty_for_io(folio)) { 1907 int r = write_folio_nounlock(folio, NULL); 1908 if (r < 0) 1909 return ERR_PTR(r); 1910 } 1911 } 1912 return NULL; 1913 } 1914 1915 static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len, 1916 struct folio **foliop, void **_fsdata) 1917 { 1918 struct inode *inode = file_inode(file); 1919 struct ceph_inode_info *ci = ceph_inode(inode); 1920 struct ceph_snap_context *snapc; 1921 1922 snapc = ceph_find_incompatible(*foliop); 1923 if (snapc) { 1924 int r; 1925 1926 folio_unlock(*foliop); 1927 folio_put(*foliop); 1928 *foliop = NULL; 1929 if (IS_ERR(snapc)) 1930 return PTR_ERR(snapc); 1931 1932 ceph_queue_writeback(inode); 1933 r = wait_event_killable(ci->i_cap_wq, 1934 context_is_writeable_or_written(inode, snapc)); 1935 ceph_put_snap_context(snapc); 1936 /* 1937 * *foliop is cleared, so a return of 0 makes 1938 * netfs_write_begin() grab the folio again and call us again 1939 * to recheck it. Any error (fatal signal) fails the write. 1940 */ 1941 return r; 1942 } 1943 return 0; 1944 } 1945 1946 /* 1947 * We are only allowed to write into/dirty the page if the page is 1948 * clean, or already dirty within the same snap context. 1949 */ 1950 static int ceph_write_begin(const struct kiocb *iocb, 1951 struct address_space *mapping, 1952 loff_t pos, unsigned len, 1953 struct folio **foliop, void **fsdata) 1954 { 1955 struct file *file = iocb->ki_filp; 1956 struct inode *inode = file_inode(file); 1957 struct ceph_inode_info *ci = ceph_inode(inode); 1958 int r; 1959 1960 r = netfs_write_begin(&ci->netfs, file, inode->i_mapping, pos, len, foliop, NULL); 1961 if (r < 0) 1962 return r; 1963 1964 folio_wait_private_2(*foliop); /* [DEPRECATED] */ 1965 WARN_ON_ONCE(!folio_test_locked(*foliop)); 1966 return 0; 1967 } 1968 1969 /* 1970 * we don't do anything in here that simple_write_end doesn't do 1971 * except adjust dirty page accounting 1972 */ 1973 static int ceph_write_end(const struct kiocb *iocb, 1974 struct address_space *mapping, loff_t pos, 1975 unsigned len, unsigned copied, 1976 struct folio *folio, void *fsdata) 1977 { 1978 struct file *file = iocb->ki_filp; 1979 struct inode *inode = file_inode(file); 1980 struct ceph_client *cl = ceph_inode_to_client(inode); 1981 bool check_cap = false; 1982 1983 doutc(cl, "%llx.%llx file %p folio %p %d~%d (%d)\n", ceph_vinop(inode), 1984 file, folio, (int)pos, (int)copied, (int)len); 1985 1986 if (!folio_test_uptodate(folio)) { 1987 /* just return that nothing was copied on a short copy */ 1988 if (copied < len) { 1989 copied = 0; 1990 goto out; 1991 } 1992 folio_mark_uptodate(folio); 1993 } 1994 1995 /* did file size increase? */ 1996 if (pos+copied > i_size_read(inode)) 1997 check_cap = ceph_inode_set_size(inode, pos+copied); 1998 1999 folio_mark_dirty(folio); 2000 2001 out: 2002 folio_unlock(folio); 2003 folio_put(folio); 2004 2005 if (check_cap) 2006 ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY); 2007 2008 return copied; 2009 } 2010 2011 const struct address_space_operations ceph_aops = { 2012 .read_folio = netfs_read_folio, 2013 .readahead = netfs_readahead, 2014 .writepages = ceph_writepages_start, 2015 .write_begin = ceph_write_begin, 2016 .write_end = ceph_write_end, 2017 .dirty_folio = ceph_dirty_folio, 2018 .invalidate_folio = ceph_invalidate_folio, 2019 .release_folio = netfs_release_folio, 2020 .direct_IO = noop_direct_IO, 2021 .migrate_folio = filemap_migrate_folio, 2022 }; 2023 2024 static void ceph_block_sigs(sigset_t *oldset) 2025 { 2026 sigset_t mask; 2027 siginitsetinv(&mask, sigmask(SIGKILL)); 2028 sigprocmask(SIG_BLOCK, &mask, oldset); 2029 } 2030 2031 static void ceph_restore_sigs(sigset_t *oldset) 2032 { 2033 sigprocmask(SIG_SETMASK, oldset, NULL); 2034 } 2035 2036 /* 2037 * True if the page cache is only being kept alive by LAZYIO: CACHE is no 2038 * longer present in the implemented cap set, while LAZYIO still is, so 2039 * handle_cap_grant() left the pages in place for the lazy openers. 2040 * 2041 * Such pages are only valid under the lazy semantics, and a non-lazy reader 2042 * must not be served from them. try_get_cap_refs() already refuses to 2043 * satisfy a non-lazy CACHE want with LAZYIO for read_iter()/write_iter(); 2044 * faults need the same gate. 2045 * 2046 * Unlike __prep_cap(), which has to reason about a single cap, the question 2047 * here is whether the cached data can have gone stale, so the aggregate of 2048 * all caps is the right granularity: as long as any one of them still has 2049 * CACHE implemented, its revocation has not been ACKed and the MDS cannot 2050 * have handed CACHE to another client. Note that __ceph_caps_issued() 2051 * reports i_snap_caps in "have" only and not in "implemented", hence the 2052 * OR; snap caps never include LAZYIO, so folding them in can only make this 2053 * more conservative. 2054 */ 2055 static bool ceph_pages_retained_for_lazyio(struct inode *inode) 2056 { 2057 struct ceph_inode_info *ci = ceph_inode(inode); 2058 int have, implemented; 2059 2060 if (!inode->i_data.nrpages) 2061 return false; 2062 2063 spin_lock(&ci->i_ceph_lock); 2064 have = __ceph_caps_issued(ci, &implemented); 2065 implemented |= have; 2066 spin_unlock(&ci->i_ceph_lock); 2067 2068 return !(implemented & CEPH_CAP_FILE_CACHE) && 2069 (implemented & CEPH_CAP_FILE_LAZYIO); 2070 } 2071 2072 /* 2073 * vm ops 2074 */ 2075 static vm_fault_t ceph_filemap_fault(struct vm_fault *vmf) 2076 { 2077 struct vm_area_struct *vma = vmf->vma; 2078 struct inode *inode = file_inode(vma->vm_file); 2079 struct ceph_inode_info *ci = ceph_inode(inode); 2080 struct ceph_client *cl = ceph_inode_to_client(inode); 2081 struct ceph_file_info *fi = vma->vm_file->private_data; 2082 loff_t off = (loff_t)vmf->pgoff << PAGE_SHIFT; 2083 int want, got, err; 2084 sigset_t oldset; 2085 vm_fault_t ret = VM_FAULT_SIGBUS; 2086 2087 if (ceph_inode_is_shutdown(inode)) 2088 return ret; 2089 2090 ceph_block_sigs(&oldset); 2091 2092 doutc(cl, "%llx.%llx %llu trying to get caps\n", 2093 ceph_vinop(inode), off); 2094 if (fi->fmode & CEPH_FILE_MODE_LAZY) 2095 want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; 2096 else 2097 want = CEPH_CAP_FILE_CACHE; 2098 2099 got = 0; 2100 err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_RD, want, -1, &got); 2101 if (err < 0) 2102 goto out_restore; 2103 2104 doutc(cl, "%llx.%llx %llu got cap refs on %s\n", ceph_vinop(inode), 2105 off, ceph_cap_string(got)); 2106 2107 /* 2108 * Don't fault a non-lazy mapping in from page cache that only LAZYIO 2109 * covers. The "every opener is lazy" gate in handle_cap_grant() can't 2110 * catch this on its own: an open that happens after the cache was 2111 * retained sends no cap message, so nothing re-evaluates the decision. 2112 * Drop the range here instead and let the fault below re-read it. 2113 */ 2114 if (!(fi->fmode & CEPH_FILE_MODE_LAZY) && 2115 !(got & CEPH_CAP_FILE_CACHE) && 2116 ceph_pages_retained_for_lazyio(inode)) { 2117 struct address_space *mapping = inode->i_mapping; 2118 2119 doutc(cl, "%llx.%llx %llu dropping LAZYIO page cache\n", 2120 ceph_vinop(inode), off); 2121 err = invalidate_inode_pages2_range(mapping, vmf->pgoff, 2122 vmf->pgoff); 2123 if (err == -EBUSY) { 2124 /* 2125 * The folio is dirty and ceph has no launder_folio(), 2126 * so invalidate_inode_pages2_range() cannot drop it. 2127 * Write it back and retry -- handing a folio that only 2128 * LAZYIO covers to a non-lazy vma is not an option. 2129 */ 2130 err = filemap_write_and_wait_range(mapping, off, 2131 off + PAGE_SIZE - 1); 2132 if (!err) 2133 err = invalidate_inode_pages2_range(mapping, 2134 vmf->pgoff, vmf->pgoff); 2135 } 2136 if (err) { 2137 doutc(cl, "%llx.%llx %llu failed to drop LAZYIO page cache: %d\n", 2138 ceph_vinop(inode), off, err); 2139 ceph_put_cap_refs(ci, got); 2140 goto out_restore; 2141 } 2142 } 2143 2144 if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || 2145 !ceph_has_inline_data(ci)) { 2146 CEPH_DEFINE_RW_CONTEXT(rw_ctx, got); 2147 ceph_add_rw_context(fi, &rw_ctx); 2148 ret = filemap_fault(vmf); 2149 ceph_del_rw_context(fi, &rw_ctx); 2150 doutc(cl, "%llx.%llx %llu drop cap refs %s ret %x\n", 2151 ceph_vinop(inode), off, ceph_cap_string(got), ret); 2152 } else 2153 err = -EAGAIN; 2154 2155 ceph_put_cap_refs(ci, got); 2156 2157 if (err != -EAGAIN) 2158 goto out_restore; 2159 2160 /* read inline data */ 2161 if (off >= PAGE_SIZE) { 2162 /* does not support inline data > PAGE_SIZE */ 2163 ret = VM_FAULT_SIGBUS; 2164 } else { 2165 struct address_space *mapping = inode->i_mapping; 2166 struct folio *folio; 2167 2168 filemap_invalidate_lock_shared(mapping); 2169 folio = __filemap_get_folio(mapping, 0, 2170 FGP_LOCK | FGP_ACCESSED | FGP_CREAT, 2171 mapping_gfp_constraint(mapping, ~__GFP_FS)); 2172 if (IS_ERR(folio)) { 2173 ret = VM_FAULT_OOM; 2174 goto out_inline; 2175 } 2176 err = __ceph_do_getattr(inode, folio, 2177 CEPH_STAT_CAP_INLINE_DATA, true); 2178 if (err < 0 || off >= i_size_read(inode)) { 2179 folio_unlock(folio); 2180 folio_put(folio); 2181 ret = vmf_error(err); 2182 goto out_inline; 2183 } 2184 if (err < folio_size(folio)) 2185 folio_zero_segment(folio, err, folio_size(folio)); 2186 else 2187 flush_dcache_folio(folio); 2188 folio_mark_uptodate(folio); 2189 vmf->page = folio_page(folio, 0); 2190 ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; 2191 out_inline: 2192 filemap_invalidate_unlock_shared(mapping); 2193 doutc(cl, "%llx.%llx %llu read inline data ret %x\n", 2194 ceph_vinop(inode), off, ret); 2195 } 2196 out_restore: 2197 ceph_restore_sigs(&oldset); 2198 if (err < 0) 2199 ret = vmf_error(err); 2200 2201 return ret; 2202 } 2203 2204 /* 2205 * Return true if the MDS has issued us a cap that covers buffered 2206 * dirtying: either BUFFER, or LAZYIO (as long as LAZYIO itself is not 2207 * being revoked). This mirrors the conditions under which 2208 * try_get_cap_refs() will hand out a BUFFER or LAZYIO ref. 2209 */ 2210 static bool ceph_have_dirtyable_caps(struct inode *inode) 2211 { 2212 struct ceph_inode_info *ci = ceph_inode(inode); 2213 int have, implemented; 2214 bool ret = false; 2215 2216 spin_lock(&ci->i_ceph_lock); 2217 have = __ceph_caps_issued(ci, &implemented); 2218 if (have & CEPH_CAP_FILE_BUFFER) { 2219 ret = true; 2220 } else if ((have & CEPH_CAP_FILE_LAZYIO) && 2221 !((implemented & ~have) & CEPH_CAP_FILE_LAZYIO)) { 2222 ret = true; 2223 } 2224 spin_unlock(&ci->i_ceph_lock); 2225 return ret; 2226 } 2227 2228 static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) 2229 { 2230 struct vm_area_struct *vma = vmf->vma; 2231 struct inode *inode = file_inode(vma->vm_file); 2232 struct ceph_client *cl = ceph_inode_to_client(inode); 2233 struct ceph_inode_info *ci = ceph_inode(inode); 2234 struct ceph_file_info *fi = vma->vm_file->private_data; 2235 struct ceph_cap_flush *prealloc_cf; 2236 struct folio *folio = page_folio(vmf->page); 2237 loff_t off = folio_pos(folio); 2238 loff_t size = i_size_read(inode); 2239 size_t len; 2240 int want, got, err; 2241 sigset_t oldset; 2242 vm_fault_t ret = VM_FAULT_SIGBUS; 2243 2244 if (ceph_inode_is_shutdown(inode)) 2245 return ret; 2246 2247 prealloc_cf = ceph_alloc_cap_flush(); 2248 if (!prealloc_cf) 2249 return VM_FAULT_OOM; 2250 2251 sb_start_pagefault(inode->i_sb); 2252 ceph_block_sigs(&oldset); 2253 2254 if (off + folio_size(folio) <= size) 2255 len = folio_size(folio); 2256 else 2257 len = offset_in_folio(folio, size); 2258 2259 doutc(cl, "%llx.%llx %llu~%zd getting caps i_size %llu\n", 2260 ceph_vinop(inode), off, len, size); 2261 if (fi->fmode & CEPH_FILE_MODE_LAZY) 2262 want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; 2263 else 2264 want = CEPH_CAP_FILE_BUFFER; 2265 2266 retry_caps: 2267 got = 0; 2268 err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_WR, want, off + len, &got); 2269 if (err < 0) 2270 goto out_free; 2271 2272 doutc(cl, "%llx.%llx %llu~%zd got cap refs on %s\n", ceph_vinop(inode), 2273 off, len, ceph_cap_string(got)); 2274 2275 /* 2276 * ceph_write_iter() makes the same check and falls back to 2277 * synchronous writes, but a page fault has no such fallback: 2278 * dirtying the folio without BUFFER or LAZYIO would leave dirty 2279 * data uncovered by any issued cap (e.g. while LAZYIO is being 2280 * revoked, or after it has been released). Wait for the MDS to 2281 * (re)grant a covering cap, matching how the exclude gate in 2282 * try_get_cap_refs() blocks buffered writes while BUFFER is 2283 * revoking. 2284 */ 2285 if ((fi->fmode & CEPH_FILE_MODE_LAZY) && 2286 (got & (CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO)) == 0) { 2287 ceph_put_cap_refs(ci, got); 2288 got = 0; 2289 doutc(cl, "%llx.%llx %llu~%zd waiting for BUFFER or LAZYIO\n", 2290 ceph_vinop(inode), off, len); 2291 /* 2292 * Bail out if the inode is being shut down: the caps are gone 2293 * for good, so ceph_have_dirtyable_caps() would never become 2294 * true again. Note that we cannot rely on ceph_get_caps() 2295 * failing with -ESTALE on the retry either -- if WR is still 2296 * held it would succeed without BUFFER or LAZYIO and send us 2297 * straight back here. 2298 */ 2299 err = wait_event_killable(ci->i_cap_wq, 2300 ceph_have_dirtyable_caps(inode) || 2301 ceph_inode_is_shutdown(inode)); 2302 if (err) 2303 goto out_free; 2304 if (ceph_inode_is_shutdown(inode)) { 2305 err = -ESTALE; 2306 goto out_free; 2307 } 2308 goto retry_caps; 2309 } 2310 2311 /* Update time before taking folio lock */ 2312 file_update_time(vma->vm_file); 2313 inode_inc_iversion_raw(inode); 2314 2315 do { 2316 struct ceph_snap_context *snapc; 2317 2318 folio_lock(folio); 2319 2320 if (folio_mkwrite_check_truncate(folio, inode) < 0) { 2321 folio_unlock(folio); 2322 ret = VM_FAULT_NOPAGE; 2323 break; 2324 } 2325 2326 snapc = ceph_find_incompatible(folio); 2327 if (!snapc) { 2328 /* success. we'll keep the folio locked. */ 2329 folio_mark_dirty(folio); 2330 ret = VM_FAULT_LOCKED; 2331 break; 2332 } 2333 2334 folio_unlock(folio); 2335 2336 if (IS_ERR(snapc)) { 2337 ret = VM_FAULT_SIGBUS; 2338 break; 2339 } 2340 2341 ceph_queue_writeback(inode); 2342 err = wait_event_killable(ci->i_cap_wq, 2343 context_is_writeable_or_written(inode, snapc)); 2344 ceph_put_snap_context(snapc); 2345 } while (err == 0); 2346 2347 if (ret == VM_FAULT_LOCKED) { 2348 int dirty; 2349 spin_lock(&ci->i_ceph_lock); 2350 dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, 2351 &prealloc_cf); 2352 spin_unlock(&ci->i_ceph_lock); 2353 if (dirty) 2354 __mark_inode_dirty(inode, dirty); 2355 } 2356 2357 doutc(cl, "%llx.%llx %llu~%zd dropping cap refs on %s ret %x\n", 2358 ceph_vinop(inode), off, len, ceph_cap_string(got), ret); 2359 ceph_put_cap_refs_async(ci, got); 2360 out_free: 2361 ceph_restore_sigs(&oldset); 2362 sb_end_pagefault(inode->i_sb); 2363 ceph_free_cap_flush(prealloc_cf); 2364 if (err < 0) 2365 ret = vmf_error(err); 2366 return ret; 2367 } 2368 2369 #ifdef CONFIG_CEPH_FS_INLINE_DATA 2370 void ceph_fill_inline_data(struct inode *inode, struct folio *locked_folio, 2371 char *data, size_t len) 2372 { 2373 struct ceph_client *cl = ceph_inode_to_client(inode); 2374 struct address_space *mapping = inode->i_mapping; 2375 struct folio *folio; 2376 2377 if (locked_folio) { 2378 folio = locked_folio; 2379 } else { 2380 if (i_size_read(inode) == 0) 2381 return; 2382 folio = __filemap_get_folio(mapping, 0, 2383 FGP_LOCK | FGP_ACCESSED | FGP_CREAT, 2384 mapping_gfp_constraint(mapping, ~__GFP_FS)); 2385 if (IS_ERR(folio)) 2386 return; 2387 if (folio_test_uptodate(folio)) { 2388 folio_unlock(folio); 2389 folio_put(folio); 2390 return; 2391 } 2392 } 2393 2394 doutc(cl, "%p %llx.%llx len %zu locked_folio %p\n", inode, 2395 ceph_vinop(inode), len, locked_folio); 2396 2397 if (len > folio_size(folio)) { 2398 pr_warn_ratelimited_client(cl, "oversized inline data %zu\n", 2399 len); 2400 len = folio_size(folio); 2401 } 2402 2403 if (len > 0) 2404 memcpy_to_folio(folio, 0, data, len); 2405 2406 if (folio != locked_folio) { 2407 if (len < folio_size(folio)) 2408 folio_zero_segment(folio, len, folio_size(folio)); 2409 2410 folio_mark_uptodate(folio); 2411 folio_unlock(folio); 2412 folio_put(folio); 2413 } 2414 } 2415 2416 int ceph_uninline_data(struct file *file) 2417 { 2418 struct inode *inode = file_inode(file); 2419 struct ceph_inode_info *ci = ceph_inode(inode); 2420 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); 2421 struct ceph_client *cl = fsc->client; 2422 struct ceph_osd_request *req = NULL; 2423 struct ceph_cap_flush *prealloc_cf = NULL; 2424 struct folio *folio = NULL; 2425 struct ceph_snap_context *snapc = NULL; 2426 u64 inline_version = CEPH_INLINE_NONE; 2427 struct page *pages[1]; 2428 int err = 0; 2429 u64 len; 2430 2431 spin_lock(&ci->i_ceph_lock); 2432 inline_version = ceph_inline_version(ci); 2433 spin_unlock(&ci->i_ceph_lock); 2434 2435 doutc(cl, "%llx.%llx inline_version %llu\n", ceph_vinop(inode), 2436 inline_version); 2437 2438 if (ceph_inode_is_shutdown(inode)) { 2439 err = -EIO; 2440 goto out; 2441 } 2442 2443 if (inline_version == CEPH_INLINE_NONE) 2444 return 0; 2445 2446 prealloc_cf = ceph_alloc_cap_flush(); 2447 if (!prealloc_cf) 2448 return -ENOMEM; 2449 2450 if (inline_version == 1) /* initial version, no data */ 2451 goto out_uninline; 2452 2453 down_read(&fsc->mdsc->snap_rwsem); 2454 spin_lock(&ci->i_ceph_lock); 2455 if (__ceph_have_pending_cap_snap(ci)) { 2456 struct ceph_cap_snap *capsnap = 2457 list_last_entry(&ci->i_cap_snaps, 2458 struct ceph_cap_snap, 2459 ci_item); 2460 snapc = ceph_get_snap_context(capsnap->context); 2461 } else { 2462 if (!ci->i_head_snapc) { 2463 ci->i_head_snapc = ceph_get_snap_context( 2464 ci->i_snap_realm->cached_context); 2465 } 2466 snapc = ceph_get_snap_context(ci->i_head_snapc); 2467 } 2468 spin_unlock(&ci->i_ceph_lock); 2469 up_read(&fsc->mdsc->snap_rwsem); 2470 2471 folio = read_mapping_folio(inode->i_mapping, 0, file); 2472 if (IS_ERR(folio)) { 2473 err = PTR_ERR(folio); 2474 goto out; 2475 } 2476 2477 folio_lock(folio); 2478 2479 len = i_size_read(inode); 2480 if (len > folio_size(folio)) 2481 len = folio_size(folio); 2482 2483 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 2484 ceph_vino(inode), 0, &len, 0, 1, 2485 CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, 2486 snapc, 0, 0, false); 2487 if (IS_ERR(req)) { 2488 err = PTR_ERR(req); 2489 goto out_unlock; 2490 } 2491 2492 req->r_mtime = inode_get_mtime(inode); 2493 ceph_osdc_start_request(&fsc->client->osdc, req); 2494 err = ceph_osdc_wait_request(&fsc->client->osdc, req); 2495 ceph_osdc_put_request(req); 2496 if (err < 0) 2497 goto out_unlock; 2498 2499 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 2500 ceph_vino(inode), 0, &len, 1, 3, 2501 CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 2502 snapc, ci->i_truncate_seq, 2503 ci->i_truncate_size, false); 2504 if (IS_ERR(req)) { 2505 err = PTR_ERR(req); 2506 goto out_unlock; 2507 } 2508 2509 pages[0] = folio_page(folio, 0); 2510 osd_req_op_extent_osd_data_pages(req, 1, pages, len, 0, false, false); 2511 2512 { 2513 __le64 xattr_buf = cpu_to_le64(inline_version); 2514 err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, 2515 "inline_version", &xattr_buf, 2516 sizeof(xattr_buf), 2517 CEPH_OSD_CMPXATTR_OP_GT, 2518 CEPH_OSD_CMPXATTR_MODE_U64); 2519 if (err) 2520 goto out_put_req; 2521 } 2522 2523 { 2524 char xattr_buf[32]; 2525 int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), 2526 "%llu", inline_version); 2527 err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, 2528 "inline_version", 2529 xattr_buf, xattr_len, 0, 0); 2530 if (err) 2531 goto out_put_req; 2532 } 2533 2534 req->r_mtime = inode_get_mtime(inode); 2535 ceph_osdc_start_request(&fsc->client->osdc, req); 2536 err = ceph_osdc_wait_request(&fsc->client->osdc, req); 2537 2538 ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, 2539 req->r_end_latency, len, err); 2540 2541 out_uninline: 2542 if (!err) { 2543 int dirty; 2544 2545 /* Set to CAP_INLINE_NONE and dirty the caps */ 2546 down_read(&fsc->mdsc->snap_rwsem); 2547 spin_lock(&ci->i_ceph_lock); 2548 ceph_set_inline_version(ci, CEPH_INLINE_NONE); 2549 dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, &prealloc_cf); 2550 spin_unlock(&ci->i_ceph_lock); 2551 up_read(&fsc->mdsc->snap_rwsem); 2552 if (dirty) 2553 __mark_inode_dirty(inode, dirty); 2554 } 2555 out_put_req: 2556 ceph_osdc_put_request(req); 2557 if (err == -ECANCELED) 2558 err = 0; 2559 out_unlock: 2560 if (folio) { 2561 folio_unlock(folio); 2562 folio_put(folio); 2563 } 2564 out: 2565 ceph_put_snap_context(snapc); 2566 ceph_free_cap_flush(prealloc_cf); 2567 doutc(cl, "%llx.%llx inline_version %llu = %d\n", 2568 ceph_vinop(inode), inline_version, err); 2569 return err; 2570 } 2571 #endif /* CONFIG_CEPH_FS_INLINE_DATA */ 2572 2573 static const struct vm_operations_struct ceph_vmops = { 2574 .fault = ceph_filemap_fault, 2575 .page_mkwrite = ceph_page_mkwrite, 2576 }; 2577 2578 int ceph_mmap_prepare(struct vm_area_desc *desc) 2579 { 2580 struct address_space *mapping = desc->file->f_mapping; 2581 2582 if (!mapping->a_ops->read_folio) 2583 return -ENOEXEC; 2584 desc->vm_ops = &ceph_vmops; 2585 return 0; 2586 } 2587 2588 enum { 2589 POOL_READ = 1, 2590 POOL_WRITE = 2, 2591 }; 2592 2593 static int __ceph_pool_perm_get(struct ceph_inode_info *ci, 2594 s64 pool, struct ceph_string *pool_ns) 2595 { 2596 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(&ci->netfs.inode); 2597 struct ceph_mds_client *mdsc = fsc->mdsc; 2598 struct ceph_client *cl = fsc->client; 2599 struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; 2600 struct rb_node **p, *parent; 2601 struct ceph_pool_perm *perm; 2602 struct page **pages; 2603 size_t pool_ns_len; 2604 int err = 0, err2 = 0, have = 0; 2605 2606 down_read(&mdsc->pool_perm_rwsem); 2607 p = &mdsc->pool_perm_tree.rb_node; 2608 while (*p) { 2609 perm = rb_entry(*p, struct ceph_pool_perm, node); 2610 if (pool < perm->pool) 2611 p = &(*p)->rb_left; 2612 else if (pool > perm->pool) 2613 p = &(*p)->rb_right; 2614 else { 2615 int ret = ceph_compare_string(pool_ns, 2616 perm->pool_ns, 2617 perm->pool_ns_len); 2618 if (ret < 0) 2619 p = &(*p)->rb_left; 2620 else if (ret > 0) 2621 p = &(*p)->rb_right; 2622 else { 2623 have = perm->perm; 2624 break; 2625 } 2626 } 2627 } 2628 up_read(&mdsc->pool_perm_rwsem); 2629 if (*p) 2630 goto out; 2631 2632 if (pool_ns) 2633 doutc(cl, "pool %lld ns %.*s no perm cached\n", pool, 2634 (int)pool_ns->len, pool_ns->str); 2635 else 2636 doutc(cl, "pool %lld no perm cached\n", pool); 2637 2638 down_write(&mdsc->pool_perm_rwsem); 2639 p = &mdsc->pool_perm_tree.rb_node; 2640 parent = NULL; 2641 while (*p) { 2642 parent = *p; 2643 perm = rb_entry(parent, struct ceph_pool_perm, node); 2644 if (pool < perm->pool) 2645 p = &(*p)->rb_left; 2646 else if (pool > perm->pool) 2647 p = &(*p)->rb_right; 2648 else { 2649 int ret = ceph_compare_string(pool_ns, 2650 perm->pool_ns, 2651 perm->pool_ns_len); 2652 if (ret < 0) 2653 p = &(*p)->rb_left; 2654 else if (ret > 0) 2655 p = &(*p)->rb_right; 2656 else { 2657 have = perm->perm; 2658 break; 2659 } 2660 } 2661 } 2662 if (*p) { 2663 up_write(&mdsc->pool_perm_rwsem); 2664 goto out; 2665 } 2666 2667 rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 2668 1, false, GFP_KERNEL); 2669 if (!rd_req) { 2670 err = -ENOMEM; 2671 goto out_unlock; 2672 } 2673 2674 rd_req->r_flags = CEPH_OSD_FLAG_READ; 2675 osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); 2676 rd_req->r_base_oloc.pool = pool; 2677 if (pool_ns) 2678 rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns); 2679 ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ceph_ino(&ci->netfs.inode)); 2680 2681 err = ceph_osdc_alloc_messages(rd_req, GFP_KERNEL); 2682 if (err) 2683 goto out_unlock; 2684 2685 wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 2686 1, false, GFP_KERNEL); 2687 if (!wr_req) { 2688 err = -ENOMEM; 2689 goto out_unlock; 2690 } 2691 2692 wr_req->r_flags = CEPH_OSD_FLAG_WRITE; 2693 osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); 2694 ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc); 2695 ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid); 2696 2697 err = ceph_osdc_alloc_messages(wr_req, GFP_KERNEL); 2698 if (err) 2699 goto out_unlock; 2700 2701 /* one page should be large enough for STAT data */ 2702 pages = ceph_alloc_page_vector(1, GFP_KERNEL); 2703 if (IS_ERR(pages)) { 2704 err = PTR_ERR(pages); 2705 goto out_unlock; 2706 } 2707 2708 osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, 2709 0, false, true); 2710 ceph_osdc_start_request(&fsc->client->osdc, rd_req); 2711 2712 wr_req->r_mtime = inode_get_mtime(&ci->netfs.inode); 2713 ceph_osdc_start_request(&fsc->client->osdc, wr_req); 2714 2715 err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); 2716 err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); 2717 2718 if (err >= 0 || err == -ENOENT) 2719 have |= POOL_READ; 2720 else if (err != -EPERM) { 2721 if (err == -EBLOCKLISTED) 2722 fsc->blocklisted = true; 2723 goto out_unlock; 2724 } 2725 2726 if (err2 == 0 || err2 == -EEXIST) 2727 have |= POOL_WRITE; 2728 else if (err2 != -EPERM) { 2729 if (err2 == -EBLOCKLISTED) 2730 fsc->blocklisted = true; 2731 err = err2; 2732 goto out_unlock; 2733 } 2734 2735 pool_ns_len = pool_ns ? pool_ns->len : 0; 2736 perm = kmalloc_flex(*perm, pool_ns, pool_ns_len + 1); 2737 if (!perm) { 2738 err = -ENOMEM; 2739 goto out_unlock; 2740 } 2741 2742 perm->pool = pool; 2743 perm->perm = have; 2744 perm->pool_ns_len = pool_ns_len; 2745 if (pool_ns_len > 0) 2746 memcpy(perm->pool_ns, pool_ns->str, pool_ns_len); 2747 perm->pool_ns[pool_ns_len] = 0; 2748 2749 rb_link_node(&perm->node, parent, p); 2750 rb_insert_color(&perm->node, &mdsc->pool_perm_tree); 2751 err = 0; 2752 out_unlock: 2753 up_write(&mdsc->pool_perm_rwsem); 2754 2755 ceph_osdc_put_request(rd_req); 2756 ceph_osdc_put_request(wr_req); 2757 out: 2758 if (!err) 2759 err = have; 2760 if (pool_ns) 2761 doutc(cl, "pool %lld ns %.*s result = %d\n", pool, 2762 (int)pool_ns->len, pool_ns->str, err); 2763 else 2764 doutc(cl, "pool %lld result = %d\n", pool, err); 2765 return err; 2766 } 2767 2768 int ceph_pool_perm_check(struct inode *inode, int need) 2769 { 2770 struct ceph_client *cl = ceph_inode_to_client(inode); 2771 struct ceph_inode_info *ci = ceph_inode(inode); 2772 struct ceph_string *pool_ns; 2773 s64 pool; 2774 int ret; 2775 unsigned long flags; 2776 2777 /* Only need to do this for regular files */ 2778 if (!S_ISREG(inode->i_mode)) 2779 return 0; 2780 2781 if (ceph_in_snap(inode)) { 2782 /* 2783 * Pool permission check needs to write to the first object. 2784 * But for snapshot, head of the first object may have already 2785 * been deleted. Skip check to avoid creating orphan object. 2786 */ 2787 return 0; 2788 } 2789 2790 if (ceph_test_mount_opt(ceph_inode_to_fs_client(inode), 2791 NOPOOLPERM)) 2792 return 0; 2793 2794 spin_lock(&ci->i_ceph_lock); 2795 flags = ci->i_ceph_flags; 2796 pool = ci->i_layout.pool_id; 2797 spin_unlock(&ci->i_ceph_lock); 2798 check: 2799 if (flags & CEPH_I_POOL_PERM) { 2800 if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { 2801 doutc(cl, "pool %lld no read perm\n", pool); 2802 return -EPERM; 2803 } 2804 if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { 2805 doutc(cl, "pool %lld no write perm\n", pool); 2806 return -EPERM; 2807 } 2808 return 0; 2809 } 2810 2811 pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); 2812 ret = __ceph_pool_perm_get(ci, pool, pool_ns); 2813 ceph_put_string(pool_ns); 2814 if (ret < 0) 2815 return ret; 2816 2817 spin_lock(&ci->i_ceph_lock); 2818 if (pool == ci->i_layout.pool_id && 2819 pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { 2820 set_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); 2821 if (ret & POOL_READ) 2822 set_bit(CEPH_I_POOL_RD_BIT, &ci->i_ceph_flags); 2823 if (ret & POOL_WRITE) 2824 set_bit(CEPH_I_POOL_WR_BIT, &ci->i_ceph_flags); 2825 } else { 2826 pool = ci->i_layout.pool_id; 2827 } 2828 /* Re-read flags under the lock so check: sees the updated bits. */ 2829 flags = ci->i_ceph_flags; 2830 spin_unlock(&ci->i_ceph_lock); 2831 goto check; 2832 } 2833 2834 void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) 2835 { 2836 struct ceph_pool_perm *perm; 2837 struct rb_node *n; 2838 2839 while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { 2840 n = rb_first(&mdsc->pool_perm_tree); 2841 perm = rb_entry(n, struct ceph_pool_perm, node); 2842 rb_erase(n, &mdsc->pool_perm_tree); 2843 kfree(perm); 2844 } 2845 } 2846