1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6 #include <crypto/hash.h>
7 #include <linux/kernel.h>
8 #include <linux/bio.h>
9 #include <linux/blk-cgroup.h>
10 #include <linux/file.h>
11 #include <linux/fs.h>
12 #include <linux/pagemap.h>
13 #include <linux/highmem.h>
14 #include <linux/time.h>
15 #include <linux/init.h>
16 #include <linux/string.h>
17 #include <linux/backing-dev.h>
18 #include <linux/writeback.h>
19 #include <linux/compat.h>
20 #include <linux/xattr.h>
21 #include <linux/posix_acl.h>
22 #include <linux/falloc.h>
23 #include <linux/slab.h>
24 #include <linux/ratelimit.h>
25 #include <linux/btrfs.h>
26 #include <linux/blkdev.h>
27 #include <linux/posix_acl_xattr.h>
28 #include <linux/uio.h>
29 #include <linux/magic.h>
30 #include <linux/iversion.h>
31 #include <linux/swap.h>
32 #include <linux/migrate.h>
33 #include <linux/sched/mm.h>
34 #include <linux/iomap.h>
35 #include <asm/unaligned.h>
36 #include <linux/fsverity.h>
37 #include "misc.h"
38 #include "ctree.h"
39 #include "disk-io.h"
40 #include "transaction.h"
41 #include "btrfs_inode.h"
42 #include "ordered-data.h"
43 #include "xattr.h"
44 #include "tree-log.h"
45 #include "bio.h"
46 #include "compression.h"
47 #include "locking.h"
48 #include "props.h"
49 #include "qgroup.h"
50 #include "delalloc-space.h"
51 #include "block-group.h"
52 #include "space-info.h"
53 #include "zoned.h"
54 #include "subpage.h"
55 #include "inode-item.h"
56 #include "fs.h"
57 #include "accessors.h"
58 #include "extent-tree.h"
59 #include "root-tree.h"
60 #include "defrag.h"
61 #include "dir-item.h"
62 #include "file-item.h"
63 #include "uuid-tree.h"
64 #include "ioctl.h"
65 #include "file.h"
66 #include "acl.h"
67 #include "relocation.h"
68 #include "verity.h"
69 #include "super.h"
70 #include "orphan.h"
71 #include "backref.h"
72 #include "raid-stripe-tree.h"
73 #include "fiemap.h"
74
75 struct btrfs_iget_args {
76 u64 ino;
77 struct btrfs_root *root;
78 };
79
80 struct btrfs_rename_ctx {
81 /* Output field. Stores the index number of the old directory entry. */
82 u64 index;
83 };
84
85 /*
86 * Used by data_reloc_print_warning_inode() to pass needed info for filename
87 * resolution and output of error message.
88 */
89 struct data_reloc_warn {
90 struct btrfs_path path;
91 struct btrfs_fs_info *fs_info;
92 u64 extent_item_size;
93 u64 logical;
94 int mirror_num;
95 };
96
97 /*
98 * For the file_extent_tree, we want to hold the inode lock when we lookup and
99 * update the disk_i_size, but lockdep will complain because our io_tree we hold
100 * the tree lock and get the inode lock when setting delalloc. These two things
101 * are unrelated, so make a class for the file_extent_tree so we don't get the
102 * two locking patterns mixed up.
103 */
104 static struct lock_class_key file_extent_tree_class;
105
106 static const struct inode_operations btrfs_dir_inode_operations;
107 static const struct inode_operations btrfs_symlink_inode_operations;
108 static const struct inode_operations btrfs_special_inode_operations;
109 static const struct inode_operations btrfs_file_inode_operations;
110 static const struct address_space_operations btrfs_aops;
111 static const struct file_operations btrfs_dir_file_operations;
112
113 static struct kmem_cache *btrfs_inode_cachep;
114
115 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
116 static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);
117
118 static noinline int run_delalloc_cow(struct btrfs_inode *inode,
119 struct folio *locked_folio, u64 start,
120 u64 end, struct writeback_control *wbc,
121 bool pages_dirty);
122
data_reloc_print_warning_inode(u64 inum,u64 offset,u64 num_bytes,u64 root,void * warn_ctx)123 static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
124 u64 root, void *warn_ctx)
125 {
126 struct data_reloc_warn *warn = warn_ctx;
127 struct btrfs_fs_info *fs_info = warn->fs_info;
128 struct extent_buffer *eb;
129 struct btrfs_inode_item *inode_item;
130 struct inode_fs_paths *ipath = NULL;
131 struct btrfs_root *local_root;
132 struct btrfs_key key;
133 unsigned int nofs_flag;
134 u32 nlink;
135 int ret;
136
137 local_root = btrfs_get_fs_root(fs_info, root, true);
138 if (IS_ERR(local_root)) {
139 ret = PTR_ERR(local_root);
140 goto err;
141 }
142
143 /* This makes the path point to (inum INODE_ITEM ioff). */
144 key.objectid = inum;
145 key.type = BTRFS_INODE_ITEM_KEY;
146 key.offset = 0;
147
148 ret = btrfs_search_slot(NULL, local_root, &key, &warn->path, 0, 0);
149 if (ret) {
150 btrfs_put_root(local_root);
151 btrfs_release_path(&warn->path);
152 goto err;
153 }
154
155 eb = warn->path.nodes[0];
156 inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item);
157 nlink = btrfs_inode_nlink(eb, inode_item);
158 btrfs_release_path(&warn->path);
159
160 nofs_flag = memalloc_nofs_save();
161 ipath = init_ipath(4096, local_root, &warn->path);
162 memalloc_nofs_restore(nofs_flag);
163 if (IS_ERR(ipath)) {
164 btrfs_put_root(local_root);
165 ret = PTR_ERR(ipath);
166 ipath = NULL;
167 /*
168 * -ENOMEM, not a critical error, just output an generic error
169 * without filename.
170 */
171 btrfs_warn(fs_info,
172 "checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu",
173 warn->logical, warn->mirror_num, root, inum, offset);
174 return ret;
175 }
176 ret = paths_from_inode(inum, ipath);
177 if (ret < 0)
178 goto err;
179
180 /*
181 * We deliberately ignore the bit ipath might have been too small to
182 * hold all of the paths here
183 */
184 for (int i = 0; i < ipath->fspath->elem_cnt; i++) {
185 btrfs_warn(fs_info,
186 "checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)",
187 warn->logical, warn->mirror_num, root, inum, offset,
188 fs_info->sectorsize, nlink,
189 (char *)(unsigned long)ipath->fspath->val[i]);
190 }
191
192 btrfs_put_root(local_root);
193 free_ipath(ipath);
194 return 0;
195
196 err:
197 btrfs_warn(fs_info,
198 "checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d",
199 warn->logical, warn->mirror_num, root, inum, offset, ret);
200
201 free_ipath(ipath);
202 return ret;
203 }
204
205 /*
206 * Do extra user-friendly error output (e.g. lookup all the affected files).
207 *
208 * Return true if we succeeded doing the backref lookup.
209 * Return false if such lookup failed, and has to fallback to the old error message.
210 */
print_data_reloc_error(const struct btrfs_inode * inode,u64 file_off,const u8 * csum,const u8 * csum_expected,int mirror_num)211 static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off,
212 const u8 *csum, const u8 *csum_expected,
213 int mirror_num)
214 {
215 struct btrfs_fs_info *fs_info = inode->root->fs_info;
216 struct btrfs_path path = { 0 };
217 struct btrfs_key found_key = { 0 };
218 struct extent_buffer *eb;
219 struct btrfs_extent_item *ei;
220 const u32 csum_size = fs_info->csum_size;
221 u64 logical;
222 u64 flags;
223 u32 item_size;
224 int ret;
225
226 mutex_lock(&fs_info->reloc_mutex);
227 logical = btrfs_get_reloc_bg_bytenr(fs_info);
228 mutex_unlock(&fs_info->reloc_mutex);
229
230 if (logical == U64_MAX) {
231 btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation");
232 btrfs_warn_rl(fs_info,
233 "csum failed root %lld ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",
234 btrfs_root_id(inode->root), btrfs_ino(inode), file_off,
235 CSUM_FMT_VALUE(csum_size, csum),
236 CSUM_FMT_VALUE(csum_size, csum_expected),
237 mirror_num);
238 return;
239 }
240
241 logical += file_off;
242 btrfs_warn_rl(fs_info,
243 "csum failed root %lld ino %llu off %llu logical %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",
244 btrfs_root_id(inode->root),
245 btrfs_ino(inode), file_off, logical,
246 CSUM_FMT_VALUE(csum_size, csum),
247 CSUM_FMT_VALUE(csum_size, csum_expected),
248 mirror_num);
249
250 ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags);
251 if (ret < 0) {
252 btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d",
253 logical, ret);
254 return;
255 }
256 eb = path.nodes[0];
257 ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item);
258 item_size = btrfs_item_size(eb, path.slots[0]);
259 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
260 unsigned long ptr = 0;
261 u64 ref_root;
262 u8 ref_level;
263
264 while (true) {
265 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
266 item_size, &ref_root,
267 &ref_level);
268 if (ret < 0) {
269 btrfs_warn_rl(fs_info,
270 "failed to resolve tree backref for logical %llu: %d",
271 logical, ret);
272 break;
273 }
274 if (ret > 0)
275 break;
276
277 btrfs_warn_rl(fs_info,
278 "csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu",
279 logical, mirror_num,
280 (ref_level ? "node" : "leaf"),
281 ref_level, ref_root);
282 }
283 btrfs_release_path(&path);
284 } else {
285 struct btrfs_backref_walk_ctx ctx = { 0 };
286 struct data_reloc_warn reloc_warn = { 0 };
287
288 btrfs_release_path(&path);
289
290 ctx.bytenr = found_key.objectid;
291 ctx.extent_item_pos = logical - found_key.objectid;
292 ctx.fs_info = fs_info;
293
294 reloc_warn.logical = logical;
295 reloc_warn.extent_item_size = found_key.offset;
296 reloc_warn.mirror_num = mirror_num;
297 reloc_warn.fs_info = fs_info;
298
299 iterate_extent_inodes(&ctx, true,
300 data_reloc_print_warning_inode, &reloc_warn);
301 }
302 }
303
btrfs_print_data_csum_error(struct btrfs_inode * inode,u64 logical_start,u8 * csum,u8 * csum_expected,int mirror_num)304 static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode,
305 u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num)
306 {
307 struct btrfs_root *root = inode->root;
308 const u32 csum_size = root->fs_info->csum_size;
309
310 /* For data reloc tree, it's better to do a backref lookup instead. */
311 if (btrfs_root_id(root) == BTRFS_DATA_RELOC_TREE_OBJECTID)
312 return print_data_reloc_error(inode, logical_start, csum,
313 csum_expected, mirror_num);
314
315 /* Output without objectid, which is more meaningful */
316 if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) {
317 btrfs_warn_rl(root->fs_info,
318 "csum failed root %lld ino %lld off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",
319 btrfs_root_id(root), btrfs_ino(inode),
320 logical_start,
321 CSUM_FMT_VALUE(csum_size, csum),
322 CSUM_FMT_VALUE(csum_size, csum_expected),
323 mirror_num);
324 } else {
325 btrfs_warn_rl(root->fs_info,
326 "csum failed root %llu ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",
327 btrfs_root_id(root), btrfs_ino(inode),
328 logical_start,
329 CSUM_FMT_VALUE(csum_size, csum),
330 CSUM_FMT_VALUE(csum_size, csum_expected),
331 mirror_num);
332 }
333 }
334
335 /*
336 * Lock inode i_rwsem based on arguments passed.
337 *
338 * ilock_flags can have the following bit set:
339 *
340 * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode
341 * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt
342 * return -EAGAIN
343 * BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock
344 */
btrfs_inode_lock(struct btrfs_inode * inode,unsigned int ilock_flags)345 int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags)
346 {
347 if (ilock_flags & BTRFS_ILOCK_SHARED) {
348 if (ilock_flags & BTRFS_ILOCK_TRY) {
349 if (!inode_trylock_shared(&inode->vfs_inode))
350 return -EAGAIN;
351 else
352 return 0;
353 }
354 inode_lock_shared(&inode->vfs_inode);
355 } else {
356 if (ilock_flags & BTRFS_ILOCK_TRY) {
357 if (!inode_trylock(&inode->vfs_inode))
358 return -EAGAIN;
359 else
360 return 0;
361 }
362 inode_lock(&inode->vfs_inode);
363 }
364 if (ilock_flags & BTRFS_ILOCK_MMAP)
365 down_write(&inode->i_mmap_lock);
366 return 0;
367 }
368
369 /*
370 * Unock inode i_rwsem.
371 *
372 * ilock_flags should contain the same bits set as passed to btrfs_inode_lock()
373 * to decide whether the lock acquired is shared or exclusive.
374 */
btrfs_inode_unlock(struct btrfs_inode * inode,unsigned int ilock_flags)375 void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags)
376 {
377 if (ilock_flags & BTRFS_ILOCK_MMAP)
378 up_write(&inode->i_mmap_lock);
379 if (ilock_flags & BTRFS_ILOCK_SHARED)
380 inode_unlock_shared(&inode->vfs_inode);
381 else
382 inode_unlock(&inode->vfs_inode);
383 }
384
385 /*
386 * Cleanup all submitted ordered extents in specified range to handle errors
387 * from the btrfs_run_delalloc_range() callback.
388 *
389 * NOTE: caller must ensure that when an error happens, it can not call
390 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
391 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
392 * to be released, which we want to happen only when finishing the ordered
393 * extent (btrfs_finish_ordered_io()).
394 */
btrfs_cleanup_ordered_extents(struct btrfs_inode * inode,struct folio * locked_folio,u64 offset,u64 bytes)395 static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,
396 struct folio *locked_folio,
397 u64 offset, u64 bytes)
398 {
399 unsigned long index = offset >> PAGE_SHIFT;
400 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
401 u64 page_start = 0, page_end = 0;
402 struct folio *folio;
403
404 if (locked_folio) {
405 page_start = folio_pos(locked_folio);
406 page_end = page_start + folio_size(locked_folio) - 1;
407 }
408
409 while (index <= end_index) {
410 /*
411 * For locked page, we will call btrfs_mark_ordered_io_finished
412 * through btrfs_mark_ordered_io_finished() on it
413 * in run_delalloc_range() for the error handling, which will
414 * clear page Ordered and run the ordered extent accounting.
415 *
416 * Here we can't just clear the Ordered bit, or
417 * btrfs_mark_ordered_io_finished() would skip the accounting
418 * for the page range, and the ordered extent will never finish.
419 */
420 if (locked_folio && index == (page_start >> PAGE_SHIFT)) {
421 index++;
422 continue;
423 }
424 folio = __filemap_get_folio(inode->vfs_inode.i_mapping, index, 0, 0);
425 index++;
426 if (IS_ERR(folio))
427 continue;
428
429 /*
430 * Here we just clear all Ordered bits for every page in the
431 * range, then btrfs_mark_ordered_io_finished() will handle
432 * the ordered extent accounting for the range.
433 */
434 btrfs_folio_clamp_clear_ordered(inode->root->fs_info, folio,
435 offset, bytes);
436 folio_put(folio);
437 }
438
439 if (locked_folio) {
440 /* The locked page covers the full range, nothing needs to be done */
441 if (bytes + offset <= page_start + folio_size(locked_folio))
442 return;
443 /*
444 * In case this page belongs to the delalloc range being
445 * instantiated then skip it, since the first page of a range is
446 * going to be properly cleaned up by the caller of
447 * run_delalloc_range
448 */
449 if (page_start >= offset && page_end <= (offset + bytes - 1)) {
450 bytes = offset + bytes - folio_pos(locked_folio) -
451 folio_size(locked_folio);
452 offset = folio_pos(locked_folio) + folio_size(locked_folio);
453 }
454 }
455
456 return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false);
457 }
458
459 static int btrfs_dirty_inode(struct btrfs_inode *inode);
460
btrfs_init_inode_security(struct btrfs_trans_handle * trans,struct btrfs_new_inode_args * args)461 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
462 struct btrfs_new_inode_args *args)
463 {
464 int err;
465
466 if (args->default_acl) {
467 err = __btrfs_set_acl(trans, args->inode, args->default_acl,
468 ACL_TYPE_DEFAULT);
469 if (err)
470 return err;
471 }
472 if (args->acl) {
473 err = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS);
474 if (err)
475 return err;
476 }
477 if (!args->default_acl && !args->acl)
478 cache_no_acl(args->inode);
479 return btrfs_xattr_security_init(trans, args->inode, args->dir,
480 &args->dentry->d_name);
481 }
482
483 /*
484 * this does all the hard work for inserting an inline extent into
485 * the btree. The caller should have done a btrfs_drop_extents so that
486 * no overlapping inline items exist in the btree
487 */
insert_inline_extent(struct btrfs_trans_handle * trans,struct btrfs_path * path,struct btrfs_inode * inode,bool extent_inserted,size_t size,size_t compressed_size,int compress_type,struct folio * compressed_folio,bool update_i_size)488 static int insert_inline_extent(struct btrfs_trans_handle *trans,
489 struct btrfs_path *path,
490 struct btrfs_inode *inode, bool extent_inserted,
491 size_t size, size_t compressed_size,
492 int compress_type,
493 struct folio *compressed_folio,
494 bool update_i_size)
495 {
496 struct btrfs_root *root = inode->root;
497 struct extent_buffer *leaf;
498 const u32 sectorsize = trans->fs_info->sectorsize;
499 char *kaddr;
500 unsigned long ptr;
501 struct btrfs_file_extent_item *ei;
502 int ret;
503 size_t cur_size = size;
504 u64 i_size;
505
506 /*
507 * The decompressed size must still be no larger than a sector. Under
508 * heavy race, we can have size == 0 passed in, but that shouldn't be a
509 * big deal and we can continue the insertion.
510 */
511 ASSERT(size <= sectorsize);
512
513 /*
514 * The compressed size also needs to be no larger than a sector.
515 * That's also why we only need one page as the parameter.
516 */
517 if (compressed_folio)
518 ASSERT(compressed_size <= sectorsize);
519 else
520 ASSERT(compressed_size == 0);
521
522 if (compressed_size && compressed_folio)
523 cur_size = compressed_size;
524
525 if (!extent_inserted) {
526 struct btrfs_key key;
527 size_t datasize;
528
529 key.objectid = btrfs_ino(inode);
530 key.offset = 0;
531 key.type = BTRFS_EXTENT_DATA_KEY;
532
533 datasize = btrfs_file_extent_calc_inline_size(cur_size);
534 ret = btrfs_insert_empty_item(trans, root, path, &key,
535 datasize);
536 if (ret)
537 goto fail;
538 }
539 leaf = path->nodes[0];
540 ei = btrfs_item_ptr(leaf, path->slots[0],
541 struct btrfs_file_extent_item);
542 btrfs_set_file_extent_generation(leaf, ei, trans->transid);
543 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
544 btrfs_set_file_extent_encryption(leaf, ei, 0);
545 btrfs_set_file_extent_other_encoding(leaf, ei, 0);
546 btrfs_set_file_extent_ram_bytes(leaf, ei, size);
547 ptr = btrfs_file_extent_inline_start(ei);
548
549 if (compress_type != BTRFS_COMPRESS_NONE) {
550 kaddr = kmap_local_folio(compressed_folio, 0);
551 write_extent_buffer(leaf, kaddr, ptr, compressed_size);
552 kunmap_local(kaddr);
553
554 btrfs_set_file_extent_compression(leaf, ei,
555 compress_type);
556 } else {
557 struct folio *folio;
558
559 folio = __filemap_get_folio(inode->vfs_inode.i_mapping,
560 0, 0, 0);
561 ASSERT(!IS_ERR(folio));
562 btrfs_set_file_extent_compression(leaf, ei, 0);
563 kaddr = kmap_local_folio(folio, 0);
564 write_extent_buffer(leaf, kaddr, ptr, size);
565 kunmap_local(kaddr);
566 folio_put(folio);
567 }
568 btrfs_mark_buffer_dirty(trans, leaf);
569 btrfs_release_path(path);
570
571 /*
572 * We align size to sectorsize for inline extents just for simplicity
573 * sake.
574 */
575 ret = btrfs_inode_set_file_extent_range(inode, 0,
576 ALIGN(size, root->fs_info->sectorsize));
577 if (ret)
578 goto fail;
579
580 /*
581 * We're an inline extent, so nobody can extend the file past i_size
582 * without locking a page we already have locked.
583 *
584 * We must do any i_size and inode updates before we unlock the pages.
585 * Otherwise we could end up racing with unlink.
586 */
587 i_size = i_size_read(&inode->vfs_inode);
588 if (update_i_size && size > i_size) {
589 i_size_write(&inode->vfs_inode, size);
590 i_size = size;
591 }
592 inode->disk_i_size = i_size;
593
594 fail:
595 return ret;
596 }
597
can_cow_file_range_inline(struct btrfs_inode * inode,u64 offset,u64 size,size_t compressed_size)598 static bool can_cow_file_range_inline(struct btrfs_inode *inode,
599 u64 offset, u64 size,
600 size_t compressed_size)
601 {
602 struct btrfs_fs_info *fs_info = inode->root->fs_info;
603 u64 data_len = (compressed_size ?: size);
604
605 /* Inline extents must start at offset 0. */
606 if (offset != 0)
607 return false;
608
609 /*
610 * Due to the page size limit, for subpage we can only trigger the
611 * writeback for the dirty sectors of page, that means data writeback
612 * is doing more writeback than what we want.
613 *
614 * This is especially unexpected for some call sites like fallocate,
615 * where we only increase i_size after everything is done.
616 * This means we can trigger inline extent even if we didn't want to.
617 * So here we skip inline extent creation completely.
618 */
619 if (fs_info->sectorsize != PAGE_SIZE)
620 return false;
621
622 /* Inline extents are limited to sectorsize. */
623 if (size > fs_info->sectorsize)
624 return false;
625
626 /* We cannot exceed the maximum inline data size. */
627 if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
628 return false;
629
630 /* We cannot exceed the user specified max_inline size. */
631 if (data_len > fs_info->max_inline)
632 return false;
633
634 /* Inline extents must be the entirety of the file. */
635 if (size < i_size_read(&inode->vfs_inode))
636 return false;
637
638 return true;
639 }
640
641 /*
642 * conditionally insert an inline extent into the file. This
643 * does the checks required to make sure the data is small enough
644 * to fit as an inline extent.
645 *
646 * If being used directly, you must have already checked we're allowed to cow
647 * the range by getting true from can_cow_file_range_inline().
648 */
__cow_file_range_inline(struct btrfs_inode * inode,u64 offset,u64 size,size_t compressed_size,int compress_type,struct folio * compressed_folio,bool update_i_size)649 static noinline int __cow_file_range_inline(struct btrfs_inode *inode, u64 offset,
650 u64 size, size_t compressed_size,
651 int compress_type,
652 struct folio *compressed_folio,
653 bool update_i_size)
654 {
655 struct btrfs_drop_extents_args drop_args = { 0 };
656 struct btrfs_root *root = inode->root;
657 struct btrfs_fs_info *fs_info = root->fs_info;
658 struct btrfs_trans_handle *trans;
659 u64 data_len = (compressed_size ?: size);
660 int ret;
661 struct btrfs_path *path;
662
663 path = btrfs_alloc_path();
664 if (!path)
665 return -ENOMEM;
666
667 trans = btrfs_join_transaction(root);
668 if (IS_ERR(trans)) {
669 btrfs_free_path(path);
670 return PTR_ERR(trans);
671 }
672 trans->block_rsv = &inode->block_rsv;
673
674 drop_args.path = path;
675 drop_args.start = 0;
676 drop_args.end = fs_info->sectorsize;
677 drop_args.drop_cache = true;
678 drop_args.replace_extent = true;
679 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len);
680 ret = btrfs_drop_extents(trans, root, inode, &drop_args);
681 if (ret) {
682 btrfs_abort_transaction(trans, ret);
683 goto out;
684 }
685
686 ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted,
687 size, compressed_size, compress_type,
688 compressed_folio, update_i_size);
689 if (ret && ret != -ENOSPC) {
690 btrfs_abort_transaction(trans, ret);
691 goto out;
692 } else if (ret == -ENOSPC) {
693 ret = 1;
694 goto out;
695 }
696
697 btrfs_update_inode_bytes(inode, size, drop_args.bytes_found);
698 ret = btrfs_update_inode(trans, inode);
699 if (ret && ret != -ENOSPC) {
700 btrfs_abort_transaction(trans, ret);
701 goto out;
702 } else if (ret == -ENOSPC) {
703 ret = 1;
704 goto out;
705 }
706
707 btrfs_set_inode_full_sync(inode);
708 out:
709 /*
710 * Don't forget to free the reserved space, as for inlined extent
711 * it won't count as data extent, free them directly here.
712 * And at reserve time, it's always aligned to page size, so
713 * just free one page here.
714 */
715 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE, NULL);
716 btrfs_free_path(path);
717 btrfs_end_transaction(trans);
718 return ret;
719 }
720
cow_file_range_inline(struct btrfs_inode * inode,struct folio * locked_folio,u64 offset,u64 end,size_t compressed_size,int compress_type,struct folio * compressed_folio,bool update_i_size)721 static noinline int cow_file_range_inline(struct btrfs_inode *inode,
722 struct folio *locked_folio,
723 u64 offset, u64 end,
724 size_t compressed_size,
725 int compress_type,
726 struct folio *compressed_folio,
727 bool update_i_size)
728 {
729 struct extent_state *cached = NULL;
730 unsigned long clear_flags = EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
731 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING | EXTENT_LOCKED;
732 u64 size = min_t(u64, i_size_read(&inode->vfs_inode), end + 1);
733 int ret;
734
735 if (!can_cow_file_range_inline(inode, offset, size, compressed_size))
736 return 1;
737
738 lock_extent(&inode->io_tree, offset, end, &cached);
739 ret = __cow_file_range_inline(inode, offset, size, compressed_size,
740 compress_type, compressed_folio,
741 update_i_size);
742 if (ret > 0) {
743 unlock_extent(&inode->io_tree, offset, end, &cached);
744 return ret;
745 }
746
747 /*
748 * In the successful case (ret == 0 here), cow_file_range will return 1.
749 *
750 * Quite a bit further up the callstack in extent_writepage(), ret == 1
751 * is treated as a short circuited success and does not unlock the folio,
752 * so we must do it here.
753 *
754 * In the failure case, the locked_folio does get unlocked by
755 * btrfs_folio_end_all_writers, which asserts that it is still locked
756 * at that point, so we must *not* unlock it here.
757 *
758 * The other two callsites in compress_file_range do not have a
759 * locked_folio, so they are not relevant to this logic.
760 */
761 if (ret == 0)
762 locked_folio = NULL;
763
764 extent_clear_unlock_delalloc(inode, offset, end, locked_folio, &cached,
765 clear_flags, PAGE_UNLOCK |
766 PAGE_START_WRITEBACK | PAGE_END_WRITEBACK);
767 return ret;
768 }
769
770 struct async_extent {
771 u64 start;
772 u64 ram_size;
773 u64 compressed_size;
774 struct folio **folios;
775 unsigned long nr_folios;
776 int compress_type;
777 struct list_head list;
778 };
779
780 struct async_chunk {
781 struct btrfs_inode *inode;
782 struct folio *locked_folio;
783 u64 start;
784 u64 end;
785 blk_opf_t write_flags;
786 struct list_head extents;
787 struct cgroup_subsys_state *blkcg_css;
788 struct btrfs_work work;
789 struct async_cow *async_cow;
790 };
791
792 struct async_cow {
793 atomic_t num_chunks;
794 struct async_chunk chunks[];
795 };
796
add_async_extent(struct async_chunk * cow,u64 start,u64 ram_size,u64 compressed_size,struct folio ** folios,unsigned long nr_folios,int compress_type)797 static noinline int add_async_extent(struct async_chunk *cow,
798 u64 start, u64 ram_size,
799 u64 compressed_size,
800 struct folio **folios,
801 unsigned long nr_folios,
802 int compress_type)
803 {
804 struct async_extent *async_extent;
805
806 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
807 if (!async_extent)
808 return -ENOMEM;
809 async_extent->start = start;
810 async_extent->ram_size = ram_size;
811 async_extent->compressed_size = compressed_size;
812 async_extent->folios = folios;
813 async_extent->nr_folios = nr_folios;
814 async_extent->compress_type = compress_type;
815 list_add_tail(&async_extent->list, &cow->extents);
816 return 0;
817 }
818
819 /*
820 * Check if the inode needs to be submitted to compression, based on mount
821 * options, defragmentation, properties or heuristics.
822 */
inode_need_compress(struct btrfs_inode * inode,u64 start,u64 end)823 static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
824 u64 end)
825 {
826 struct btrfs_fs_info *fs_info = inode->root->fs_info;
827
828 if (!btrfs_inode_can_compress(inode)) {
829 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
830 KERN_ERR "BTRFS: unexpected compression for ino %llu\n",
831 btrfs_ino(inode));
832 return 0;
833 }
834 /*
835 * Special check for subpage.
836 *
837 * We lock the full page then run each delalloc range in the page, thus
838 * for the following case, we will hit some subpage specific corner case:
839 *
840 * 0 32K 64K
841 * | |///////| |///////|
842 * \- A \- B
843 *
844 * In above case, both range A and range B will try to unlock the full
845 * page [0, 64K), causing the one finished later will have page
846 * unlocked already, triggering various page lock requirement BUG_ON()s.
847 *
848 * So here we add an artificial limit that subpage compression can only
849 * if the range is fully page aligned.
850 *
851 * In theory we only need to ensure the first page is fully covered, but
852 * the tailing partial page will be locked until the full compression
853 * finishes, delaying the write of other range.
854 *
855 * TODO: Make btrfs_run_delalloc_range() to lock all delalloc range
856 * first to prevent any submitted async extent to unlock the full page.
857 * By this, we can ensure for subpage case that only the last async_cow
858 * will unlock the full page.
859 */
860 if (fs_info->sectorsize < PAGE_SIZE) {
861 if (!PAGE_ALIGNED(start) ||
862 !PAGE_ALIGNED(end + 1))
863 return 0;
864 }
865
866 /* force compress */
867 if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
868 return 1;
869 /* defrag ioctl */
870 if (inode->defrag_compress)
871 return 1;
872 /* bad compression ratios */
873 if (inode->flags & BTRFS_INODE_NOCOMPRESS)
874 return 0;
875 if (btrfs_test_opt(fs_info, COMPRESS) ||
876 inode->flags & BTRFS_INODE_COMPRESS ||
877 inode->prop_compress)
878 return btrfs_compress_heuristic(inode, start, end);
879 return 0;
880 }
881
inode_should_defrag(struct btrfs_inode * inode,u64 start,u64 end,u64 num_bytes,u32 small_write)882 static inline void inode_should_defrag(struct btrfs_inode *inode,
883 u64 start, u64 end, u64 num_bytes, u32 small_write)
884 {
885 /* If this is a small write inside eof, kick off a defrag */
886 if (num_bytes < small_write &&
887 (start > 0 || end + 1 < inode->disk_i_size))
888 btrfs_add_inode_defrag(inode, small_write);
889 }
890
extent_range_clear_dirty_for_io(struct inode * inode,u64 start,u64 end)891 static int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
892 {
893 unsigned long end_index = end >> PAGE_SHIFT;
894 struct folio *folio;
895 int ret = 0;
896
897 for (unsigned long index = start >> PAGE_SHIFT;
898 index <= end_index; index++) {
899 folio = __filemap_get_folio(inode->i_mapping, index, 0, 0);
900 if (IS_ERR(folio)) {
901 if (!ret)
902 ret = PTR_ERR(folio);
903 continue;
904 }
905 folio_clear_dirty_for_io(folio);
906 folio_put(folio);
907 }
908 return ret;
909 }
910
911 /*
912 * Work queue call back to started compression on a file and pages.
913 *
914 * This is done inside an ordered work queue, and the compression is spread
915 * across many cpus. The actual IO submission is step two, and the ordered work
916 * queue takes care of making sure that happens in the same order things were
917 * put onto the queue by writepages and friends.
918 *
919 * If this code finds it can't get good compression, it puts an entry onto the
920 * work queue to write the uncompressed bytes. This makes sure that both
921 * compressed inodes and uncompressed inodes are written in the same order that
922 * the flusher thread sent them down.
923 */
compress_file_range(struct btrfs_work * work)924 static void compress_file_range(struct btrfs_work *work)
925 {
926 struct async_chunk *async_chunk =
927 container_of(work, struct async_chunk, work);
928 struct btrfs_inode *inode = async_chunk->inode;
929 struct btrfs_fs_info *fs_info = inode->root->fs_info;
930 struct address_space *mapping = inode->vfs_inode.i_mapping;
931 u64 blocksize = fs_info->sectorsize;
932 u64 start = async_chunk->start;
933 u64 end = async_chunk->end;
934 u64 actual_end;
935 u64 i_size;
936 int ret = 0;
937 struct folio **folios;
938 unsigned long nr_folios;
939 unsigned long total_compressed = 0;
940 unsigned long total_in = 0;
941 unsigned int poff;
942 int i;
943 int compress_type = fs_info->compress_type;
944
945 inode_should_defrag(inode, start, end, end - start + 1, SZ_16K);
946
947 /*
948 * We need to call clear_page_dirty_for_io on each page in the range.
949 * Otherwise applications with the file mmap'd can wander in and change
950 * the page contents while we are compressing them.
951 */
952 ret = extent_range_clear_dirty_for_io(&inode->vfs_inode, start, end);
953
954 /*
955 * All the folios should have been locked thus no failure.
956 *
957 * And even if some folios are missing, btrfs_compress_folios()
958 * would handle them correctly, so here just do an ASSERT() check for
959 * early logic errors.
960 */
961 ASSERT(ret == 0);
962
963 /*
964 * We need to save i_size before now because it could change in between
965 * us evaluating the size and assigning it. This is because we lock and
966 * unlock the page in truncate and fallocate, and then modify the i_size
967 * later on.
968 *
969 * The barriers are to emulate READ_ONCE, remove that once i_size_read
970 * does that for us.
971 */
972 barrier();
973 i_size = i_size_read(&inode->vfs_inode);
974 barrier();
975 actual_end = min_t(u64, i_size, end + 1);
976 again:
977 folios = NULL;
978 nr_folios = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
979 nr_folios = min_t(unsigned long, nr_folios, BTRFS_MAX_COMPRESSED_PAGES);
980
981 /*
982 * we don't want to send crud past the end of i_size through
983 * compression, that's just a waste of CPU time. So, if the
984 * end of the file is before the start of our current
985 * requested range of bytes, we bail out to the uncompressed
986 * cleanup code that can deal with all of this.
987 *
988 * It isn't really the fastest way to fix things, but this is a
989 * very uncommon corner.
990 */
991 if (actual_end <= start)
992 goto cleanup_and_bail_uncompressed;
993
994 total_compressed = actual_end - start;
995
996 /*
997 * Skip compression for a small file range(<=blocksize) that
998 * isn't an inline extent, since it doesn't save disk space at all.
999 */
1000 if (total_compressed <= blocksize &&
1001 (start > 0 || end + 1 < inode->disk_i_size))
1002 goto cleanup_and_bail_uncompressed;
1003
1004 /*
1005 * For subpage case, we require full page alignment for the sector
1006 * aligned range.
1007 * Thus we must also check against @actual_end, not just @end.
1008 */
1009 if (blocksize < PAGE_SIZE) {
1010 if (!PAGE_ALIGNED(start) ||
1011 !PAGE_ALIGNED(round_up(actual_end, blocksize)))
1012 goto cleanup_and_bail_uncompressed;
1013 }
1014
1015 total_compressed = min_t(unsigned long, total_compressed,
1016 BTRFS_MAX_UNCOMPRESSED);
1017 total_in = 0;
1018 ret = 0;
1019
1020 /*
1021 * We do compression for mount -o compress and when the inode has not
1022 * been flagged as NOCOMPRESS. This flag can change at any time if we
1023 * discover bad compression ratios.
1024 */
1025 if (!inode_need_compress(inode, start, end))
1026 goto cleanup_and_bail_uncompressed;
1027
1028 folios = kcalloc(nr_folios, sizeof(struct folio *), GFP_NOFS);
1029 if (!folios) {
1030 /*
1031 * Memory allocation failure is not a fatal error, we can fall
1032 * back to uncompressed code.
1033 */
1034 goto cleanup_and_bail_uncompressed;
1035 }
1036
1037 if (inode->defrag_compress)
1038 compress_type = inode->defrag_compress;
1039 else if (inode->prop_compress)
1040 compress_type = inode->prop_compress;
1041
1042 /* Compression level is applied here. */
1043 ret = btrfs_compress_folios(compress_type | (fs_info->compress_level << 4),
1044 mapping, start, folios, &nr_folios, &total_in,
1045 &total_compressed);
1046 if (ret)
1047 goto mark_incompressible;
1048
1049 /*
1050 * Zero the tail end of the last page, as we might be sending it down
1051 * to disk.
1052 */
1053 poff = offset_in_page(total_compressed);
1054 if (poff)
1055 folio_zero_range(folios[nr_folios - 1], poff, PAGE_SIZE - poff);
1056
1057 /*
1058 * Try to create an inline extent.
1059 *
1060 * If we didn't compress the entire range, try to create an uncompressed
1061 * inline extent, else a compressed one.
1062 *
1063 * Check cow_file_range() for why we don't even try to create inline
1064 * extent for the subpage case.
1065 */
1066 if (total_in < actual_end)
1067 ret = cow_file_range_inline(inode, NULL, start, end, 0,
1068 BTRFS_COMPRESS_NONE, NULL, false);
1069 else
1070 ret = cow_file_range_inline(inode, NULL, start, end, total_compressed,
1071 compress_type, folios[0], false);
1072 if (ret <= 0) {
1073 if (ret < 0)
1074 mapping_set_error(mapping, -EIO);
1075 goto free_pages;
1076 }
1077
1078 /*
1079 * We aren't doing an inline extent. Round the compressed size up to a
1080 * block size boundary so the allocator does sane things.
1081 */
1082 total_compressed = ALIGN(total_compressed, blocksize);
1083
1084 /*
1085 * One last check to make sure the compression is really a win, compare
1086 * the page count read with the blocks on disk, compression must free at
1087 * least one sector.
1088 */
1089 total_in = round_up(total_in, fs_info->sectorsize);
1090 if (total_compressed + blocksize > total_in)
1091 goto mark_incompressible;
1092
1093 /*
1094 * The async work queues will take care of doing actual allocation on
1095 * disk for these compressed pages, and will submit the bios.
1096 */
1097 ret = add_async_extent(async_chunk, start, total_in, total_compressed, folios,
1098 nr_folios, compress_type);
1099 BUG_ON(ret);
1100 if (start + total_in < end) {
1101 start += total_in;
1102 cond_resched();
1103 goto again;
1104 }
1105 return;
1106
1107 mark_incompressible:
1108 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress)
1109 inode->flags |= BTRFS_INODE_NOCOMPRESS;
1110 cleanup_and_bail_uncompressed:
1111 ret = add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0,
1112 BTRFS_COMPRESS_NONE);
1113 BUG_ON(ret);
1114 free_pages:
1115 if (folios) {
1116 for (i = 0; i < nr_folios; i++) {
1117 WARN_ON(folios[i]->mapping);
1118 btrfs_free_compr_folio(folios[i]);
1119 }
1120 kfree(folios);
1121 }
1122 }
1123
free_async_extent_pages(struct async_extent * async_extent)1124 static void free_async_extent_pages(struct async_extent *async_extent)
1125 {
1126 int i;
1127
1128 if (!async_extent->folios)
1129 return;
1130
1131 for (i = 0; i < async_extent->nr_folios; i++) {
1132 WARN_ON(async_extent->folios[i]->mapping);
1133 btrfs_free_compr_folio(async_extent->folios[i]);
1134 }
1135 kfree(async_extent->folios);
1136 async_extent->nr_folios = 0;
1137 async_extent->folios = NULL;
1138 }
1139
submit_uncompressed_range(struct btrfs_inode * inode,struct async_extent * async_extent,struct folio * locked_folio)1140 static void submit_uncompressed_range(struct btrfs_inode *inode,
1141 struct async_extent *async_extent,
1142 struct folio *locked_folio)
1143 {
1144 u64 start = async_extent->start;
1145 u64 end = async_extent->start + async_extent->ram_size - 1;
1146 int ret;
1147 struct writeback_control wbc = {
1148 .sync_mode = WB_SYNC_ALL,
1149 .range_start = start,
1150 .range_end = end,
1151 .no_cgroup_owner = 1,
1152 };
1153
1154 wbc_attach_fdatawrite_inode(&wbc, &inode->vfs_inode);
1155 ret = run_delalloc_cow(inode, locked_folio, start, end,
1156 &wbc, false);
1157 wbc_detach_inode(&wbc);
1158 if (ret < 0) {
1159 btrfs_cleanup_ordered_extents(inode, locked_folio,
1160 start, end - start + 1);
1161 if (locked_folio) {
1162 const u64 page_start = folio_pos(locked_folio);
1163
1164 folio_start_writeback(locked_folio);
1165 folio_end_writeback(locked_folio);
1166 btrfs_mark_ordered_io_finished(inode, locked_folio,
1167 page_start, PAGE_SIZE,
1168 !ret);
1169 mapping_set_error(locked_folio->mapping, ret);
1170 folio_unlock(locked_folio);
1171 }
1172 }
1173 }
1174
submit_one_async_extent(struct async_chunk * async_chunk,struct async_extent * async_extent,u64 * alloc_hint)1175 static void submit_one_async_extent(struct async_chunk *async_chunk,
1176 struct async_extent *async_extent,
1177 u64 *alloc_hint)
1178 {
1179 struct btrfs_inode *inode = async_chunk->inode;
1180 struct extent_io_tree *io_tree = &inode->io_tree;
1181 struct btrfs_root *root = inode->root;
1182 struct btrfs_fs_info *fs_info = root->fs_info;
1183 struct btrfs_ordered_extent *ordered;
1184 struct btrfs_file_extent file_extent;
1185 struct btrfs_key ins;
1186 struct folio *locked_folio = NULL;
1187 struct extent_state *cached = NULL;
1188 struct extent_map *em;
1189 int ret = 0;
1190 u64 start = async_extent->start;
1191 u64 end = async_extent->start + async_extent->ram_size - 1;
1192
1193 if (async_chunk->blkcg_css)
1194 kthread_associate_blkcg(async_chunk->blkcg_css);
1195
1196 /*
1197 * If async_chunk->locked_folio is in the async_extent range, we need to
1198 * handle it.
1199 */
1200 if (async_chunk->locked_folio) {
1201 u64 locked_folio_start = folio_pos(async_chunk->locked_folio);
1202 u64 locked_folio_end = locked_folio_start +
1203 folio_size(async_chunk->locked_folio) - 1;
1204
1205 if (!(start >= locked_folio_end || end <= locked_folio_start))
1206 locked_folio = async_chunk->locked_folio;
1207 }
1208
1209 if (async_extent->compress_type == BTRFS_COMPRESS_NONE) {
1210 submit_uncompressed_range(inode, async_extent, locked_folio);
1211 goto done;
1212 }
1213
1214 ret = btrfs_reserve_extent(root, async_extent->ram_size,
1215 async_extent->compressed_size,
1216 async_extent->compressed_size,
1217 0, *alloc_hint, &ins, 1, 1);
1218 if (ret) {
1219 /*
1220 * We can't reserve contiguous space for the compressed size.
1221 * Unlikely, but it's possible that we could have enough
1222 * non-contiguous space for the uncompressed size instead. So
1223 * fall back to uncompressed.
1224 */
1225 submit_uncompressed_range(inode, async_extent, locked_folio);
1226 goto done;
1227 }
1228
1229 lock_extent(io_tree, start, end, &cached);
1230
1231 /* Here we're doing allocation and writeback of the compressed pages */
1232 file_extent.disk_bytenr = ins.objectid;
1233 file_extent.disk_num_bytes = ins.offset;
1234 file_extent.ram_bytes = async_extent->ram_size;
1235 file_extent.num_bytes = async_extent->ram_size;
1236 file_extent.offset = 0;
1237 file_extent.compression = async_extent->compress_type;
1238
1239 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
1240 if (IS_ERR(em)) {
1241 ret = PTR_ERR(em);
1242 goto out_free_reserve;
1243 }
1244 free_extent_map(em);
1245
1246 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
1247 1 << BTRFS_ORDERED_COMPRESSED);
1248 if (IS_ERR(ordered)) {
1249 btrfs_drop_extent_map_range(inode, start, end, false);
1250 ret = PTR_ERR(ordered);
1251 goto out_free_reserve;
1252 }
1253 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1254
1255 /* Clear dirty, set writeback and unlock the pages. */
1256 extent_clear_unlock_delalloc(inode, start, end,
1257 NULL, &cached, EXTENT_LOCKED | EXTENT_DELALLOC,
1258 PAGE_UNLOCK | PAGE_START_WRITEBACK);
1259 btrfs_submit_compressed_write(ordered,
1260 async_extent->folios, /* compressed_folios */
1261 async_extent->nr_folios,
1262 async_chunk->write_flags, true);
1263 *alloc_hint = ins.objectid + ins.offset;
1264 done:
1265 if (async_chunk->blkcg_css)
1266 kthread_associate_blkcg(NULL);
1267 kfree(async_extent);
1268 return;
1269
1270 out_free_reserve:
1271 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1272 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1273 mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
1274 extent_clear_unlock_delalloc(inode, start, end,
1275 NULL, &cached,
1276 EXTENT_LOCKED | EXTENT_DELALLOC |
1277 EXTENT_DELALLOC_NEW |
1278 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
1279 PAGE_UNLOCK | PAGE_START_WRITEBACK |
1280 PAGE_END_WRITEBACK);
1281 free_async_extent_pages(async_extent);
1282 if (async_chunk->blkcg_css)
1283 kthread_associate_blkcg(NULL);
1284 btrfs_debug(fs_info,
1285 "async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",
1286 btrfs_root_id(root), btrfs_ino(inode), start,
1287 async_extent->ram_size, ret);
1288 kfree(async_extent);
1289 }
1290
btrfs_get_extent_allocation_hint(struct btrfs_inode * inode,u64 start,u64 num_bytes)1291 u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
1292 u64 num_bytes)
1293 {
1294 struct extent_map_tree *em_tree = &inode->extent_tree;
1295 struct extent_map *em;
1296 u64 alloc_hint = 0;
1297
1298 read_lock(&em_tree->lock);
1299 em = search_extent_mapping(em_tree, start, num_bytes);
1300 if (em) {
1301 /*
1302 * if block start isn't an actual block number then find the
1303 * first block in this inode and use that as a hint. If that
1304 * block is also bogus then just don't worry about it.
1305 */
1306 if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) {
1307 free_extent_map(em);
1308 em = search_extent_mapping(em_tree, 0, 0);
1309 if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
1310 alloc_hint = extent_map_block_start(em);
1311 if (em)
1312 free_extent_map(em);
1313 } else {
1314 alloc_hint = extent_map_block_start(em);
1315 free_extent_map(em);
1316 }
1317 }
1318 read_unlock(&em_tree->lock);
1319
1320 return alloc_hint;
1321 }
1322
1323 /*
1324 * when extent_io.c finds a delayed allocation range in the file,
1325 * the call backs end up in this code. The basic idea is to
1326 * allocate extents on disk for the range, and create ordered data structs
1327 * in ram to track those extents.
1328 *
1329 * locked_folio is the folio that writepage had locked already. We use
1330 * it to make sure we don't do extra locks or unlocks.
1331 *
1332 * When this function fails, it unlocks all pages except @locked_folio.
1333 *
1334 * When this function successfully creates an inline extent, it returns 1 and
1335 * unlocks all pages including locked_folio and starts I/O on them.
1336 * (In reality inline extents are limited to a single page, so locked_folio is
1337 * the only page handled anyway).
1338 *
1339 * When this function succeed and creates a normal extent, the page locking
1340 * status depends on the passed in flags:
1341 *
1342 * - If @keep_locked is set, all pages are kept locked.
1343 * - Else all pages except for @locked_folio are unlocked.
1344 *
1345 * When a failure happens in the second or later iteration of the
1346 * while-loop, the ordered extents created in previous iterations are kept
1347 * intact. So, the caller must clean them up by calling
1348 * btrfs_cleanup_ordered_extents(). See btrfs_run_delalloc_range() for
1349 * example.
1350 */
cow_file_range(struct btrfs_inode * inode,struct folio * locked_folio,u64 start,u64 end,u64 * done_offset,bool keep_locked,bool no_inline)1351 static noinline int cow_file_range(struct btrfs_inode *inode,
1352 struct folio *locked_folio, u64 start,
1353 u64 end, u64 *done_offset,
1354 bool keep_locked, bool no_inline)
1355 {
1356 struct btrfs_root *root = inode->root;
1357 struct btrfs_fs_info *fs_info = root->fs_info;
1358 struct extent_state *cached = NULL;
1359 u64 alloc_hint = 0;
1360 u64 orig_start = start;
1361 u64 num_bytes;
1362 unsigned long ram_size;
1363 u64 cur_alloc_size = 0;
1364 u64 min_alloc_size;
1365 u64 blocksize = fs_info->sectorsize;
1366 struct btrfs_key ins;
1367 struct extent_map *em;
1368 unsigned clear_bits;
1369 unsigned long page_ops;
1370 bool extent_reserved = false;
1371 int ret = 0;
1372
1373 if (btrfs_is_free_space_inode(inode)) {
1374 ret = -EINVAL;
1375 goto out_unlock;
1376 }
1377
1378 num_bytes = ALIGN(end - start + 1, blocksize);
1379 num_bytes = max(blocksize, num_bytes);
1380 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
1381
1382 inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
1383
1384 if (!no_inline) {
1385 /* lets try to make an inline extent */
1386 ret = cow_file_range_inline(inode, locked_folio, start, end, 0,
1387 BTRFS_COMPRESS_NONE, NULL, false);
1388 if (ret <= 0) {
1389 /*
1390 * We succeeded, return 1 so the caller knows we're done
1391 * with this page and already handled the IO.
1392 *
1393 * If there was an error then cow_file_range_inline() has
1394 * already done the cleanup.
1395 */
1396 if (ret == 0)
1397 ret = 1;
1398 goto done;
1399 }
1400 }
1401
1402 alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);
1403
1404 /*
1405 * Relocation relies on the relocated extents to have exactly the same
1406 * size as the original extents. Normally writeback for relocation data
1407 * extents follows a NOCOW path because relocation preallocates the
1408 * extents. However, due to an operation such as scrub turning a block
1409 * group to RO mode, it may fallback to COW mode, so we must make sure
1410 * an extent allocated during COW has exactly the requested size and can
1411 * not be split into smaller extents, otherwise relocation breaks and
1412 * fails during the stage where it updates the bytenr of file extent
1413 * items.
1414 */
1415 if (btrfs_is_data_reloc_root(root))
1416 min_alloc_size = num_bytes;
1417 else
1418 min_alloc_size = fs_info->sectorsize;
1419
1420 while (num_bytes > 0) {
1421 struct btrfs_ordered_extent *ordered;
1422 struct btrfs_file_extent file_extent;
1423
1424 cur_alloc_size = num_bytes;
1425 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1426 min_alloc_size, 0, alloc_hint,
1427 &ins, 1, 1);
1428 if (ret == -EAGAIN) {
1429 /*
1430 * btrfs_reserve_extent only returns -EAGAIN for zoned
1431 * file systems, which is an indication that there are
1432 * no active zones to allocate from at the moment.
1433 *
1434 * If this is the first loop iteration, wait for at
1435 * least one zone to finish before retrying the
1436 * allocation. Otherwise ask the caller to write out
1437 * the already allocated blocks before coming back to
1438 * us, or return -ENOSPC if it can't handle retries.
1439 */
1440 ASSERT(btrfs_is_zoned(fs_info));
1441 if (start == orig_start) {
1442 wait_on_bit_io(&inode->root->fs_info->flags,
1443 BTRFS_FS_NEED_ZONE_FINISH,
1444 TASK_UNINTERRUPTIBLE);
1445 continue;
1446 }
1447 if (done_offset) {
1448 *done_offset = start - 1;
1449 return 0;
1450 }
1451 ret = -ENOSPC;
1452 }
1453 if (ret < 0)
1454 goto out_unlock;
1455 cur_alloc_size = ins.offset;
1456 extent_reserved = true;
1457
1458 ram_size = ins.offset;
1459 file_extent.disk_bytenr = ins.objectid;
1460 file_extent.disk_num_bytes = ins.offset;
1461 file_extent.num_bytes = ins.offset;
1462 file_extent.ram_bytes = ins.offset;
1463 file_extent.offset = 0;
1464 file_extent.compression = BTRFS_COMPRESS_NONE;
1465
1466 lock_extent(&inode->io_tree, start, start + ram_size - 1,
1467 &cached);
1468
1469 em = btrfs_create_io_em(inode, start, &file_extent,
1470 BTRFS_ORDERED_REGULAR);
1471 if (IS_ERR(em)) {
1472 unlock_extent(&inode->io_tree, start,
1473 start + ram_size - 1, &cached);
1474 ret = PTR_ERR(em);
1475 goto out_reserve;
1476 }
1477 free_extent_map(em);
1478
1479 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
1480 1 << BTRFS_ORDERED_REGULAR);
1481 if (IS_ERR(ordered)) {
1482 unlock_extent(&inode->io_tree, start,
1483 start + ram_size - 1, &cached);
1484 ret = PTR_ERR(ordered);
1485 goto out_drop_extent_cache;
1486 }
1487
1488 if (btrfs_is_data_reloc_root(root)) {
1489 ret = btrfs_reloc_clone_csums(ordered);
1490
1491 /*
1492 * Only drop cache here, and process as normal.
1493 *
1494 * We must not allow extent_clear_unlock_delalloc()
1495 * at out_unlock label to free meta of this ordered
1496 * extent, as its meta should be freed by
1497 * btrfs_finish_ordered_io().
1498 *
1499 * So we must continue until @start is increased to
1500 * skip current ordered extent.
1501 */
1502 if (ret)
1503 btrfs_drop_extent_map_range(inode, start,
1504 start + ram_size - 1,
1505 false);
1506 }
1507 btrfs_put_ordered_extent(ordered);
1508
1509 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1510
1511 /*
1512 * We're not doing compressed IO, don't unlock the first page
1513 * (which the caller expects to stay locked), don't clear any
1514 * dirty bits and don't set any writeback bits
1515 *
1516 * Do set the Ordered (Private2) bit so we know this page was
1517 * properly setup for writepage.
1518 */
1519 page_ops = (keep_locked ? 0 : PAGE_UNLOCK);
1520 page_ops |= PAGE_SET_ORDERED;
1521
1522 extent_clear_unlock_delalloc(inode, start, start + ram_size - 1,
1523 locked_folio, &cached,
1524 EXTENT_LOCKED | EXTENT_DELALLOC,
1525 page_ops);
1526 if (num_bytes < cur_alloc_size)
1527 num_bytes = 0;
1528 else
1529 num_bytes -= cur_alloc_size;
1530 alloc_hint = ins.objectid + ins.offset;
1531 start += cur_alloc_size;
1532 extent_reserved = false;
1533
1534 /*
1535 * btrfs_reloc_clone_csums() error, since start is increased
1536 * extent_clear_unlock_delalloc() at out_unlock label won't
1537 * free metadata of current ordered extent, we're OK to exit.
1538 */
1539 if (ret)
1540 goto out_unlock;
1541 }
1542 done:
1543 if (done_offset)
1544 *done_offset = end;
1545 return ret;
1546
1547 out_drop_extent_cache:
1548 btrfs_drop_extent_map_range(inode, start, start + ram_size - 1, false);
1549 out_reserve:
1550 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1551 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1552 out_unlock:
1553 /*
1554 * Now, we have three regions to clean up:
1555 *
1556 * |-------(1)----|---(2)---|-------------(3)----------|
1557 * `- orig_start `- start `- start + cur_alloc_size `- end
1558 *
1559 * We process each region below.
1560 */
1561
1562 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1563 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1564 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK;
1565
1566 /*
1567 * For the range (1). We have already instantiated the ordered extents
1568 * for this region. They are cleaned up by
1569 * btrfs_cleanup_ordered_extents() in e.g,
1570 * btrfs_run_delalloc_range(). EXTENT_LOCKED | EXTENT_DELALLOC are
1571 * already cleared in the above loop. And, EXTENT_DELALLOC_NEW |
1572 * EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV are handled by the cleanup
1573 * function.
1574 *
1575 * However, in case of @keep_locked, we still need to unlock the pages
1576 * (except @locked_folio) to ensure all the pages are unlocked.
1577 */
1578 if (keep_locked && orig_start < start) {
1579 if (!locked_folio)
1580 mapping_set_error(inode->vfs_inode.i_mapping, ret);
1581 extent_clear_unlock_delalloc(inode, orig_start, start - 1,
1582 locked_folio, NULL, 0, page_ops);
1583 }
1584
1585 /*
1586 * At this point we're unlocked, we want to make sure we're only
1587 * clearing these flags under the extent lock, so lock the rest of the
1588 * range and clear everything up.
1589 */
1590 lock_extent(&inode->io_tree, start, end, NULL);
1591
1592 /*
1593 * For the range (2). If we reserved an extent for our delalloc range
1594 * (or a subrange) and failed to create the respective ordered extent,
1595 * then it means that when we reserved the extent we decremented the
1596 * extent's size from the data space_info's bytes_may_use counter and
1597 * incremented the space_info's bytes_reserved counter by the same
1598 * amount. We must make sure extent_clear_unlock_delalloc() does not try
1599 * to decrement again the data space_info's bytes_may_use counter,
1600 * therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV.
1601 */
1602 if (extent_reserved) {
1603 extent_clear_unlock_delalloc(inode, start,
1604 start + cur_alloc_size - 1,
1605 locked_folio, &cached, clear_bits,
1606 page_ops);
1607 btrfs_qgroup_free_data(inode, NULL, start, cur_alloc_size, NULL);
1608 start += cur_alloc_size;
1609 }
1610
1611 /*
1612 * For the range (3). We never touched the region. In addition to the
1613 * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data
1614 * space_info's bytes_may_use counter, reserved in
1615 * btrfs_check_data_free_space().
1616 */
1617 if (start < end) {
1618 clear_bits |= EXTENT_CLEAR_DATA_RESV;
1619 extent_clear_unlock_delalloc(inode, start, end, locked_folio,
1620 &cached, clear_bits, page_ops);
1621 btrfs_qgroup_free_data(inode, NULL, start, cur_alloc_size, NULL);
1622 }
1623 return ret;
1624 }
1625
1626 /*
1627 * Phase two of compressed writeback. This is the ordered portion of the code,
1628 * which only gets called in the order the work was queued. We walk all the
1629 * async extents created by compress_file_range and send them down to the disk.
1630 *
1631 * If called with @do_free == true then it'll try to finish the work and free
1632 * the work struct eventually.
1633 */
submit_compressed_extents(struct btrfs_work * work,bool do_free)1634 static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free)
1635 {
1636 struct async_chunk *async_chunk = container_of(work, struct async_chunk,
1637 work);
1638 struct btrfs_fs_info *fs_info = btrfs_work_owner(work);
1639 struct async_extent *async_extent;
1640 unsigned long nr_pages;
1641 u64 alloc_hint = 0;
1642
1643 if (do_free) {
1644 struct async_cow *async_cow;
1645
1646 btrfs_add_delayed_iput(async_chunk->inode);
1647 if (async_chunk->blkcg_css)
1648 css_put(async_chunk->blkcg_css);
1649
1650 async_cow = async_chunk->async_cow;
1651 if (atomic_dec_and_test(&async_cow->num_chunks))
1652 kvfree(async_cow);
1653 return;
1654 }
1655
1656 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>
1657 PAGE_SHIFT;
1658
1659 while (!list_empty(&async_chunk->extents)) {
1660 async_extent = list_entry(async_chunk->extents.next,
1661 struct async_extent, list);
1662 list_del(&async_extent->list);
1663 submit_one_async_extent(async_chunk, async_extent, &alloc_hint);
1664 }
1665
1666 /* atomic_sub_return implies a barrier */
1667 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1668 5 * SZ_1M)
1669 cond_wake_up_nomb(&fs_info->async_submit_wait);
1670 }
1671
run_delalloc_compressed(struct btrfs_inode * inode,struct folio * locked_folio,u64 start,u64 end,struct writeback_control * wbc)1672 static bool run_delalloc_compressed(struct btrfs_inode *inode,
1673 struct folio *locked_folio, u64 start,
1674 u64 end, struct writeback_control *wbc)
1675 {
1676 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1677 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc);
1678 struct async_cow *ctx;
1679 struct async_chunk *async_chunk;
1680 unsigned long nr_pages;
1681 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);
1682 int i;
1683 unsigned nofs_flag;
1684 const blk_opf_t write_flags = wbc_to_write_flags(wbc);
1685
1686 nofs_flag = memalloc_nofs_save();
1687 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL);
1688 memalloc_nofs_restore(nofs_flag);
1689 if (!ctx)
1690 return false;
1691
1692 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);
1693
1694 async_chunk = ctx->chunks;
1695 atomic_set(&ctx->num_chunks, num_chunks);
1696
1697 for (i = 0; i < num_chunks; i++) {
1698 u64 cur_end = min(end, start + SZ_512K - 1);
1699
1700 /*
1701 * igrab is called higher up in the call chain, take only the
1702 * lightweight reference for the callback lifetime
1703 */
1704 ihold(&inode->vfs_inode);
1705 async_chunk[i].async_cow = ctx;
1706 async_chunk[i].inode = inode;
1707 async_chunk[i].start = start;
1708 async_chunk[i].end = cur_end;
1709 async_chunk[i].write_flags = write_flags;
1710 INIT_LIST_HEAD(&async_chunk[i].extents);
1711
1712 /*
1713 * The locked_folio comes all the way from writepage and its
1714 * the original folio we were actually given. As we spread
1715 * this large delalloc region across multiple async_chunk
1716 * structs, only the first struct needs a pointer to
1717 * locked_folio.
1718 *
1719 * This way we don't need racey decisions about who is supposed
1720 * to unlock it.
1721 */
1722 if (locked_folio) {
1723 /*
1724 * Depending on the compressibility, the pages might or
1725 * might not go through async. We want all of them to
1726 * be accounted against wbc once. Let's do it here
1727 * before the paths diverge. wbc accounting is used
1728 * only for foreign writeback detection and doesn't
1729 * need full accuracy. Just account the whole thing
1730 * against the first page.
1731 */
1732 wbc_account_cgroup_owner(wbc, &locked_folio->page,
1733 cur_end - start);
1734 async_chunk[i].locked_folio = locked_folio;
1735 locked_folio = NULL;
1736 } else {
1737 async_chunk[i].locked_folio = NULL;
1738 }
1739
1740 if (blkcg_css != blkcg_root_css) {
1741 css_get(blkcg_css);
1742 async_chunk[i].blkcg_css = blkcg_css;
1743 async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT;
1744 } else {
1745 async_chunk[i].blkcg_css = NULL;
1746 }
1747
1748 btrfs_init_work(&async_chunk[i].work, compress_file_range,
1749 submit_compressed_extents);
1750
1751 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);
1752 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1753
1754 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work);
1755
1756 start = cur_end + 1;
1757 }
1758 return true;
1759 }
1760
1761 /*
1762 * Run the delalloc range from start to end, and write back any dirty pages
1763 * covered by the range.
1764 */
run_delalloc_cow(struct btrfs_inode * inode,struct folio * locked_folio,u64 start,u64 end,struct writeback_control * wbc,bool pages_dirty)1765 static noinline int run_delalloc_cow(struct btrfs_inode *inode,
1766 struct folio *locked_folio, u64 start,
1767 u64 end, struct writeback_control *wbc,
1768 bool pages_dirty)
1769 {
1770 u64 done_offset = end;
1771 int ret;
1772
1773 while (start <= end) {
1774 ret = cow_file_range(inode, locked_folio, start, end,
1775 &done_offset, true, false);
1776 if (ret)
1777 return ret;
1778 extent_write_locked_range(&inode->vfs_inode, locked_folio,
1779 start, done_offset, wbc, pages_dirty);
1780 start = done_offset + 1;
1781 }
1782
1783 return 1;
1784 }
1785
fallback_to_cow(struct btrfs_inode * inode,struct folio * locked_folio,const u64 start,const u64 end)1786 static int fallback_to_cow(struct btrfs_inode *inode,
1787 struct folio *locked_folio, const u64 start,
1788 const u64 end)
1789 {
1790 const bool is_space_ino = btrfs_is_free_space_inode(inode);
1791 const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root);
1792 const u64 range_bytes = end + 1 - start;
1793 struct extent_io_tree *io_tree = &inode->io_tree;
1794 struct extent_state *cached_state = NULL;
1795 u64 range_start = start;
1796 u64 count;
1797 int ret;
1798
1799 /*
1800 * If EXTENT_NORESERVE is set it means that when the buffered write was
1801 * made we had not enough available data space and therefore we did not
1802 * reserve data space for it, since we though we could do NOCOW for the
1803 * respective file range (either there is prealloc extent or the inode
1804 * has the NOCOW bit set).
1805 *
1806 * However when we need to fallback to COW mode (because for example the
1807 * block group for the corresponding extent was turned to RO mode by a
1808 * scrub or relocation) we need to do the following:
1809 *
1810 * 1) We increment the bytes_may_use counter of the data space info.
1811 * If COW succeeds, it allocates a new data extent and after doing
1812 * that it decrements the space info's bytes_may_use counter and
1813 * increments its bytes_reserved counter by the same amount (we do
1814 * this at btrfs_add_reserved_bytes()). So we need to increment the
1815 * bytes_may_use counter to compensate (when space is reserved at
1816 * buffered write time, the bytes_may_use counter is incremented);
1817 *
1818 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1819 * that if the COW path fails for any reason, it decrements (through
1820 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1821 * data space info, which we incremented in the step above.
1822 *
1823 * If we need to fallback to cow and the inode corresponds to a free
1824 * space cache inode or an inode of the data relocation tree, we must
1825 * also increment bytes_may_use of the data space_info for the same
1826 * reason. Space caches and relocated data extents always get a prealloc
1827 * extent for them, however scrub or balance may have set the block
1828 * group that contains that extent to RO mode and therefore force COW
1829 * when starting writeback.
1830 */
1831 lock_extent(io_tree, start, end, &cached_state);
1832 count = count_range_bits(io_tree, &range_start, end, range_bytes,
1833 EXTENT_NORESERVE, 0, NULL);
1834 if (count > 0 || is_space_ino || is_reloc_ino) {
1835 u64 bytes = count;
1836 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1837 struct btrfs_space_info *sinfo = fs_info->data_sinfo;
1838
1839 if (is_space_ino || is_reloc_ino)
1840 bytes = range_bytes;
1841
1842 spin_lock(&sinfo->lock);
1843 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes);
1844 spin_unlock(&sinfo->lock);
1845
1846 if (count > 0)
1847 clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,
1848 NULL);
1849 }
1850 unlock_extent(io_tree, start, end, &cached_state);
1851
1852 /*
1853 * Don't try to create inline extents, as a mix of inline extent that
1854 * is written out and unlocked directly and a normal NOCOW extent
1855 * doesn't work.
1856 */
1857 ret = cow_file_range(inode, locked_folio, start, end, NULL, false,
1858 true);
1859 ASSERT(ret != 1);
1860 return ret;
1861 }
1862
1863 struct can_nocow_file_extent_args {
1864 /* Input fields. */
1865
1866 /* Start file offset of the range we want to NOCOW. */
1867 u64 start;
1868 /* End file offset (inclusive) of the range we want to NOCOW. */
1869 u64 end;
1870 bool writeback_path;
1871 bool strict;
1872 /*
1873 * Free the path passed to can_nocow_file_extent() once it's not needed
1874 * anymore.
1875 */
1876 bool free_path;
1877
1878 /*
1879 * Output fields. Only set when can_nocow_file_extent() returns 1.
1880 * The expected file extent for the NOCOW write.
1881 */
1882 struct btrfs_file_extent file_extent;
1883 };
1884
1885 /*
1886 * Check if we can NOCOW the file extent that the path points to.
1887 * This function may return with the path released, so the caller should check
1888 * if path->nodes[0] is NULL or not if it needs to use the path afterwards.
1889 *
1890 * Returns: < 0 on error
1891 * 0 if we can not NOCOW
1892 * 1 if we can NOCOW
1893 */
can_nocow_file_extent(struct btrfs_path * path,struct btrfs_key * key,struct btrfs_inode * inode,struct can_nocow_file_extent_args * args)1894 static int can_nocow_file_extent(struct btrfs_path *path,
1895 struct btrfs_key *key,
1896 struct btrfs_inode *inode,
1897 struct can_nocow_file_extent_args *args)
1898 {
1899 const bool is_freespace_inode = btrfs_is_free_space_inode(inode);
1900 struct extent_buffer *leaf = path->nodes[0];
1901 struct btrfs_root *root = inode->root;
1902 struct btrfs_file_extent_item *fi;
1903 struct btrfs_root *csum_root;
1904 u64 io_start;
1905 u64 extent_end;
1906 u8 extent_type;
1907 int can_nocow = 0;
1908 int ret = 0;
1909 bool nowait = path->nowait;
1910
1911 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
1912 extent_type = btrfs_file_extent_type(leaf, fi);
1913
1914 if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1915 goto out;
1916
1917 if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
1918 extent_type == BTRFS_FILE_EXTENT_REG)
1919 goto out;
1920
1921 /*
1922 * If the extent was created before the generation where the last snapshot
1923 * for its subvolume was created, then this implies the extent is shared,
1924 * hence we must COW.
1925 */
1926 if (!args->strict &&
1927 btrfs_file_extent_generation(leaf, fi) <=
1928 btrfs_root_last_snapshot(&root->root_item))
1929 goto out;
1930
1931 /* An explicit hole, must COW. */
1932 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
1933 goto out;
1934
1935 /* Compressed/encrypted/encoded extents must be COWed. */
1936 if (btrfs_file_extent_compression(leaf, fi) ||
1937 btrfs_file_extent_encryption(leaf, fi) ||
1938 btrfs_file_extent_other_encoding(leaf, fi))
1939 goto out;
1940
1941 extent_end = btrfs_file_extent_end(path);
1942
1943 args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1944 args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1945 args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1946 args->file_extent.offset = btrfs_file_extent_offset(leaf, fi);
1947 args->file_extent.compression = btrfs_file_extent_compression(leaf, fi);
1948
1949 /*
1950 * The following checks can be expensive, as they need to take other
1951 * locks and do btree or rbtree searches, so release the path to avoid
1952 * blocking other tasks for too long.
1953 */
1954 btrfs_release_path(path);
1955
1956 ret = btrfs_cross_ref_exist(root, btrfs_ino(inode),
1957 key->offset - args->file_extent.offset,
1958 args->file_extent.disk_bytenr, args->strict, path);
1959 WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1960 if (ret != 0)
1961 goto out;
1962
1963 if (args->free_path) {
1964 /*
1965 * We don't need the path anymore, plus through the
1966 * btrfs_lookup_csums_list() call below we will end up allocating
1967 * another path. So free the path to avoid unnecessary extra
1968 * memory usage.
1969 */
1970 btrfs_free_path(path);
1971 path = NULL;
1972 }
1973
1974 /* If there are pending snapshots for this root, we must COW. */
1975 if (args->writeback_path && !is_freespace_inode &&
1976 atomic_read(&root->snapshot_force_cow))
1977 goto out;
1978
1979 args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start;
1980 args->file_extent.offset += args->start - key->offset;
1981 io_start = args->file_extent.disk_bytenr + args->file_extent.offset;
1982
1983 /*
1984 * Force COW if csums exist in the range. This ensures that csums for a
1985 * given extent are either valid or do not exist.
1986 */
1987
1988 csum_root = btrfs_csum_root(root->fs_info, io_start);
1989 ret = btrfs_lookup_csums_list(csum_root, io_start,
1990 io_start + args->file_extent.num_bytes - 1,
1991 NULL, nowait);
1992 WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1993 if (ret != 0)
1994 goto out;
1995
1996 can_nocow = 1;
1997 out:
1998 if (args->free_path && path)
1999 btrfs_free_path(path);
2000
2001 return ret < 0 ? ret : can_nocow;
2002 }
2003
2004 /*
2005 * when nowcow writeback call back. This checks for snapshots or COW copies
2006 * of the extents that exist in the file, and COWs the file as required.
2007 *
2008 * If no cow copies or snapshots exist, we write directly to the existing
2009 * blocks on disk
2010 */
run_delalloc_nocow(struct btrfs_inode * inode,struct folio * locked_folio,const u64 start,const u64 end)2011 static noinline int run_delalloc_nocow(struct btrfs_inode *inode,
2012 struct folio *locked_folio,
2013 const u64 start, const u64 end)
2014 {
2015 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2016 struct btrfs_root *root = inode->root;
2017 struct btrfs_path *path;
2018 u64 cow_start = (u64)-1;
2019 u64 cur_offset = start;
2020 int ret;
2021 bool check_prev = true;
2022 u64 ino = btrfs_ino(inode);
2023 struct can_nocow_file_extent_args nocow_args = { 0 };
2024
2025 /*
2026 * Normally on a zoned device we're only doing COW writes, but in case
2027 * of relocation on a zoned filesystem serializes I/O so that we're only
2028 * writing sequentially and can end up here as well.
2029 */
2030 ASSERT(!btrfs_is_zoned(fs_info) || btrfs_is_data_reloc_root(root));
2031
2032 path = btrfs_alloc_path();
2033 if (!path) {
2034 ret = -ENOMEM;
2035 goto error;
2036 }
2037
2038 nocow_args.end = end;
2039 nocow_args.writeback_path = true;
2040
2041 while (cur_offset <= end) {
2042 struct btrfs_block_group *nocow_bg = NULL;
2043 struct btrfs_ordered_extent *ordered;
2044 struct btrfs_key found_key;
2045 struct btrfs_file_extent_item *fi;
2046 struct extent_buffer *leaf;
2047 struct extent_state *cached_state = NULL;
2048 u64 extent_end;
2049 u64 nocow_end;
2050 int extent_type;
2051 bool is_prealloc;
2052
2053 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
2054 cur_offset, 0);
2055 if (ret < 0)
2056 goto error;
2057
2058 /*
2059 * If there is no extent for our range when doing the initial
2060 * search, then go back to the previous slot as it will be the
2061 * one containing the search offset
2062 */
2063 if (ret > 0 && path->slots[0] > 0 && check_prev) {
2064 leaf = path->nodes[0];
2065 btrfs_item_key_to_cpu(leaf, &found_key,
2066 path->slots[0] - 1);
2067 if (found_key.objectid == ino &&
2068 found_key.type == BTRFS_EXTENT_DATA_KEY)
2069 path->slots[0]--;
2070 }
2071 check_prev = false;
2072 next_slot:
2073 /* Go to next leaf if we have exhausted the current one */
2074 leaf = path->nodes[0];
2075 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2076 ret = btrfs_next_leaf(root, path);
2077 if (ret < 0)
2078 goto error;
2079 if (ret > 0)
2080 break;
2081 leaf = path->nodes[0];
2082 }
2083
2084 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2085
2086 /* Didn't find anything for our INO */
2087 if (found_key.objectid > ino)
2088 break;
2089 /*
2090 * Keep searching until we find an EXTENT_ITEM or there are no
2091 * more extents for this inode
2092 */
2093 if (WARN_ON_ONCE(found_key.objectid < ino) ||
2094 found_key.type < BTRFS_EXTENT_DATA_KEY) {
2095 path->slots[0]++;
2096 goto next_slot;
2097 }
2098
2099 /* Found key is not EXTENT_DATA_KEY or starts after req range */
2100 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
2101 found_key.offset > end)
2102 break;
2103
2104 /*
2105 * If the found extent starts after requested offset, then
2106 * adjust extent_end to be right before this extent begins
2107 */
2108 if (found_key.offset > cur_offset) {
2109 extent_end = found_key.offset;
2110 extent_type = 0;
2111 goto must_cow;
2112 }
2113
2114 /*
2115 * Found extent which begins before our range and potentially
2116 * intersect it
2117 */
2118 fi = btrfs_item_ptr(leaf, path->slots[0],
2119 struct btrfs_file_extent_item);
2120 extent_type = btrfs_file_extent_type(leaf, fi);
2121 /* If this is triggered then we have a memory corruption. */
2122 ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES);
2123 if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) {
2124 ret = -EUCLEAN;
2125 goto error;
2126 }
2127 extent_end = btrfs_file_extent_end(path);
2128
2129 /*
2130 * If the extent we got ends before our current offset, skip to
2131 * the next extent.
2132 */
2133 if (extent_end <= cur_offset) {
2134 path->slots[0]++;
2135 goto next_slot;
2136 }
2137
2138 nocow_args.start = cur_offset;
2139 ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args);
2140 if (ret < 0)
2141 goto error;
2142 if (ret == 0)
2143 goto must_cow;
2144
2145 ret = 0;
2146 nocow_bg = btrfs_inc_nocow_writers(fs_info,
2147 nocow_args.file_extent.disk_bytenr +
2148 nocow_args.file_extent.offset);
2149 if (!nocow_bg) {
2150 must_cow:
2151 /*
2152 * If we can't perform NOCOW writeback for the range,
2153 * then record the beginning of the range that needs to
2154 * be COWed. It will be written out before the next
2155 * NOCOW range if we find one, or when exiting this
2156 * loop.
2157 */
2158 if (cow_start == (u64)-1)
2159 cow_start = cur_offset;
2160 cur_offset = extent_end;
2161 if (cur_offset > end)
2162 break;
2163 if (!path->nodes[0])
2164 continue;
2165 path->slots[0]++;
2166 goto next_slot;
2167 }
2168
2169 /*
2170 * COW range from cow_start to found_key.offset - 1. As the key
2171 * will contain the beginning of the first extent that can be
2172 * NOCOW, following one which needs to be COW'ed
2173 */
2174 if (cow_start != (u64)-1) {
2175 ret = fallback_to_cow(inode, locked_folio, cow_start,
2176 found_key.offset - 1);
2177 cow_start = (u64)-1;
2178 if (ret) {
2179 btrfs_dec_nocow_writers(nocow_bg);
2180 goto error;
2181 }
2182 }
2183
2184 nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;
2185 lock_extent(&inode->io_tree, cur_offset, nocow_end, &cached_state);
2186
2187 is_prealloc = extent_type == BTRFS_FILE_EXTENT_PREALLOC;
2188 if (is_prealloc) {
2189 struct extent_map *em;
2190
2191 em = btrfs_create_io_em(inode, cur_offset,
2192 &nocow_args.file_extent,
2193 BTRFS_ORDERED_PREALLOC);
2194 if (IS_ERR(em)) {
2195 unlock_extent(&inode->io_tree, cur_offset,
2196 nocow_end, &cached_state);
2197 btrfs_dec_nocow_writers(nocow_bg);
2198 ret = PTR_ERR(em);
2199 goto error;
2200 }
2201 free_extent_map(em);
2202 }
2203
2204 ordered = btrfs_alloc_ordered_extent(inode, cur_offset,
2205 &nocow_args.file_extent,
2206 is_prealloc
2207 ? (1 << BTRFS_ORDERED_PREALLOC)
2208 : (1 << BTRFS_ORDERED_NOCOW));
2209 btrfs_dec_nocow_writers(nocow_bg);
2210 if (IS_ERR(ordered)) {
2211 if (is_prealloc) {
2212 btrfs_drop_extent_map_range(inode, cur_offset,
2213 nocow_end, false);
2214 }
2215 unlock_extent(&inode->io_tree, cur_offset,
2216 nocow_end, &cached_state);
2217 ret = PTR_ERR(ordered);
2218 goto error;
2219 }
2220
2221 if (btrfs_is_data_reloc_root(root))
2222 /*
2223 * Error handled later, as we must prevent
2224 * extent_clear_unlock_delalloc() in error handler
2225 * from freeing metadata of created ordered extent.
2226 */
2227 ret = btrfs_reloc_clone_csums(ordered);
2228 btrfs_put_ordered_extent(ordered);
2229
2230 extent_clear_unlock_delalloc(inode, cur_offset, nocow_end,
2231 locked_folio, &cached_state,
2232 EXTENT_LOCKED | EXTENT_DELALLOC |
2233 EXTENT_CLEAR_DATA_RESV,
2234 PAGE_UNLOCK | PAGE_SET_ORDERED);
2235
2236 cur_offset = extent_end;
2237
2238 /*
2239 * btrfs_reloc_clone_csums() error, now we're OK to call error
2240 * handler, as metadata for created ordered extent will only
2241 * be freed by btrfs_finish_ordered_io().
2242 */
2243 if (ret)
2244 goto error;
2245 }
2246 btrfs_release_path(path);
2247
2248 if (cur_offset <= end && cow_start == (u64)-1)
2249 cow_start = cur_offset;
2250
2251 if (cow_start != (u64)-1) {
2252 cur_offset = end;
2253 ret = fallback_to_cow(inode, locked_folio, cow_start, end);
2254 cow_start = (u64)-1;
2255 if (ret)
2256 goto error;
2257 }
2258
2259 btrfs_free_path(path);
2260 return 0;
2261
2262 error:
2263 /*
2264 * If an error happened while a COW region is outstanding, cur_offset
2265 * needs to be reset to cow_start to ensure the COW region is unlocked
2266 * as well.
2267 */
2268 if (cow_start != (u64)-1)
2269 cur_offset = cow_start;
2270
2271 /*
2272 * We need to lock the extent here because we're clearing DELALLOC and
2273 * we're not locked at this point.
2274 */
2275 if (cur_offset < end) {
2276 struct extent_state *cached = NULL;
2277
2278 lock_extent(&inode->io_tree, cur_offset, end, &cached);
2279 extent_clear_unlock_delalloc(inode, cur_offset, end,
2280 locked_folio, &cached,
2281 EXTENT_LOCKED | EXTENT_DELALLOC |
2282 EXTENT_DEFRAG |
2283 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
2284 PAGE_START_WRITEBACK |
2285 PAGE_END_WRITEBACK);
2286 btrfs_qgroup_free_data(inode, NULL, cur_offset, end - cur_offset + 1, NULL);
2287 }
2288 btrfs_free_path(path);
2289 return ret;
2290 }
2291
should_nocow(struct btrfs_inode * inode,u64 start,u64 end)2292 static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end)
2293 {
2294 if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) {
2295 if (inode->defrag_bytes &&
2296 test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG))
2297 return false;
2298 return true;
2299 }
2300 return false;
2301 }
2302
2303 /*
2304 * Function to process delayed allocation (create CoW) for ranges which are
2305 * being touched for the first time.
2306 */
btrfs_run_delalloc_range(struct btrfs_inode * inode,struct folio * locked_folio,u64 start,u64 end,struct writeback_control * wbc)2307 int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,
2308 u64 start, u64 end, struct writeback_control *wbc)
2309 {
2310 const bool zoned = btrfs_is_zoned(inode->root->fs_info);
2311 int ret;
2312
2313 /*
2314 * The range must cover part of the @locked_folio, or a return of 1
2315 * can confuse the caller.
2316 */
2317 ASSERT(!(end <= folio_pos(locked_folio) ||
2318 start >= folio_pos(locked_folio) + folio_size(locked_folio)));
2319
2320 if (should_nocow(inode, start, end)) {
2321 ret = run_delalloc_nocow(inode, locked_folio, start, end);
2322 goto out;
2323 }
2324
2325 if (btrfs_inode_can_compress(inode) &&
2326 inode_need_compress(inode, start, end) &&
2327 run_delalloc_compressed(inode, locked_folio, start, end, wbc))
2328 return 1;
2329
2330 if (zoned)
2331 ret = run_delalloc_cow(inode, locked_folio, start, end, wbc,
2332 true);
2333 else
2334 ret = cow_file_range(inode, locked_folio, start, end, NULL,
2335 false, false);
2336
2337 out:
2338 if (ret < 0)
2339 btrfs_cleanup_ordered_extents(inode, locked_folio, start,
2340 end - start + 1);
2341 return ret;
2342 }
2343
btrfs_split_delalloc_extent(struct btrfs_inode * inode,struct extent_state * orig,u64 split)2344 void btrfs_split_delalloc_extent(struct btrfs_inode *inode,
2345 struct extent_state *orig, u64 split)
2346 {
2347 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2348 u64 size;
2349
2350 lockdep_assert_held(&inode->io_tree.lock);
2351
2352 /* not delalloc, ignore it */
2353 if (!(orig->state & EXTENT_DELALLOC))
2354 return;
2355
2356 size = orig->end - orig->start + 1;
2357 if (size > fs_info->max_extent_size) {
2358 u32 num_extents;
2359 u64 new_size;
2360
2361 /*
2362 * See the explanation in btrfs_merge_delalloc_extent, the same
2363 * applies here, just in reverse.
2364 */
2365 new_size = orig->end - split + 1;
2366 num_extents = count_max_extents(fs_info, new_size);
2367 new_size = split - orig->start;
2368 num_extents += count_max_extents(fs_info, new_size);
2369 if (count_max_extents(fs_info, size) >= num_extents)
2370 return;
2371 }
2372
2373 spin_lock(&inode->lock);
2374 btrfs_mod_outstanding_extents(inode, 1);
2375 spin_unlock(&inode->lock);
2376 }
2377
2378 /*
2379 * Handle merged delayed allocation extents so we can keep track of new extents
2380 * that are just merged onto old extents, such as when we are doing sequential
2381 * writes, so we can properly account for the metadata space we'll need.
2382 */
btrfs_merge_delalloc_extent(struct btrfs_inode * inode,struct extent_state * new,struct extent_state * other)2383 void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new,
2384 struct extent_state *other)
2385 {
2386 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2387 u64 new_size, old_size;
2388 u32 num_extents;
2389
2390 lockdep_assert_held(&inode->io_tree.lock);
2391
2392 /* not delalloc, ignore it */
2393 if (!(other->state & EXTENT_DELALLOC))
2394 return;
2395
2396 if (new->start > other->start)
2397 new_size = new->end - other->start + 1;
2398 else
2399 new_size = other->end - new->start + 1;
2400
2401 /* we're not bigger than the max, unreserve the space and go */
2402 if (new_size <= fs_info->max_extent_size) {
2403 spin_lock(&inode->lock);
2404 btrfs_mod_outstanding_extents(inode, -1);
2405 spin_unlock(&inode->lock);
2406 return;
2407 }
2408
2409 /*
2410 * We have to add up either side to figure out how many extents were
2411 * accounted for before we merged into one big extent. If the number of
2412 * extents we accounted for is <= the amount we need for the new range
2413 * then we can return, otherwise drop. Think of it like this
2414 *
2415 * [ 4k][MAX_SIZE]
2416 *
2417 * So we've grown the extent by a MAX_SIZE extent, this would mean we
2418 * need 2 outstanding extents, on one side we have 1 and the other side
2419 * we have 1 so they are == and we can return. But in this case
2420 *
2421 * [MAX_SIZE+4k][MAX_SIZE+4k]
2422 *
2423 * Each range on their own accounts for 2 extents, but merged together
2424 * they are only 3 extents worth of accounting, so we need to drop in
2425 * this case.
2426 */
2427 old_size = other->end - other->start + 1;
2428 num_extents = count_max_extents(fs_info, old_size);
2429 old_size = new->end - new->start + 1;
2430 num_extents += count_max_extents(fs_info, old_size);
2431 if (count_max_extents(fs_info, new_size) >= num_extents)
2432 return;
2433
2434 spin_lock(&inode->lock);
2435 btrfs_mod_outstanding_extents(inode, -1);
2436 spin_unlock(&inode->lock);
2437 }
2438
btrfs_add_delalloc_inode(struct btrfs_inode * inode)2439 static void btrfs_add_delalloc_inode(struct btrfs_inode *inode)
2440 {
2441 struct btrfs_root *root = inode->root;
2442 struct btrfs_fs_info *fs_info = root->fs_info;
2443
2444 spin_lock(&root->delalloc_lock);
2445 ASSERT(list_empty(&inode->delalloc_inodes));
2446 list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes);
2447 root->nr_delalloc_inodes++;
2448 if (root->nr_delalloc_inodes == 1) {
2449 spin_lock(&fs_info->delalloc_root_lock);
2450 ASSERT(list_empty(&root->delalloc_root));
2451 list_add_tail(&root->delalloc_root, &fs_info->delalloc_roots);
2452 spin_unlock(&fs_info->delalloc_root_lock);
2453 }
2454 spin_unlock(&root->delalloc_lock);
2455 }
2456
btrfs_del_delalloc_inode(struct btrfs_inode * inode)2457 void btrfs_del_delalloc_inode(struct btrfs_inode *inode)
2458 {
2459 struct btrfs_root *root = inode->root;
2460 struct btrfs_fs_info *fs_info = root->fs_info;
2461
2462 lockdep_assert_held(&root->delalloc_lock);
2463
2464 /*
2465 * We may be called after the inode was already deleted from the list,
2466 * namely in the transaction abort path btrfs_destroy_delalloc_inodes(),
2467 * and then later through btrfs_clear_delalloc_extent() while the inode
2468 * still has ->delalloc_bytes > 0.
2469 */
2470 if (!list_empty(&inode->delalloc_inodes)) {
2471 list_del_init(&inode->delalloc_inodes);
2472 root->nr_delalloc_inodes--;
2473 if (!root->nr_delalloc_inodes) {
2474 ASSERT(list_empty(&root->delalloc_inodes));
2475 spin_lock(&fs_info->delalloc_root_lock);
2476 ASSERT(!list_empty(&root->delalloc_root));
2477 list_del_init(&root->delalloc_root);
2478 spin_unlock(&fs_info->delalloc_root_lock);
2479 }
2480 }
2481 }
2482
2483 /*
2484 * Properly track delayed allocation bytes in the inode and to maintain the
2485 * list of inodes that have pending delalloc work to be done.
2486 */
btrfs_set_delalloc_extent(struct btrfs_inode * inode,struct extent_state * state,u32 bits)2487 void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state,
2488 u32 bits)
2489 {
2490 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2491
2492 lockdep_assert_held(&inode->io_tree.lock);
2493
2494 if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC))
2495 WARN_ON(1);
2496 /*
2497 * set_bit and clear bit hooks normally require _irqsave/restore
2498 * but in this case, we are only testing for the DELALLOC
2499 * bit, which is only set or cleared with irqs on
2500 */
2501 if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2502 u64 len = state->end + 1 - state->start;
2503 u64 prev_delalloc_bytes;
2504 u32 num_extents = count_max_extents(fs_info, len);
2505
2506 spin_lock(&inode->lock);
2507 btrfs_mod_outstanding_extents(inode, num_extents);
2508 spin_unlock(&inode->lock);
2509
2510 /* For sanity tests */
2511 if (btrfs_is_testing(fs_info))
2512 return;
2513
2514 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
2515 fs_info->delalloc_batch);
2516 spin_lock(&inode->lock);
2517 prev_delalloc_bytes = inode->delalloc_bytes;
2518 inode->delalloc_bytes += len;
2519 if (bits & EXTENT_DEFRAG)
2520 inode->defrag_bytes += len;
2521 spin_unlock(&inode->lock);
2522
2523 /*
2524 * We don't need to be under the protection of the inode's lock,
2525 * because we are called while holding the inode's io_tree lock
2526 * and are therefore protected against concurrent calls of this
2527 * function and btrfs_clear_delalloc_extent().
2528 */
2529 if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0)
2530 btrfs_add_delalloc_inode(inode);
2531 }
2532
2533 if (!(state->state & EXTENT_DELALLOC_NEW) &&
2534 (bits & EXTENT_DELALLOC_NEW)) {
2535 spin_lock(&inode->lock);
2536 inode->new_delalloc_bytes += state->end + 1 - state->start;
2537 spin_unlock(&inode->lock);
2538 }
2539 }
2540
2541 /*
2542 * Once a range is no longer delalloc this function ensures that proper
2543 * accounting happens.
2544 */
btrfs_clear_delalloc_extent(struct btrfs_inode * inode,struct extent_state * state,u32 bits)2545 void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
2546 struct extent_state *state, u32 bits)
2547 {
2548 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2549 u64 len = state->end + 1 - state->start;
2550 u32 num_extents = count_max_extents(fs_info, len);
2551
2552 lockdep_assert_held(&inode->io_tree.lock);
2553
2554 if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) {
2555 spin_lock(&inode->lock);
2556 inode->defrag_bytes -= len;
2557 spin_unlock(&inode->lock);
2558 }
2559
2560 /*
2561 * set_bit and clear bit hooks normally require _irqsave/restore
2562 * but in this case, we are only testing for the DELALLOC
2563 * bit, which is only set or cleared with irqs on
2564 */
2565 if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2566 struct btrfs_root *root = inode->root;
2567 u64 new_delalloc_bytes;
2568
2569 spin_lock(&inode->lock);
2570 btrfs_mod_outstanding_extents(inode, -num_extents);
2571 spin_unlock(&inode->lock);
2572
2573 /*
2574 * We don't reserve metadata space for space cache inodes so we
2575 * don't need to call delalloc_release_metadata if there is an
2576 * error.
2577 */
2578 if (bits & EXTENT_CLEAR_META_RESV &&
2579 root != fs_info->tree_root)
2580 btrfs_delalloc_release_metadata(inode, len, true);
2581
2582 /* For sanity tests. */
2583 if (btrfs_is_testing(fs_info))
2584 return;
2585
2586 if (!btrfs_is_data_reloc_root(root) &&
2587 !btrfs_is_free_space_inode(inode) &&
2588 !(state->state & EXTENT_NORESERVE) &&
2589 (bits & EXTENT_CLEAR_DATA_RESV))
2590 btrfs_free_reserved_data_space_noquota(fs_info, len);
2591
2592 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
2593 fs_info->delalloc_batch);
2594 spin_lock(&inode->lock);
2595 inode->delalloc_bytes -= len;
2596 new_delalloc_bytes = inode->delalloc_bytes;
2597 spin_unlock(&inode->lock);
2598
2599 /*
2600 * We don't need to be under the protection of the inode's lock,
2601 * because we are called while holding the inode's io_tree lock
2602 * and are therefore protected against concurrent calls of this
2603 * function and btrfs_set_delalloc_extent().
2604 */
2605 if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) {
2606 spin_lock(&root->delalloc_lock);
2607 btrfs_del_delalloc_inode(inode);
2608 spin_unlock(&root->delalloc_lock);
2609 }
2610 }
2611
2612 if ((state->state & EXTENT_DELALLOC_NEW) &&
2613 (bits & EXTENT_DELALLOC_NEW)) {
2614 spin_lock(&inode->lock);
2615 ASSERT(inode->new_delalloc_bytes >= len);
2616 inode->new_delalloc_bytes -= len;
2617 if (bits & EXTENT_ADD_INODE_BYTES)
2618 inode_add_bytes(&inode->vfs_inode, len);
2619 spin_unlock(&inode->lock);
2620 }
2621 }
2622
2623 /*
2624 * given a list of ordered sums record them in the inode. This happens
2625 * at IO completion time based on sums calculated at bio submission time.
2626 */
add_pending_csums(struct btrfs_trans_handle * trans,struct list_head * list)2627 static int add_pending_csums(struct btrfs_trans_handle *trans,
2628 struct list_head *list)
2629 {
2630 struct btrfs_ordered_sum *sum;
2631 struct btrfs_root *csum_root = NULL;
2632 int ret;
2633
2634 list_for_each_entry(sum, list, list) {
2635 trans->adding_csums = true;
2636 if (!csum_root)
2637 csum_root = btrfs_csum_root(trans->fs_info,
2638 sum->logical);
2639 ret = btrfs_csum_file_blocks(trans, csum_root, sum);
2640 trans->adding_csums = false;
2641 if (ret)
2642 return ret;
2643 }
2644 return 0;
2645 }
2646
btrfs_find_new_delalloc_bytes(struct btrfs_inode * inode,const u64 start,const u64 len,struct extent_state ** cached_state)2647 static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
2648 const u64 start,
2649 const u64 len,
2650 struct extent_state **cached_state)
2651 {
2652 u64 search_start = start;
2653 const u64 end = start + len - 1;
2654
2655 while (search_start < end) {
2656 const u64 search_len = end - search_start + 1;
2657 struct extent_map *em;
2658 u64 em_len;
2659 int ret = 0;
2660
2661 em = btrfs_get_extent(inode, NULL, search_start, search_len);
2662 if (IS_ERR(em))
2663 return PTR_ERR(em);
2664
2665 if (em->disk_bytenr != EXTENT_MAP_HOLE)
2666 goto next;
2667
2668 em_len = em->len;
2669 if (em->start < search_start)
2670 em_len -= search_start - em->start;
2671 if (em_len > search_len)
2672 em_len = search_len;
2673
2674 ret = set_extent_bit(&inode->io_tree, search_start,
2675 search_start + em_len - 1,
2676 EXTENT_DELALLOC_NEW, cached_state);
2677 next:
2678 search_start = extent_map_end(em);
2679 free_extent_map(em);
2680 if (ret)
2681 return ret;
2682 }
2683 return 0;
2684 }
2685
btrfs_set_extent_delalloc(struct btrfs_inode * inode,u64 start,u64 end,unsigned int extra_bits,struct extent_state ** cached_state)2686 int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
2687 unsigned int extra_bits,
2688 struct extent_state **cached_state)
2689 {
2690 WARN_ON(PAGE_ALIGNED(end));
2691
2692 if (start >= i_size_read(&inode->vfs_inode) &&
2693 !(inode->flags & BTRFS_INODE_PREALLOC)) {
2694 /*
2695 * There can't be any extents following eof in this case so just
2696 * set the delalloc new bit for the range directly.
2697 */
2698 extra_bits |= EXTENT_DELALLOC_NEW;
2699 } else {
2700 int ret;
2701
2702 ret = btrfs_find_new_delalloc_bytes(inode, start,
2703 end + 1 - start,
2704 cached_state);
2705 if (ret)
2706 return ret;
2707 }
2708
2709 return set_extent_bit(&inode->io_tree, start, end,
2710 EXTENT_DELALLOC | extra_bits, cached_state);
2711 }
2712
2713 /* see btrfs_writepage_start_hook for details on why this is required */
2714 struct btrfs_writepage_fixup {
2715 struct folio *folio;
2716 struct btrfs_inode *inode;
2717 struct btrfs_work work;
2718 };
2719
btrfs_writepage_fixup_worker(struct btrfs_work * work)2720 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2721 {
2722 struct btrfs_writepage_fixup *fixup =
2723 container_of(work, struct btrfs_writepage_fixup, work);
2724 struct btrfs_ordered_extent *ordered;
2725 struct extent_state *cached_state = NULL;
2726 struct extent_changeset *data_reserved = NULL;
2727 struct folio *folio = fixup->folio;
2728 struct btrfs_inode *inode = fixup->inode;
2729 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2730 u64 page_start = folio_pos(folio);
2731 u64 page_end = folio_pos(folio) + folio_size(folio) - 1;
2732 int ret = 0;
2733 bool free_delalloc_space = true;
2734
2735 /*
2736 * This is similar to page_mkwrite, we need to reserve the space before
2737 * we take the folio lock.
2738 */
2739 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2740 folio_size(folio));
2741 again:
2742 folio_lock(folio);
2743
2744 /*
2745 * Before we queued this fixup, we took a reference on the folio.
2746 * folio->mapping may go NULL, but it shouldn't be moved to a different
2747 * address space.
2748 */
2749 if (!folio->mapping || !folio_test_dirty(folio) ||
2750 !folio_test_checked(folio)) {
2751 /*
2752 * Unfortunately this is a little tricky, either
2753 *
2754 * 1) We got here and our folio had already been dealt with and
2755 * we reserved our space, thus ret == 0, so we need to just
2756 * drop our space reservation and bail. This can happen the
2757 * first time we come into the fixup worker, or could happen
2758 * while waiting for the ordered extent.
2759 * 2) Our folio was already dealt with, but we happened to get an
2760 * ENOSPC above from the btrfs_delalloc_reserve_space. In
2761 * this case we obviously don't have anything to release, but
2762 * because the folio was already dealt with we don't want to
2763 * mark the folio with an error, so make sure we're resetting
2764 * ret to 0. This is why we have this check _before_ the ret
2765 * check, because we do not want to have a surprise ENOSPC
2766 * when the folio was already properly dealt with.
2767 */
2768 if (!ret) {
2769 btrfs_delalloc_release_extents(inode, folio_size(folio));
2770 btrfs_delalloc_release_space(inode, data_reserved,
2771 page_start, folio_size(folio),
2772 true);
2773 }
2774 ret = 0;
2775 goto out_page;
2776 }
2777
2778 /*
2779 * We can't mess with the folio state unless it is locked, so now that
2780 * it is locked bail if we failed to make our space reservation.
2781 */
2782 if (ret)
2783 goto out_page;
2784
2785 lock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2786
2787 /* already ordered? We're done */
2788 if (folio_test_ordered(folio))
2789 goto out_reserved;
2790
2791 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);
2792 if (ordered) {
2793 unlock_extent(&inode->io_tree, page_start, page_end,
2794 &cached_state);
2795 folio_unlock(folio);
2796 btrfs_start_ordered_extent(ordered);
2797 btrfs_put_ordered_extent(ordered);
2798 goto again;
2799 }
2800
2801 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,
2802 &cached_state);
2803 if (ret)
2804 goto out_reserved;
2805
2806 /*
2807 * Everything went as planned, we're now the owner of a dirty page with
2808 * delayed allocation bits set and space reserved for our COW
2809 * destination.
2810 *
2811 * The page was dirty when we started, nothing should have cleaned it.
2812 */
2813 BUG_ON(!folio_test_dirty(folio));
2814 free_delalloc_space = false;
2815 out_reserved:
2816 btrfs_delalloc_release_extents(inode, PAGE_SIZE);
2817 if (free_delalloc_space)
2818 btrfs_delalloc_release_space(inode, data_reserved, page_start,
2819 PAGE_SIZE, true);
2820 unlock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2821 out_page:
2822 if (ret) {
2823 /*
2824 * We hit ENOSPC or other errors. Update the mapping and page
2825 * to reflect the errors and clean the page.
2826 */
2827 mapping_set_error(folio->mapping, ret);
2828 btrfs_mark_ordered_io_finished(inode, folio, page_start,
2829 folio_size(folio), !ret);
2830 folio_clear_dirty_for_io(folio);
2831 }
2832 btrfs_folio_clear_checked(fs_info, folio, page_start, PAGE_SIZE);
2833 folio_unlock(folio);
2834 folio_put(folio);
2835 kfree(fixup);
2836 extent_changeset_free(data_reserved);
2837 /*
2838 * As a precaution, do a delayed iput in case it would be the last iput
2839 * that could need flushing space. Recursing back to fixup worker would
2840 * deadlock.
2841 */
2842 btrfs_add_delayed_iput(inode);
2843 }
2844
2845 /*
2846 * There are a few paths in the higher layers of the kernel that directly
2847 * set the folio dirty bit without asking the filesystem if it is a
2848 * good idea. This causes problems because we want to make sure COW
2849 * properly happens and the data=ordered rules are followed.
2850 *
2851 * In our case any range that doesn't have the ORDERED bit set
2852 * hasn't been properly setup for IO. We kick off an async process
2853 * to fix it up. The async helper will wait for ordered extents, set
2854 * the delalloc bit and make it safe to write the folio.
2855 */
btrfs_writepage_cow_fixup(struct folio * folio)2856 int btrfs_writepage_cow_fixup(struct folio *folio)
2857 {
2858 struct inode *inode = folio->mapping->host;
2859 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2860 struct btrfs_writepage_fixup *fixup;
2861
2862 /* This folio has ordered extent covering it already */
2863 if (folio_test_ordered(folio))
2864 return 0;
2865
2866 /*
2867 * folio_checked is set below when we create a fixup worker for this
2868 * folio, don't try to create another one if we're already
2869 * folio_test_checked.
2870 *
2871 * The extent_io writepage code will redirty the foio if we send back
2872 * EAGAIN.
2873 */
2874 if (folio_test_checked(folio))
2875 return -EAGAIN;
2876
2877 fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2878 if (!fixup)
2879 return -EAGAIN;
2880
2881 /*
2882 * We are already holding a reference to this inode from
2883 * write_cache_pages. We need to hold it because the space reservation
2884 * takes place outside of the folio lock, and we can't trust
2885 * page->mapping outside of the folio lock.
2886 */
2887 ihold(inode);
2888 btrfs_folio_set_checked(fs_info, folio, folio_pos(folio), folio_size(folio));
2889 folio_get(folio);
2890 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL);
2891 fixup->folio = folio;
2892 fixup->inode = BTRFS_I(inode);
2893 btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2894
2895 return -EAGAIN;
2896 }
2897
insert_reserved_file_extent(struct btrfs_trans_handle * trans,struct btrfs_inode * inode,u64 file_pos,struct btrfs_file_extent_item * stack_fi,const bool update_inode_bytes,u64 qgroup_reserved)2898 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2899 struct btrfs_inode *inode, u64 file_pos,
2900 struct btrfs_file_extent_item *stack_fi,
2901 const bool update_inode_bytes,
2902 u64 qgroup_reserved)
2903 {
2904 struct btrfs_root *root = inode->root;
2905 const u64 sectorsize = root->fs_info->sectorsize;
2906 struct btrfs_path *path;
2907 struct extent_buffer *leaf;
2908 struct btrfs_key ins;
2909 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi);
2910 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi);
2911 u64 offset = btrfs_stack_file_extent_offset(stack_fi);
2912 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi);
2913 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi);
2914 struct btrfs_drop_extents_args drop_args = { 0 };
2915 int ret;
2916
2917 path = btrfs_alloc_path();
2918 if (!path)
2919 return -ENOMEM;
2920
2921 /*
2922 * we may be replacing one extent in the tree with another.
2923 * The new extent is pinned in the extent map, and we don't want
2924 * to drop it from the cache until it is completely in the btree.
2925 *
2926 * So, tell btrfs_drop_extents to leave this extent in the cache.
2927 * the caller is expected to unpin it and allow it to be merged
2928 * with the others.
2929 */
2930 drop_args.path = path;
2931 drop_args.start = file_pos;
2932 drop_args.end = file_pos + num_bytes;
2933 drop_args.replace_extent = true;
2934 drop_args.extent_item_size = sizeof(*stack_fi);
2935 ret = btrfs_drop_extents(trans, root, inode, &drop_args);
2936 if (ret)
2937 goto out;
2938
2939 if (!drop_args.extent_inserted) {
2940 ins.objectid = btrfs_ino(inode);
2941 ins.offset = file_pos;
2942 ins.type = BTRFS_EXTENT_DATA_KEY;
2943
2944 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2945 sizeof(*stack_fi));
2946 if (ret)
2947 goto out;
2948 }
2949 leaf = path->nodes[0];
2950 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid);
2951 write_extent_buffer(leaf, stack_fi,
2952 btrfs_item_ptr_offset(leaf, path->slots[0]),
2953 sizeof(struct btrfs_file_extent_item));
2954
2955 btrfs_mark_buffer_dirty(trans, leaf);
2956 btrfs_release_path(path);
2957
2958 /*
2959 * If we dropped an inline extent here, we know the range where it is
2960 * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the
2961 * number of bytes only for that range containing the inline extent.
2962 * The remaining of the range will be processed when clearning the
2963 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.
2964 */
2965 if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) {
2966 u64 inline_size = round_down(drop_args.bytes_found, sectorsize);
2967
2968 inline_size = drop_args.bytes_found - inline_size;
2969 btrfs_update_inode_bytes(inode, sectorsize, inline_size);
2970 drop_args.bytes_found -= inline_size;
2971 num_bytes -= sectorsize;
2972 }
2973
2974 if (update_inode_bytes)
2975 btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found);
2976
2977 ins.objectid = disk_bytenr;
2978 ins.offset = disk_num_bytes;
2979 ins.type = BTRFS_EXTENT_ITEM_KEY;
2980
2981 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes);
2982 if (ret)
2983 goto out;
2984
2985 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode),
2986 file_pos - offset,
2987 qgroup_reserved, &ins);
2988 out:
2989 btrfs_free_path(path);
2990
2991 return ret;
2992 }
2993
btrfs_release_delalloc_bytes(struct btrfs_fs_info * fs_info,u64 start,u64 len)2994 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2995 u64 start, u64 len)
2996 {
2997 struct btrfs_block_group *cache;
2998
2999 cache = btrfs_lookup_block_group(fs_info, start);
3000 ASSERT(cache);
3001
3002 spin_lock(&cache->lock);
3003 cache->delalloc_bytes -= len;
3004 spin_unlock(&cache->lock);
3005
3006 btrfs_put_block_group(cache);
3007 }
3008
insert_ordered_extent_file_extent(struct btrfs_trans_handle * trans,struct btrfs_ordered_extent * oe)3009 static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,
3010 struct btrfs_ordered_extent *oe)
3011 {
3012 struct btrfs_file_extent_item stack_fi;
3013 bool update_inode_bytes;
3014 u64 num_bytes = oe->num_bytes;
3015 u64 ram_bytes = oe->ram_bytes;
3016
3017 memset(&stack_fi, 0, sizeof(stack_fi));
3018 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG);
3019 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr);
3020 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,
3021 oe->disk_num_bytes);
3022 btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset);
3023 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))
3024 num_bytes = oe->truncated_len;
3025 btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes);
3026 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes);
3027 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);
3028 /* Encryption and other encoding is reserved and all 0 */
3029
3030 /*
3031 * For delalloc, when completing an ordered extent we update the inode's
3032 * bytes when clearing the range in the inode's io tree, so pass false
3033 * as the argument 'update_inode_bytes' to insert_reserved_file_extent(),
3034 * except if the ordered extent was truncated.
3035 */
3036 update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) ||
3037 test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) ||
3038 test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags);
3039
3040 return insert_reserved_file_extent(trans, oe->inode,
3041 oe->file_offset, &stack_fi,
3042 update_inode_bytes, oe->qgroup_rsv);
3043 }
3044
3045 /*
3046 * As ordered data IO finishes, this gets called so we can finish
3047 * an ordered extent if the range of bytes in the file it covers are
3048 * fully written.
3049 */
btrfs_finish_one_ordered(struct btrfs_ordered_extent * ordered_extent)3050 int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
3051 {
3052 struct btrfs_inode *inode = ordered_extent->inode;
3053 struct btrfs_root *root = inode->root;
3054 struct btrfs_fs_info *fs_info = root->fs_info;
3055 struct btrfs_trans_handle *trans = NULL;
3056 struct extent_io_tree *io_tree = &inode->io_tree;
3057 struct extent_state *cached_state = NULL;
3058 u64 start, end;
3059 int compress_type = 0;
3060 int ret = 0;
3061 u64 logical_len = ordered_extent->num_bytes;
3062 bool freespace_inode;
3063 bool truncated = false;
3064 bool clear_reserved_extent = true;
3065 unsigned int clear_bits = EXTENT_DEFRAG;
3066
3067 start = ordered_extent->file_offset;
3068 end = start + ordered_extent->num_bytes - 1;
3069
3070 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3071 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
3072 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) &&
3073 !test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags))
3074 clear_bits |= EXTENT_DELALLOC_NEW;
3075
3076 freespace_inode = btrfs_is_free_space_inode(inode);
3077 if (!freespace_inode)
3078 btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);
3079
3080 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
3081 ret = -EIO;
3082 goto out;
3083 }
3084
3085 if (btrfs_is_zoned(fs_info))
3086 btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr,
3087 ordered_extent->disk_num_bytes);
3088
3089 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
3090 truncated = true;
3091 logical_len = ordered_extent->truncated_len;
3092 /* Truncated the entire extent, don't bother adding */
3093 if (!logical_len)
3094 goto out;
3095 }
3096
3097 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3098 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
3099
3100 btrfs_inode_safe_disk_i_size_write(inode, 0);
3101 if (freespace_inode)
3102 trans = btrfs_join_transaction_spacecache(root);
3103 else
3104 trans = btrfs_join_transaction(root);
3105 if (IS_ERR(trans)) {
3106 ret = PTR_ERR(trans);
3107 trans = NULL;
3108 goto out;
3109 }
3110 trans->block_rsv = &inode->block_rsv;
3111 ret = btrfs_update_inode_fallback(trans, inode);
3112 if (ret) /* -ENOMEM or corruption */
3113 btrfs_abort_transaction(trans, ret);
3114 goto out;
3115 }
3116
3117 clear_bits |= EXTENT_LOCKED;
3118 lock_extent(io_tree, start, end, &cached_state);
3119
3120 if (freespace_inode)
3121 trans = btrfs_join_transaction_spacecache(root);
3122 else
3123 trans = btrfs_join_transaction(root);
3124 if (IS_ERR(trans)) {
3125 ret = PTR_ERR(trans);
3126 trans = NULL;
3127 goto out;
3128 }
3129
3130 trans->block_rsv = &inode->block_rsv;
3131
3132 ret = btrfs_insert_raid_extent(trans, ordered_extent);
3133 if (ret)
3134 goto out;
3135
3136 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3137 compress_type = ordered_extent->compress_type;
3138 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3139 BUG_ON(compress_type);
3140 ret = btrfs_mark_extent_written(trans, inode,
3141 ordered_extent->file_offset,
3142 ordered_extent->file_offset +
3143 logical_len);
3144 btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr,
3145 ordered_extent->disk_num_bytes);
3146 } else {
3147 BUG_ON(root == fs_info->tree_root);
3148 ret = insert_ordered_extent_file_extent(trans, ordered_extent);
3149 if (!ret) {
3150 clear_reserved_extent = false;
3151 btrfs_release_delalloc_bytes(fs_info,
3152 ordered_extent->disk_bytenr,
3153 ordered_extent->disk_num_bytes);
3154 }
3155 }
3156 if (ret < 0) {
3157 btrfs_abort_transaction(trans, ret);
3158 goto out;
3159 }
3160
3161 ret = unpin_extent_cache(inode, ordered_extent->file_offset,
3162 ordered_extent->num_bytes, trans->transid);
3163 if (ret < 0) {
3164 btrfs_abort_transaction(trans, ret);
3165 goto out;
3166 }
3167
3168 ret = add_pending_csums(trans, &ordered_extent->list);
3169 if (ret) {
3170 btrfs_abort_transaction(trans, ret);
3171 goto out;
3172 }
3173
3174 /*
3175 * If this is a new delalloc range, clear its new delalloc flag to
3176 * update the inode's number of bytes. This needs to be done first
3177 * before updating the inode item.
3178 */
3179 if ((clear_bits & EXTENT_DELALLOC_NEW) &&
3180 !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags))
3181 clear_extent_bit(&inode->io_tree, start, end,
3182 EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES,
3183 &cached_state);
3184
3185 btrfs_inode_safe_disk_i_size_write(inode, 0);
3186 ret = btrfs_update_inode_fallback(trans, inode);
3187 if (ret) { /* -ENOMEM or corruption */
3188 btrfs_abort_transaction(trans, ret);
3189 goto out;
3190 }
3191 out:
3192 clear_extent_bit(&inode->io_tree, start, end, clear_bits,
3193 &cached_state);
3194
3195 if (trans)
3196 btrfs_end_transaction(trans);
3197
3198 if (ret || truncated) {
3199 u64 unwritten_start = start;
3200
3201 /*
3202 * If we failed to finish this ordered extent for any reason we
3203 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered
3204 * extent, and mark the inode with the error if it wasn't
3205 * already set. Any error during writeback would have already
3206 * set the mapping error, so we need to set it if we're the ones
3207 * marking this ordered extent as failed.
3208 */
3209 if (ret)
3210 btrfs_mark_ordered_extent_error(ordered_extent);
3211
3212 if (truncated)
3213 unwritten_start += logical_len;
3214 clear_extent_uptodate(io_tree, unwritten_start, end, NULL);
3215
3216 /*
3217 * Drop extent maps for the part of the extent we didn't write.
3218 *
3219 * We have an exception here for the free_space_inode, this is
3220 * because when we do btrfs_get_extent() on the free space inode
3221 * we will search the commit root. If this is a new block group
3222 * we won't find anything, and we will trip over the assert in
3223 * writepage where we do ASSERT(em->block_start !=
3224 * EXTENT_MAP_HOLE).
3225 *
3226 * Theoretically we could also skip this for any NOCOW extent as
3227 * we don't mess with the extent map tree in the NOCOW case, but
3228 * for now simply skip this if we are the free space inode.
3229 */
3230 if (!btrfs_is_free_space_inode(inode))
3231 btrfs_drop_extent_map_range(inode, unwritten_start,
3232 end, false);
3233
3234 /*
3235 * If the ordered extent had an IOERR or something else went
3236 * wrong we need to return the space for this ordered extent
3237 * back to the allocator. We only free the extent in the
3238 * truncated case if we didn't write out the extent at all.
3239 *
3240 * If we made it past insert_reserved_file_extent before we
3241 * errored out then we don't need to do this as the accounting
3242 * has already been done.
3243 */
3244 if ((ret || !logical_len) &&
3245 clear_reserved_extent &&
3246 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3247 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3248 /*
3249 * Discard the range before returning it back to the
3250 * free space pool
3251 */
3252 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC))
3253 btrfs_discard_extent(fs_info,
3254 ordered_extent->disk_bytenr,
3255 ordered_extent->disk_num_bytes,
3256 NULL);
3257 btrfs_free_reserved_extent(fs_info,
3258 ordered_extent->disk_bytenr,
3259 ordered_extent->disk_num_bytes, 1);
3260 /*
3261 * Actually free the qgroup rsv which was released when
3262 * the ordered extent was created.
3263 */
3264 btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(inode->root),
3265 ordered_extent->qgroup_rsv,
3266 BTRFS_QGROUP_RSV_DATA);
3267 }
3268 }
3269
3270 /*
3271 * This needs to be done to make sure anybody waiting knows we are done
3272 * updating everything for this ordered extent.
3273 */
3274 btrfs_remove_ordered_extent(inode, ordered_extent);
3275
3276 /* once for us */
3277 btrfs_put_ordered_extent(ordered_extent);
3278 /* once for the tree */
3279 btrfs_put_ordered_extent(ordered_extent);
3280
3281 return ret;
3282 }
3283
btrfs_finish_ordered_io(struct btrfs_ordered_extent * ordered)3284 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)
3285 {
3286 if (btrfs_is_zoned(ordered->inode->root->fs_info) &&
3287 !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) &&
3288 list_empty(&ordered->bioc_list))
3289 btrfs_finish_ordered_zoned(ordered);
3290 return btrfs_finish_one_ordered(ordered);
3291 }
3292
3293 /*
3294 * Verify the checksum for a single sector without any extra action that depend
3295 * on the type of I/O.
3296 */
btrfs_check_sector_csum(struct btrfs_fs_info * fs_info,struct page * page,u32 pgoff,u8 * csum,const u8 * const csum_expected)3297 int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page,
3298 u32 pgoff, u8 *csum, const u8 * const csum_expected)
3299 {
3300 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3301 char *kaddr;
3302
3303 ASSERT(pgoff + fs_info->sectorsize <= PAGE_SIZE);
3304
3305 shash->tfm = fs_info->csum_shash;
3306
3307 kaddr = kmap_local_page(page) + pgoff;
3308 crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);
3309 kunmap_local(kaddr);
3310
3311 if (memcmp(csum, csum_expected, fs_info->csum_size))
3312 return -EIO;
3313 return 0;
3314 }
3315
3316 /*
3317 * Verify the checksum of a single data sector.
3318 *
3319 * @bbio: btrfs_io_bio which contains the csum
3320 * @dev: device the sector is on
3321 * @bio_offset: offset to the beginning of the bio (in bytes)
3322 * @bv: bio_vec to check
3323 *
3324 * Check if the checksum on a data block is valid. When a checksum mismatch is
3325 * detected, report the error and fill the corrupted range with zero.
3326 *
3327 * Return %true if the sector is ok or had no checksum to start with, else %false.
3328 */
btrfs_data_csum_ok(struct btrfs_bio * bbio,struct btrfs_device * dev,u32 bio_offset,struct bio_vec * bv)3329 bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
3330 u32 bio_offset, struct bio_vec *bv)
3331 {
3332 struct btrfs_inode *inode = bbio->inode;
3333 struct btrfs_fs_info *fs_info = inode->root->fs_info;
3334 u64 file_offset = bbio->file_offset + bio_offset;
3335 u64 end = file_offset + bv->bv_len - 1;
3336 u8 *csum_expected;
3337 u8 csum[BTRFS_CSUM_SIZE];
3338
3339 ASSERT(bv->bv_len == fs_info->sectorsize);
3340
3341 if (!bbio->csum)
3342 return true;
3343
3344 if (btrfs_is_data_reloc_root(inode->root) &&
3345 test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM,
3346 NULL)) {
3347 /* Skip the range without csum for data reloc inode */
3348 clear_extent_bits(&inode->io_tree, file_offset, end,
3349 EXTENT_NODATASUM);
3350 return true;
3351 }
3352
3353 csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *
3354 fs_info->csum_size;
3355 if (btrfs_check_sector_csum(fs_info, bv->bv_page, bv->bv_offset, csum,
3356 csum_expected))
3357 goto zeroit;
3358 return true;
3359
3360 zeroit:
3361 btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected,
3362 bbio->mirror_num);
3363 if (dev)
3364 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS);
3365 memzero_bvec(bv);
3366 return false;
3367 }
3368
3369 /*
3370 * Perform a delayed iput on @inode.
3371 *
3372 * @inode: The inode we want to perform iput on
3373 *
3374 * This function uses the generic vfs_inode::i_count to track whether we should
3375 * just decrement it (in case it's > 1) or if this is the last iput then link
3376 * the inode to the delayed iput machinery. Delayed iputs are processed at
3377 * transaction commit time/superblock commit/cleaner kthread.
3378 */
btrfs_add_delayed_iput(struct btrfs_inode * inode)3379 void btrfs_add_delayed_iput(struct btrfs_inode *inode)
3380 {
3381 struct btrfs_fs_info *fs_info = inode->root->fs_info;
3382 unsigned long flags;
3383
3384 if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1))
3385 return;
3386
3387 atomic_inc(&fs_info->nr_delayed_iputs);
3388 /*
3389 * Need to be irq safe here because we can be called from either an irq
3390 * context (see bio.c and btrfs_put_ordered_extent()) or a non-irq
3391 * context.
3392 */
3393 spin_lock_irqsave(&fs_info->delayed_iput_lock, flags);
3394 ASSERT(list_empty(&inode->delayed_iput));
3395 list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs);
3396 spin_unlock_irqrestore(&fs_info->delayed_iput_lock, flags);
3397 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))
3398 wake_up_process(fs_info->cleaner_kthread);
3399 }
3400
run_delayed_iput_locked(struct btrfs_fs_info * fs_info,struct btrfs_inode * inode)3401 static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,
3402 struct btrfs_inode *inode)
3403 {
3404 list_del_init(&inode->delayed_iput);
3405 spin_unlock_irq(&fs_info->delayed_iput_lock);
3406 iput(&inode->vfs_inode);
3407 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs))
3408 wake_up(&fs_info->delayed_iputs_wait);
3409 spin_lock_irq(&fs_info->delayed_iput_lock);
3410 }
3411
btrfs_run_delayed_iput(struct btrfs_fs_info * fs_info,struct btrfs_inode * inode)3412 static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,
3413 struct btrfs_inode *inode)
3414 {
3415 if (!list_empty(&inode->delayed_iput)) {
3416 spin_lock_irq(&fs_info->delayed_iput_lock);
3417 if (!list_empty(&inode->delayed_iput))
3418 run_delayed_iput_locked(fs_info, inode);
3419 spin_unlock_irq(&fs_info->delayed_iput_lock);
3420 }
3421 }
3422
btrfs_run_delayed_iputs(struct btrfs_fs_info * fs_info)3423 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3424 {
3425 /*
3426 * btrfs_put_ordered_extent() can run in irq context (see bio.c), which
3427 * calls btrfs_add_delayed_iput() and that needs to lock
3428 * fs_info->delayed_iput_lock. So we need to disable irqs here to
3429 * prevent a deadlock.
3430 */
3431 spin_lock_irq(&fs_info->delayed_iput_lock);
3432 while (!list_empty(&fs_info->delayed_iputs)) {
3433 struct btrfs_inode *inode;
3434
3435 inode = list_first_entry(&fs_info->delayed_iputs,
3436 struct btrfs_inode, delayed_iput);
3437 run_delayed_iput_locked(fs_info, inode);
3438 if (need_resched()) {
3439 spin_unlock_irq(&fs_info->delayed_iput_lock);
3440 cond_resched();
3441 spin_lock_irq(&fs_info->delayed_iput_lock);
3442 }
3443 }
3444 spin_unlock_irq(&fs_info->delayed_iput_lock);
3445 }
3446
3447 /*
3448 * Wait for flushing all delayed iputs
3449 *
3450 * @fs_info: the filesystem
3451 *
3452 * This will wait on any delayed iputs that are currently running with KILLABLE
3453 * set. Once they are all done running we will return, unless we are killed in
3454 * which case we return EINTR. This helps in user operations like fallocate etc
3455 * that might get blocked on the iputs.
3456 *
3457 * Return EINTR if we were killed, 0 if nothing's pending
3458 */
btrfs_wait_on_delayed_iputs(struct btrfs_fs_info * fs_info)3459 int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)
3460 {
3461 int ret = wait_event_killable(fs_info->delayed_iputs_wait,
3462 atomic_read(&fs_info->nr_delayed_iputs) == 0);
3463 if (ret)
3464 return -EINTR;
3465 return 0;
3466 }
3467
3468 /*
3469 * This creates an orphan entry for the given inode in case something goes wrong
3470 * in the middle of an unlink.
3471 */
btrfs_orphan_add(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)3472 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3473 struct btrfs_inode *inode)
3474 {
3475 int ret;
3476
3477 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode));
3478 if (ret && ret != -EEXIST) {
3479 btrfs_abort_transaction(trans, ret);
3480 return ret;
3481 }
3482
3483 return 0;
3484 }
3485
3486 /*
3487 * We have done the delete so we can go ahead and remove the orphan item for
3488 * this particular inode.
3489 */
btrfs_orphan_del(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)3490 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3491 struct btrfs_inode *inode)
3492 {
3493 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode));
3494 }
3495
3496 /*
3497 * this cleans up any orphans that may be left on the list from the last use
3498 * of this root.
3499 */
btrfs_orphan_cleanup(struct btrfs_root * root)3500 int btrfs_orphan_cleanup(struct btrfs_root *root)
3501 {
3502 struct btrfs_fs_info *fs_info = root->fs_info;
3503 struct btrfs_path *path;
3504 struct extent_buffer *leaf;
3505 struct btrfs_key key, found_key;
3506 struct btrfs_trans_handle *trans;
3507 struct inode *inode;
3508 u64 last_objectid = 0;
3509 int ret = 0, nr_unlink = 0;
3510
3511 if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state))
3512 return 0;
3513
3514 path = btrfs_alloc_path();
3515 if (!path) {
3516 ret = -ENOMEM;
3517 goto out;
3518 }
3519 path->reada = READA_BACK;
3520
3521 key.objectid = BTRFS_ORPHAN_OBJECTID;
3522 key.type = BTRFS_ORPHAN_ITEM_KEY;
3523 key.offset = (u64)-1;
3524
3525 while (1) {
3526 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3527 if (ret < 0)
3528 goto out;
3529
3530 /*
3531 * if ret == 0 means we found what we were searching for, which
3532 * is weird, but possible, so only screw with path if we didn't
3533 * find the key and see if we have stuff that matches
3534 */
3535 if (ret > 0) {
3536 ret = 0;
3537 if (path->slots[0] == 0)
3538 break;
3539 path->slots[0]--;
3540 }
3541
3542 /* pull out the item */
3543 leaf = path->nodes[0];
3544 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3545
3546 /* make sure the item matches what we want */
3547 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3548 break;
3549 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3550 break;
3551
3552 /* release the path since we're done with it */
3553 btrfs_release_path(path);
3554
3555 /*
3556 * this is where we are basically btrfs_lookup, without the
3557 * crossing root thing. we store the inode number in the
3558 * offset of the orphan item.
3559 */
3560
3561 if (found_key.offset == last_objectid) {
3562 /*
3563 * We found the same inode as before. This means we were
3564 * not able to remove its items via eviction triggered
3565 * by an iput(). A transaction abort may have happened,
3566 * due to -ENOSPC for example, so try to grab the error
3567 * that lead to a transaction abort, if any.
3568 */
3569 btrfs_err(fs_info,
3570 "Error removing orphan entry, stopping orphan cleanup");
3571 ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL;
3572 goto out;
3573 }
3574
3575 last_objectid = found_key.offset;
3576
3577 found_key.objectid = found_key.offset;
3578 found_key.type = BTRFS_INODE_ITEM_KEY;
3579 found_key.offset = 0;
3580 inode = btrfs_iget(last_objectid, root);
3581 if (IS_ERR(inode)) {
3582 ret = PTR_ERR(inode);
3583 inode = NULL;
3584 if (ret != -ENOENT)
3585 goto out;
3586 }
3587
3588 if (!inode && root == fs_info->tree_root) {
3589 struct btrfs_root *dead_root;
3590 int is_dead_root = 0;
3591
3592 /*
3593 * This is an orphan in the tree root. Currently these
3594 * could come from 2 sources:
3595 * a) a root (snapshot/subvolume) deletion in progress
3596 * b) a free space cache inode
3597 * We need to distinguish those two, as the orphan item
3598 * for a root must not get deleted before the deletion
3599 * of the snapshot/subvolume's tree completes.
3600 *
3601 * btrfs_find_orphan_roots() ran before us, which has
3602 * found all deleted roots and loaded them into
3603 * fs_info->fs_roots_radix. So here we can find if an
3604 * orphan item corresponds to a deleted root by looking
3605 * up the root from that radix tree.
3606 */
3607
3608 spin_lock(&fs_info->fs_roots_radix_lock);
3609 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,
3610 (unsigned long)found_key.objectid);
3611 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0)
3612 is_dead_root = 1;
3613 spin_unlock(&fs_info->fs_roots_radix_lock);
3614
3615 if (is_dead_root) {
3616 /* prevent this orphan from being found again */
3617 key.offset = found_key.objectid - 1;
3618 continue;
3619 }
3620
3621 }
3622
3623 /*
3624 * If we have an inode with links, there are a couple of
3625 * possibilities:
3626 *
3627 * 1. We were halfway through creating fsverity metadata for the
3628 * file. In that case, the orphan item represents incomplete
3629 * fsverity metadata which must be cleaned up with
3630 * btrfs_drop_verity_items and deleting the orphan item.
3631
3632 * 2. Old kernels (before v3.12) used to create an
3633 * orphan item for truncate indicating that there were possibly
3634 * extent items past i_size that needed to be deleted. In v3.12,
3635 * truncate was changed to update i_size in sync with the extent
3636 * items, but the (useless) orphan item was still created. Since
3637 * v4.18, we don't create the orphan item for truncate at all.
3638 *
3639 * So, this item could mean that we need to do a truncate, but
3640 * only if this filesystem was last used on a pre-v3.12 kernel
3641 * and was not cleanly unmounted. The odds of that are quite
3642 * slim, and it's a pain to do the truncate now, so just delete
3643 * the orphan item.
3644 *
3645 * It's also possible that this orphan item was supposed to be
3646 * deleted but wasn't. The inode number may have been reused,
3647 * but either way, we can delete the orphan item.
3648 */
3649 if (!inode || inode->i_nlink) {
3650 if (inode) {
3651 ret = btrfs_drop_verity_items(BTRFS_I(inode));
3652 iput(inode);
3653 inode = NULL;
3654 if (ret)
3655 goto out;
3656 }
3657 trans = btrfs_start_transaction(root, 1);
3658 if (IS_ERR(trans)) {
3659 ret = PTR_ERR(trans);
3660 goto out;
3661 }
3662 btrfs_debug(fs_info, "auto deleting %Lu",
3663 found_key.objectid);
3664 ret = btrfs_del_orphan_item(trans, root,
3665 found_key.objectid);
3666 btrfs_end_transaction(trans);
3667 if (ret)
3668 goto out;
3669 continue;
3670 }
3671
3672 nr_unlink++;
3673
3674 /* this will do delete_inode and everything for us */
3675 iput(inode);
3676 }
3677 /* release the path since we're done with it */
3678 btrfs_release_path(path);
3679
3680 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3681 trans = btrfs_join_transaction(root);
3682 if (!IS_ERR(trans))
3683 btrfs_end_transaction(trans);
3684 }
3685
3686 if (nr_unlink)
3687 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3688
3689 out:
3690 if (ret)
3691 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3692 btrfs_free_path(path);
3693 return ret;
3694 }
3695
3696 /*
3697 * very simple check to peek ahead in the leaf looking for xattrs. If we
3698 * don't find any xattrs, we know there can't be any acls.
3699 *
3700 * slot is the slot the inode is in, objectid is the objectid of the inode
3701 */
acls_after_inode_item(struct extent_buffer * leaf,int slot,u64 objectid,int * first_xattr_slot)3702 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3703 int slot, u64 objectid,
3704 int *first_xattr_slot)
3705 {
3706 u32 nritems = btrfs_header_nritems(leaf);
3707 struct btrfs_key found_key;
3708 static u64 xattr_access = 0;
3709 static u64 xattr_default = 0;
3710 int scanned = 0;
3711
3712 if (!xattr_access) {
3713 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3714 strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3715 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3716 strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3717 }
3718
3719 slot++;
3720 *first_xattr_slot = -1;
3721 while (slot < nritems) {
3722 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3723
3724 /* we found a different objectid, there must not be acls */
3725 if (found_key.objectid != objectid)
3726 return 0;
3727
3728 /* we found an xattr, assume we've got an acl */
3729 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3730 if (*first_xattr_slot == -1)
3731 *first_xattr_slot = slot;
3732 if (found_key.offset == xattr_access ||
3733 found_key.offset == xattr_default)
3734 return 1;
3735 }
3736
3737 /*
3738 * we found a key greater than an xattr key, there can't
3739 * be any acls later on
3740 */
3741 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3742 return 0;
3743
3744 slot++;
3745 scanned++;
3746
3747 /*
3748 * it goes inode, inode backrefs, xattrs, extents,
3749 * so if there are a ton of hard links to an inode there can
3750 * be a lot of backrefs. Don't waste time searching too hard,
3751 * this is just an optimization
3752 */
3753 if (scanned >= 8)
3754 break;
3755 }
3756 /* we hit the end of the leaf before we found an xattr or
3757 * something larger than an xattr. We have to assume the inode
3758 * has acls
3759 */
3760 if (*first_xattr_slot == -1)
3761 *first_xattr_slot = slot;
3762 return 1;
3763 }
3764
btrfs_init_file_extent_tree(struct btrfs_inode * inode)3765 static int btrfs_init_file_extent_tree(struct btrfs_inode *inode)
3766 {
3767 struct btrfs_fs_info *fs_info = inode->root->fs_info;
3768
3769 if (WARN_ON_ONCE(inode->file_extent_tree))
3770 return 0;
3771 if (btrfs_fs_incompat(fs_info, NO_HOLES))
3772 return 0;
3773 if (!S_ISREG(inode->vfs_inode.i_mode))
3774 return 0;
3775 if (btrfs_is_free_space_inode(inode))
3776 return 0;
3777
3778 inode->file_extent_tree = kmalloc(sizeof(struct extent_io_tree), GFP_KERNEL);
3779 if (!inode->file_extent_tree)
3780 return -ENOMEM;
3781
3782 extent_io_tree_init(fs_info, inode->file_extent_tree, IO_TREE_INODE_FILE_EXTENT);
3783 /* Lockdep class is set only for the file extent tree. */
3784 lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class);
3785
3786 return 0;
3787 }
3788
3789 /*
3790 * read an inode from the btree into the in-memory inode
3791 */
btrfs_read_locked_inode(struct inode * inode,struct btrfs_path * in_path)3792 static int btrfs_read_locked_inode(struct inode *inode,
3793 struct btrfs_path *in_path)
3794 {
3795 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
3796 struct btrfs_path *path = in_path;
3797 struct extent_buffer *leaf;
3798 struct btrfs_inode_item *inode_item;
3799 struct btrfs_root *root = BTRFS_I(inode)->root;
3800 struct btrfs_key location;
3801 unsigned long ptr;
3802 int maybe_acls;
3803 u32 rdev;
3804 int ret;
3805 bool filled = false;
3806 int first_xattr_slot;
3807
3808 ret = btrfs_init_file_extent_tree(BTRFS_I(inode));
3809 if (ret)
3810 return ret;
3811
3812 ret = btrfs_fill_inode(inode, &rdev);
3813 if (!ret)
3814 filled = true;
3815
3816 if (!path) {
3817 path = btrfs_alloc_path();
3818 if (!path)
3819 return -ENOMEM;
3820 }
3821
3822 btrfs_get_inode_key(BTRFS_I(inode), &location);
3823
3824 ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3825 if (ret) {
3826 if (path != in_path)
3827 btrfs_free_path(path);
3828 return ret;
3829 }
3830
3831 leaf = path->nodes[0];
3832
3833 if (filled)
3834 goto cache_index;
3835
3836 inode_item = btrfs_item_ptr(leaf, path->slots[0],
3837 struct btrfs_inode_item);
3838 inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3839 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3840 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3841 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3842 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3843 btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0,
3844 round_up(i_size_read(inode), fs_info->sectorsize));
3845
3846 inode_set_atime(inode, btrfs_timespec_sec(leaf, &inode_item->atime),
3847 btrfs_timespec_nsec(leaf, &inode_item->atime));
3848
3849 inode_set_mtime(inode, btrfs_timespec_sec(leaf, &inode_item->mtime),
3850 btrfs_timespec_nsec(leaf, &inode_item->mtime));
3851
3852 inode_set_ctime(inode, btrfs_timespec_sec(leaf, &inode_item->ctime),
3853 btrfs_timespec_nsec(leaf, &inode_item->ctime));
3854
3855 BTRFS_I(inode)->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime);
3856 BTRFS_I(inode)->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime);
3857
3858 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3859 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3860 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3861
3862 inode_set_iversion_queried(inode,
3863 btrfs_inode_sequence(leaf, inode_item));
3864 inode->i_generation = BTRFS_I(inode)->generation;
3865 inode->i_rdev = 0;
3866 rdev = btrfs_inode_rdev(leaf, inode_item);
3867
3868 if (S_ISDIR(inode->i_mode))
3869 BTRFS_I(inode)->index_cnt = (u64)-1;
3870
3871 btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item),
3872 &BTRFS_I(inode)->flags, &BTRFS_I(inode)->ro_flags);
3873
3874 cache_index:
3875 /*
3876 * If we were modified in the current generation and evicted from memory
3877 * and then re-read we need to do a full sync since we don't have any
3878 * idea about which extents were modified before we were evicted from
3879 * cache.
3880 *
3881 * This is required for both inode re-read from disk and delayed inode
3882 * in the delayed_nodes xarray.
3883 */
3884 if (BTRFS_I(inode)->last_trans == btrfs_get_fs_generation(fs_info))
3885 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3886 &BTRFS_I(inode)->runtime_flags);
3887
3888 /*
3889 * We don't persist the id of the transaction where an unlink operation
3890 * against the inode was last made. So here we assume the inode might
3891 * have been evicted, and therefore the exact value of last_unlink_trans
3892 * lost, and set it to last_trans to avoid metadata inconsistencies
3893 * between the inode and its parent if the inode is fsync'ed and the log
3894 * replayed. For example, in the scenario:
3895 *
3896 * touch mydir/foo
3897 * ln mydir/foo mydir/bar
3898 * sync
3899 * unlink mydir/bar
3900 * echo 2 > /proc/sys/vm/drop_caches # evicts inode
3901 * xfs_io -c fsync mydir/foo
3902 * <power failure>
3903 * mount fs, triggers fsync log replay
3904 *
3905 * We must make sure that when we fsync our inode foo we also log its
3906 * parent inode, otherwise after log replay the parent still has the
3907 * dentry with the "bar" name but our inode foo has a link count of 1
3908 * and doesn't have an inode ref with the name "bar" anymore.
3909 *
3910 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3911 * but it guarantees correctness at the expense of occasional full
3912 * transaction commits on fsync if our inode is a directory, or if our
3913 * inode is not a directory, logging its parent unnecessarily.
3914 */
3915 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3916
3917 /*
3918 * Same logic as for last_unlink_trans. We don't persist the generation
3919 * of the last transaction where this inode was used for a reflink
3920 * operation, so after eviction and reloading the inode we must be
3921 * pessimistic and assume the last transaction that modified the inode.
3922 */
3923 BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans;
3924
3925 path->slots[0]++;
3926 if (inode->i_nlink != 1 ||
3927 path->slots[0] >= btrfs_header_nritems(leaf))
3928 goto cache_acl;
3929
3930 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3931 if (location.objectid != btrfs_ino(BTRFS_I(inode)))
3932 goto cache_acl;
3933
3934 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3935 if (location.type == BTRFS_INODE_REF_KEY) {
3936 struct btrfs_inode_ref *ref;
3937
3938 ref = (struct btrfs_inode_ref *)ptr;
3939 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3940 } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3941 struct btrfs_inode_extref *extref;
3942
3943 extref = (struct btrfs_inode_extref *)ptr;
3944 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3945 extref);
3946 }
3947 cache_acl:
3948 /*
3949 * try to precache a NULL acl entry for files that don't have
3950 * any xattrs or acls
3951 */
3952 maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3953 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
3954 if (first_xattr_slot != -1) {
3955 path->slots[0] = first_xattr_slot;
3956 ret = btrfs_load_inode_props(inode, path);
3957 if (ret)
3958 btrfs_err(fs_info,
3959 "error loading props for ino %llu (root %llu): %d",
3960 btrfs_ino(BTRFS_I(inode)),
3961 btrfs_root_id(root), ret);
3962 }
3963 if (path != in_path)
3964 btrfs_free_path(path);
3965
3966 if (!maybe_acls)
3967 cache_no_acl(inode);
3968
3969 switch (inode->i_mode & S_IFMT) {
3970 case S_IFREG:
3971 inode->i_mapping->a_ops = &btrfs_aops;
3972 inode->i_fop = &btrfs_file_operations;
3973 inode->i_op = &btrfs_file_inode_operations;
3974 break;
3975 case S_IFDIR:
3976 inode->i_fop = &btrfs_dir_file_operations;
3977 inode->i_op = &btrfs_dir_inode_operations;
3978 break;
3979 case S_IFLNK:
3980 inode->i_op = &btrfs_symlink_inode_operations;
3981 inode_nohighmem(inode);
3982 inode->i_mapping->a_ops = &btrfs_aops;
3983 break;
3984 default:
3985 inode->i_op = &btrfs_special_inode_operations;
3986 init_special_inode(inode, inode->i_mode, rdev);
3987 break;
3988 }
3989
3990 btrfs_sync_inode_flags_to_i_flags(inode);
3991 return 0;
3992 }
3993
3994 /*
3995 * given a leaf and an inode, copy the inode fields into the leaf
3996 */
fill_inode_item(struct btrfs_trans_handle * trans,struct extent_buffer * leaf,struct btrfs_inode_item * item,struct inode * inode)3997 static void fill_inode_item(struct btrfs_trans_handle *trans,
3998 struct extent_buffer *leaf,
3999 struct btrfs_inode_item *item,
4000 struct inode *inode)
4001 {
4002 struct btrfs_map_token token;
4003 u64 flags;
4004
4005 btrfs_init_map_token(&token, leaf);
4006
4007 btrfs_set_token_inode_uid(&token, item, i_uid_read(inode));
4008 btrfs_set_token_inode_gid(&token, item, i_gid_read(inode));
4009 btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size);
4010 btrfs_set_token_inode_mode(&token, item, inode->i_mode);
4011 btrfs_set_token_inode_nlink(&token, item, inode->i_nlink);
4012
4013 btrfs_set_token_timespec_sec(&token, &item->atime,
4014 inode_get_atime_sec(inode));
4015 btrfs_set_token_timespec_nsec(&token, &item->atime,
4016 inode_get_atime_nsec(inode));
4017
4018 btrfs_set_token_timespec_sec(&token, &item->mtime,
4019 inode_get_mtime_sec(inode));
4020 btrfs_set_token_timespec_nsec(&token, &item->mtime,
4021 inode_get_mtime_nsec(inode));
4022
4023 btrfs_set_token_timespec_sec(&token, &item->ctime,
4024 inode_get_ctime_sec(inode));
4025 btrfs_set_token_timespec_nsec(&token, &item->ctime,
4026 inode_get_ctime_nsec(inode));
4027
4028 btrfs_set_token_timespec_sec(&token, &item->otime, BTRFS_I(inode)->i_otime_sec);
4029 btrfs_set_token_timespec_nsec(&token, &item->otime, BTRFS_I(inode)->i_otime_nsec);
4030
4031 btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode));
4032 btrfs_set_token_inode_generation(&token, item,
4033 BTRFS_I(inode)->generation);
4034 btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode));
4035 btrfs_set_token_inode_transid(&token, item, trans->transid);
4036 btrfs_set_token_inode_rdev(&token, item, inode->i_rdev);
4037 flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags,
4038 BTRFS_I(inode)->ro_flags);
4039 btrfs_set_token_inode_flags(&token, item, flags);
4040 btrfs_set_token_inode_block_group(&token, item, 0);
4041 }
4042
4043 /*
4044 * copy everything in the in-memory inode into the btree.
4045 */
btrfs_update_inode_item(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)4046 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
4047 struct btrfs_inode *inode)
4048 {
4049 struct btrfs_inode_item *inode_item;
4050 struct btrfs_path *path;
4051 struct extent_buffer *leaf;
4052 struct btrfs_key key;
4053 int ret;
4054
4055 path = btrfs_alloc_path();
4056 if (!path)
4057 return -ENOMEM;
4058
4059 btrfs_get_inode_key(inode, &key);
4060 ret = btrfs_lookup_inode(trans, inode->root, path, &key, 1);
4061 if (ret) {
4062 if (ret > 0)
4063 ret = -ENOENT;
4064 goto failed;
4065 }
4066
4067 leaf = path->nodes[0];
4068 inode_item = btrfs_item_ptr(leaf, path->slots[0],
4069 struct btrfs_inode_item);
4070
4071 fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode);
4072 btrfs_mark_buffer_dirty(trans, leaf);
4073 btrfs_set_inode_last_trans(trans, inode);
4074 ret = 0;
4075 failed:
4076 btrfs_free_path(path);
4077 return ret;
4078 }
4079
4080 /*
4081 * copy everything in the in-memory inode into the btree.
4082 */
btrfs_update_inode(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)4083 int btrfs_update_inode(struct btrfs_trans_handle *trans,
4084 struct btrfs_inode *inode)
4085 {
4086 struct btrfs_root *root = inode->root;
4087 struct btrfs_fs_info *fs_info = root->fs_info;
4088 int ret;
4089
4090 /*
4091 * If the inode is a free space inode, we can deadlock during commit
4092 * if we put it into the delayed code.
4093 *
4094 * The data relocation inode should also be directly updated
4095 * without delay
4096 */
4097 if (!btrfs_is_free_space_inode(inode)
4098 && !btrfs_is_data_reloc_root(root)
4099 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4100 btrfs_update_root_times(trans, root);
4101
4102 ret = btrfs_delayed_update_inode(trans, inode);
4103 if (!ret)
4104 btrfs_set_inode_last_trans(trans, inode);
4105 return ret;
4106 }
4107
4108 return btrfs_update_inode_item(trans, inode);
4109 }
4110
btrfs_update_inode_fallback(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)4111 int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4112 struct btrfs_inode *inode)
4113 {
4114 int ret;
4115
4116 ret = btrfs_update_inode(trans, inode);
4117 if (ret == -ENOSPC)
4118 return btrfs_update_inode_item(trans, inode);
4119 return ret;
4120 }
4121
4122 /*
4123 * unlink helper that gets used here in inode.c and in the tree logging
4124 * recovery code. It remove a link in a directory with a given name, and
4125 * also drops the back refs in the inode to the directory
4126 */
__btrfs_unlink_inode(struct btrfs_trans_handle * trans,struct btrfs_inode * dir,struct btrfs_inode * inode,const struct fscrypt_str * name,struct btrfs_rename_ctx * rename_ctx)4127 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4128 struct btrfs_inode *dir,
4129 struct btrfs_inode *inode,
4130 const struct fscrypt_str *name,
4131 struct btrfs_rename_ctx *rename_ctx)
4132 {
4133 struct btrfs_root *root = dir->root;
4134 struct btrfs_fs_info *fs_info = root->fs_info;
4135 struct btrfs_path *path;
4136 int ret = 0;
4137 struct btrfs_dir_item *di;
4138 u64 index;
4139 u64 ino = btrfs_ino(inode);
4140 u64 dir_ino = btrfs_ino(dir);
4141
4142 path = btrfs_alloc_path();
4143 if (!path) {
4144 ret = -ENOMEM;
4145 goto out;
4146 }
4147
4148 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1);
4149 if (IS_ERR_OR_NULL(di)) {
4150 ret = di ? PTR_ERR(di) : -ENOENT;
4151 goto err;
4152 }
4153 ret = btrfs_delete_one_dir_name(trans, root, path, di);
4154 if (ret)
4155 goto err;
4156 btrfs_release_path(path);
4157
4158 /*
4159 * If we don't have dir index, we have to get it by looking up
4160 * the inode ref, since we get the inode ref, remove it directly,
4161 * it is unnecessary to do delayed deletion.
4162 *
4163 * But if we have dir index, needn't search inode ref to get it.
4164 * Since the inode ref is close to the inode item, it is better
4165 * that we delay to delete it, and just do this deletion when
4166 * we update the inode item.
4167 */
4168 if (inode->dir_index) {
4169 ret = btrfs_delayed_delete_inode_ref(inode);
4170 if (!ret) {
4171 index = inode->dir_index;
4172 goto skip_backref;
4173 }
4174 }
4175
4176 ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index);
4177 if (ret) {
4178 btrfs_info(fs_info,
4179 "failed to delete reference to %.*s, inode %llu parent %llu",
4180 name->len, name->name, ino, dir_ino);
4181 btrfs_abort_transaction(trans, ret);
4182 goto err;
4183 }
4184 skip_backref:
4185 if (rename_ctx)
4186 rename_ctx->index = index;
4187
4188 ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4189 if (ret) {
4190 btrfs_abort_transaction(trans, ret);
4191 goto err;
4192 }
4193
4194 /*
4195 * If we are in a rename context, we don't need to update anything in the
4196 * log. That will be done later during the rename by btrfs_log_new_name().
4197 * Besides that, doing it here would only cause extra unnecessary btree
4198 * operations on the log tree, increasing latency for applications.
4199 */
4200 if (!rename_ctx) {
4201 btrfs_del_inode_ref_in_log(trans, root, name, inode, dir_ino);
4202 btrfs_del_dir_entries_in_log(trans, root, name, dir, index);
4203 }
4204
4205 /*
4206 * If we have a pending delayed iput we could end up with the final iput
4207 * being run in btrfs-cleaner context. If we have enough of these built
4208 * up we can end up burning a lot of time in btrfs-cleaner without any
4209 * way to throttle the unlinks. Since we're currently holding a ref on
4210 * the inode we can run the delayed iput here without any issues as the
4211 * final iput won't be done until after we drop the ref we're currently
4212 * holding.
4213 */
4214 btrfs_run_delayed_iput(fs_info, inode);
4215 err:
4216 btrfs_free_path(path);
4217 if (ret)
4218 goto out;
4219
4220 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2);
4221 inode_inc_iversion(&inode->vfs_inode);
4222 inode_set_ctime_current(&inode->vfs_inode);
4223 inode_inc_iversion(&dir->vfs_inode);
4224 inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode));
4225 ret = btrfs_update_inode(trans, dir);
4226 out:
4227 return ret;
4228 }
4229
btrfs_unlink_inode(struct btrfs_trans_handle * trans,struct btrfs_inode * dir,struct btrfs_inode * inode,const struct fscrypt_str * name)4230 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4231 struct btrfs_inode *dir, struct btrfs_inode *inode,
4232 const struct fscrypt_str *name)
4233 {
4234 int ret;
4235
4236 ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL);
4237 if (!ret) {
4238 drop_nlink(&inode->vfs_inode);
4239 ret = btrfs_update_inode(trans, inode);
4240 }
4241 return ret;
4242 }
4243
4244 /*
4245 * helper to start transaction for unlink and rmdir.
4246 *
4247 * unlink and rmdir are special in btrfs, they do not always free space, so
4248 * if we cannot make our reservations the normal way try and see if there is
4249 * plenty of slack room in the global reserve to migrate, otherwise we cannot
4250 * allow the unlink to occur.
4251 */
__unlink_start_trans(struct btrfs_inode * dir)4252 static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir)
4253 {
4254 struct btrfs_root *root = dir->root;
4255
4256 return btrfs_start_transaction_fallback_global_rsv(root,
4257 BTRFS_UNLINK_METADATA_UNITS);
4258 }
4259
btrfs_unlink(struct inode * dir,struct dentry * dentry)4260 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4261 {
4262 struct btrfs_trans_handle *trans;
4263 struct inode *inode = d_inode(dentry);
4264 int ret;
4265 struct fscrypt_name fname;
4266
4267 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname);
4268 if (ret)
4269 return ret;
4270
4271 /* This needs to handle no-key deletions later on */
4272
4273 trans = __unlink_start_trans(BTRFS_I(dir));
4274 if (IS_ERR(trans)) {
4275 ret = PTR_ERR(trans);
4276 goto fscrypt_free;
4277 }
4278
4279 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4280 false);
4281
4282 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4283 &fname.disk_name);
4284 if (ret)
4285 goto end_trans;
4286
4287 if (inode->i_nlink == 0) {
4288 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4289 if (ret)
4290 goto end_trans;
4291 }
4292
4293 end_trans:
4294 btrfs_end_transaction(trans);
4295 btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info);
4296 fscrypt_free:
4297 fscrypt_free_filename(&fname);
4298 return ret;
4299 }
4300
btrfs_unlink_subvol(struct btrfs_trans_handle * trans,struct btrfs_inode * dir,struct dentry * dentry)4301 static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4302 struct btrfs_inode *dir, struct dentry *dentry)
4303 {
4304 struct btrfs_root *root = dir->root;
4305 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4306 struct btrfs_path *path;
4307 struct extent_buffer *leaf;
4308 struct btrfs_dir_item *di;
4309 struct btrfs_key key;
4310 u64 index;
4311 int ret;
4312 u64 objectid;
4313 u64 dir_ino = btrfs_ino(dir);
4314 struct fscrypt_name fname;
4315
4316 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);
4317 if (ret)
4318 return ret;
4319
4320 /* This needs to handle no-key deletions later on */
4321
4322 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4323 objectid = btrfs_root_id(inode->root);
4324 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4325 objectid = inode->ref_root_id;
4326 } else {
4327 WARN_ON(1);
4328 fscrypt_free_filename(&fname);
4329 return -EINVAL;
4330 }
4331
4332 path = btrfs_alloc_path();
4333 if (!path) {
4334 ret = -ENOMEM;
4335 goto out;
4336 }
4337
4338 di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4339 &fname.disk_name, -1);
4340 if (IS_ERR_OR_NULL(di)) {
4341 ret = di ? PTR_ERR(di) : -ENOENT;
4342 goto out;
4343 }
4344
4345 leaf = path->nodes[0];
4346 btrfs_dir_item_key_to_cpu(leaf, di, &key);
4347 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4348 ret = btrfs_delete_one_dir_name(trans, root, path, di);
4349 if (ret) {
4350 btrfs_abort_transaction(trans, ret);
4351 goto out;
4352 }
4353 btrfs_release_path(path);
4354
4355 /*
4356 * This is a placeholder inode for a subvolume we didn't have a
4357 * reference to at the time of the snapshot creation. In the meantime
4358 * we could have renamed the real subvol link into our snapshot, so
4359 * depending on btrfs_del_root_ref to return -ENOENT here is incorrect.
4360 * Instead simply lookup the dir_index_item for this entry so we can
4361 * remove it. Otherwise we know we have a ref to the root and we can
4362 * call btrfs_del_root_ref, and it _shouldn't_ fail.
4363 */
4364 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4365 di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name);
4366 if (IS_ERR_OR_NULL(di)) {
4367 if (!di)
4368 ret = -ENOENT;
4369 else
4370 ret = PTR_ERR(di);
4371 btrfs_abort_transaction(trans, ret);
4372 goto out;
4373 }
4374
4375 leaf = path->nodes[0];
4376 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4377 index = key.offset;
4378 btrfs_release_path(path);
4379 } else {
4380 ret = btrfs_del_root_ref(trans, objectid,
4381 btrfs_root_id(root), dir_ino,
4382 &index, &fname.disk_name);
4383 if (ret) {
4384 btrfs_abort_transaction(trans, ret);
4385 goto out;
4386 }
4387 }
4388
4389 ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4390 if (ret) {
4391 btrfs_abort_transaction(trans, ret);
4392 goto out;
4393 }
4394
4395 btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2);
4396 inode_inc_iversion(&dir->vfs_inode);
4397 inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode));
4398 ret = btrfs_update_inode_fallback(trans, dir);
4399 if (ret)
4400 btrfs_abort_transaction(trans, ret);
4401 out:
4402 btrfs_free_path(path);
4403 fscrypt_free_filename(&fname);
4404 return ret;
4405 }
4406
4407 /*
4408 * Helper to check if the subvolume references other subvolumes or if it's
4409 * default.
4410 */
may_destroy_subvol(struct btrfs_root * root)4411 static noinline int may_destroy_subvol(struct btrfs_root *root)
4412 {
4413 struct btrfs_fs_info *fs_info = root->fs_info;
4414 struct btrfs_path *path;
4415 struct btrfs_dir_item *di;
4416 struct btrfs_key key;
4417 struct fscrypt_str name = FSTR_INIT("default", 7);
4418 u64 dir_id;
4419 int ret;
4420
4421 path = btrfs_alloc_path();
4422 if (!path)
4423 return -ENOMEM;
4424
4425 /* Make sure this root isn't set as the default subvol */
4426 dir_id = btrfs_super_root_dir(fs_info->super_copy);
4427 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,
4428 dir_id, &name, 0);
4429 if (di && !IS_ERR(di)) {
4430 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
4431 if (key.objectid == btrfs_root_id(root)) {
4432 ret = -EPERM;
4433 btrfs_err(fs_info,
4434 "deleting default subvolume %llu is not allowed",
4435 key.objectid);
4436 goto out;
4437 }
4438 btrfs_release_path(path);
4439 }
4440
4441 key.objectid = btrfs_root_id(root);
4442 key.type = BTRFS_ROOT_REF_KEY;
4443 key.offset = (u64)-1;
4444
4445 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4446 if (ret < 0)
4447 goto out;
4448 if (ret == 0) {
4449 /*
4450 * Key with offset -1 found, there would have to exist a root
4451 * with such id, but this is out of valid range.
4452 */
4453 ret = -EUCLEAN;
4454 goto out;
4455 }
4456
4457 ret = 0;
4458 if (path->slots[0] > 0) {
4459 path->slots[0]--;
4460 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
4461 if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY)
4462 ret = -ENOTEMPTY;
4463 }
4464 out:
4465 btrfs_free_path(path);
4466 return ret;
4467 }
4468
4469 /* Delete all dentries for inodes belonging to the root */
btrfs_prune_dentries(struct btrfs_root * root)4470 static void btrfs_prune_dentries(struct btrfs_root *root)
4471 {
4472 struct btrfs_fs_info *fs_info = root->fs_info;
4473 struct btrfs_inode *inode;
4474 u64 min_ino = 0;
4475
4476 if (!BTRFS_FS_ERROR(fs_info))
4477 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
4478
4479 inode = btrfs_find_first_inode(root, min_ino);
4480 while (inode) {
4481 if (atomic_read(&inode->vfs_inode.i_count) > 1)
4482 d_prune_aliases(&inode->vfs_inode);
4483
4484 min_ino = btrfs_ino(inode) + 1;
4485 /*
4486 * btrfs_drop_inode() will have it removed from the inode
4487 * cache when its usage count hits zero.
4488 */
4489 iput(&inode->vfs_inode);
4490 cond_resched();
4491 inode = btrfs_find_first_inode(root, min_ino);
4492 }
4493 }
4494
btrfs_delete_subvolume(struct btrfs_inode * dir,struct dentry * dentry)4495 int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry)
4496 {
4497 struct btrfs_root *root = dir->root;
4498 struct btrfs_fs_info *fs_info = root->fs_info;
4499 struct inode *inode = d_inode(dentry);
4500 struct btrfs_root *dest = BTRFS_I(inode)->root;
4501 struct btrfs_trans_handle *trans;
4502 struct btrfs_block_rsv block_rsv;
4503 u64 root_flags;
4504 u64 qgroup_reserved = 0;
4505 int ret;
4506
4507 down_write(&fs_info->subvol_sem);
4508
4509 /*
4510 * Don't allow to delete a subvolume with send in progress. This is
4511 * inside the inode lock so the error handling that has to drop the bit
4512 * again is not run concurrently.
4513 */
4514 spin_lock(&dest->root_item_lock);
4515 if (dest->send_in_progress) {
4516 spin_unlock(&dest->root_item_lock);
4517 btrfs_warn(fs_info,
4518 "attempt to delete subvolume %llu during send",
4519 btrfs_root_id(dest));
4520 ret = -EPERM;
4521 goto out_up_write;
4522 }
4523 if (atomic_read(&dest->nr_swapfiles)) {
4524 spin_unlock(&dest->root_item_lock);
4525 btrfs_warn(fs_info,
4526 "attempt to delete subvolume %llu with active swapfile",
4527 btrfs_root_id(root));
4528 ret = -EPERM;
4529 goto out_up_write;
4530 }
4531 root_flags = btrfs_root_flags(&dest->root_item);
4532 btrfs_set_root_flags(&dest->root_item,
4533 root_flags | BTRFS_ROOT_SUBVOL_DEAD);
4534 spin_unlock(&dest->root_item_lock);
4535
4536 ret = may_destroy_subvol(dest);
4537 if (ret)
4538 goto out_undead;
4539
4540 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP);
4541 /*
4542 * One for dir inode,
4543 * two for dir entries,
4544 * two for root ref/backref.
4545 */
4546 ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true);
4547 if (ret)
4548 goto out_undead;
4549 qgroup_reserved = block_rsv.qgroup_rsv_reserved;
4550
4551 trans = btrfs_start_transaction(root, 0);
4552 if (IS_ERR(trans)) {
4553 ret = PTR_ERR(trans);
4554 goto out_release;
4555 }
4556 btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved);
4557 qgroup_reserved = 0;
4558 trans->block_rsv = &block_rsv;
4559 trans->bytes_reserved = block_rsv.size;
4560
4561 btrfs_record_snapshot_destroy(trans, dir);
4562
4563 ret = btrfs_unlink_subvol(trans, dir, dentry);
4564 if (ret) {
4565 btrfs_abort_transaction(trans, ret);
4566 goto out_end_trans;
4567 }
4568
4569 ret = btrfs_record_root_in_trans(trans, dest);
4570 if (ret) {
4571 btrfs_abort_transaction(trans, ret);
4572 goto out_end_trans;
4573 }
4574
4575 memset(&dest->root_item.drop_progress, 0,
4576 sizeof(dest->root_item.drop_progress));
4577 btrfs_set_root_drop_level(&dest->root_item, 0);
4578 btrfs_set_root_refs(&dest->root_item, 0);
4579
4580 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) {
4581 ret = btrfs_insert_orphan_item(trans,
4582 fs_info->tree_root,
4583 btrfs_root_id(dest));
4584 if (ret) {
4585 btrfs_abort_transaction(trans, ret);
4586 goto out_end_trans;
4587 }
4588 }
4589
4590 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid,
4591 BTRFS_UUID_KEY_SUBVOL, btrfs_root_id(dest));
4592 if (ret && ret != -ENOENT) {
4593 btrfs_abort_transaction(trans, ret);
4594 goto out_end_trans;
4595 }
4596 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) {
4597 ret = btrfs_uuid_tree_remove(trans,
4598 dest->root_item.received_uuid,
4599 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4600 btrfs_root_id(dest));
4601 if (ret && ret != -ENOENT) {
4602 btrfs_abort_transaction(trans, ret);
4603 goto out_end_trans;
4604 }
4605 }
4606
4607 free_anon_bdev(dest->anon_dev);
4608 dest->anon_dev = 0;
4609 out_end_trans:
4610 trans->block_rsv = NULL;
4611 trans->bytes_reserved = 0;
4612 ret = btrfs_end_transaction(trans);
4613 inode->i_flags |= S_DEAD;
4614 out_release:
4615 btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL);
4616 if (qgroup_reserved)
4617 btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved);
4618 out_undead:
4619 if (ret) {
4620 spin_lock(&dest->root_item_lock);
4621 root_flags = btrfs_root_flags(&dest->root_item);
4622 btrfs_set_root_flags(&dest->root_item,
4623 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);
4624 spin_unlock(&dest->root_item_lock);
4625 }
4626 out_up_write:
4627 up_write(&fs_info->subvol_sem);
4628 if (!ret) {
4629 d_invalidate(dentry);
4630 btrfs_prune_dentries(dest);
4631 ASSERT(dest->send_in_progress == 0);
4632 }
4633
4634 return ret;
4635 }
4636
btrfs_rmdir(struct inode * dir,struct dentry * dentry)4637 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4638 {
4639 struct inode *inode = d_inode(dentry);
4640 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
4641 int ret = 0;
4642 struct btrfs_trans_handle *trans;
4643 u64 last_unlink_trans;
4644 struct fscrypt_name fname;
4645
4646 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4647 return -ENOTEMPTY;
4648 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) {
4649 if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) {
4650 btrfs_err(fs_info,
4651 "extent tree v2 doesn't support snapshot deletion yet");
4652 return -EOPNOTSUPP;
4653 }
4654 return btrfs_delete_subvolume(BTRFS_I(dir), dentry);
4655 }
4656
4657 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname);
4658 if (ret)
4659 return ret;
4660
4661 /* This needs to handle no-key deletions later on */
4662
4663 trans = __unlink_start_trans(BTRFS_I(dir));
4664 if (IS_ERR(trans)) {
4665 ret = PTR_ERR(trans);
4666 goto out_notrans;
4667 }
4668
4669 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4670 ret = btrfs_unlink_subvol(trans, BTRFS_I(dir), dentry);
4671 goto out;
4672 }
4673
4674 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4675 if (ret)
4676 goto out;
4677
4678 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4679
4680 /* now the directory is empty */
4681 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4682 &fname.disk_name);
4683 if (!ret) {
4684 btrfs_i_size_write(BTRFS_I(inode), 0);
4685 /*
4686 * Propagate the last_unlink_trans value of the deleted dir to
4687 * its parent directory. This is to prevent an unrecoverable
4688 * log tree in the case we do something like this:
4689 * 1) create dir foo
4690 * 2) create snapshot under dir foo
4691 * 3) delete the snapshot
4692 * 4) rmdir foo
4693 * 5) mkdir foo
4694 * 6) fsync foo or some file inside foo
4695 */
4696 if (last_unlink_trans >= trans->transid)
4697 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4698 }
4699 out:
4700 btrfs_end_transaction(trans);
4701 out_notrans:
4702 btrfs_btree_balance_dirty(fs_info);
4703 fscrypt_free_filename(&fname);
4704
4705 return ret;
4706 }
4707
4708 /*
4709 * Read, zero a chunk and write a block.
4710 *
4711 * @inode - inode that we're zeroing
4712 * @from - the offset to start zeroing
4713 * @len - the length to zero, 0 to zero the entire range respective to the
4714 * offset
4715 * @front - zero up to the offset instead of from the offset on
4716 *
4717 * This will find the block for the "from" offset and cow the block and zero the
4718 * part we want to zero. This is used with truncate and hole punching.
4719 */
btrfs_truncate_block(struct btrfs_inode * inode,loff_t from,loff_t len,int front)4720 int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len,
4721 int front)
4722 {
4723 struct btrfs_fs_info *fs_info = inode->root->fs_info;
4724 struct address_space *mapping = inode->vfs_inode.i_mapping;
4725 struct extent_io_tree *io_tree = &inode->io_tree;
4726 struct btrfs_ordered_extent *ordered;
4727 struct extent_state *cached_state = NULL;
4728 struct extent_changeset *data_reserved = NULL;
4729 bool only_release_metadata = false;
4730 u32 blocksize = fs_info->sectorsize;
4731 pgoff_t index = from >> PAGE_SHIFT;
4732 unsigned offset = from & (blocksize - 1);
4733 struct folio *folio;
4734 gfp_t mask = btrfs_alloc_write_mask(mapping);
4735 size_t write_bytes = blocksize;
4736 int ret = 0;
4737 u64 block_start;
4738 u64 block_end;
4739
4740 if (IS_ALIGNED(offset, blocksize) &&
4741 (!len || IS_ALIGNED(len, blocksize)))
4742 goto out;
4743
4744 block_start = round_down(from, blocksize);
4745 block_end = block_start + blocksize - 1;
4746
4747 ret = btrfs_check_data_free_space(inode, &data_reserved, block_start,
4748 blocksize, false);
4749 if (ret < 0) {
4750 if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) {
4751 /* For nocow case, no need to reserve data space */
4752 only_release_metadata = true;
4753 } else {
4754 goto out;
4755 }
4756 }
4757 ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false);
4758 if (ret < 0) {
4759 if (!only_release_metadata)
4760 btrfs_free_reserved_data_space(inode, data_reserved,
4761 block_start, blocksize);
4762 goto out;
4763 }
4764 again:
4765 folio = __filemap_get_folio(mapping, index,
4766 FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mask);
4767 if (IS_ERR(folio)) {
4768 btrfs_delalloc_release_space(inode, data_reserved, block_start,
4769 blocksize, true);
4770 btrfs_delalloc_release_extents(inode, blocksize);
4771 ret = -ENOMEM;
4772 goto out;
4773 }
4774
4775 if (!folio_test_uptodate(folio)) {
4776 ret = btrfs_read_folio(NULL, folio);
4777 folio_lock(folio);
4778 if (folio->mapping != mapping) {
4779 folio_unlock(folio);
4780 folio_put(folio);
4781 goto again;
4782 }
4783 if (!folio_test_uptodate(folio)) {
4784 ret = -EIO;
4785 goto out_unlock;
4786 }
4787 }
4788
4789 /*
4790 * We unlock the page after the io is completed and then re-lock it
4791 * above. release_folio() could have come in between that and cleared
4792 * folio private, but left the page in the mapping. Set the page mapped
4793 * here to make sure it's properly set for the subpage stuff.
4794 */
4795 ret = set_folio_extent_mapped(folio);
4796 if (ret < 0)
4797 goto out_unlock;
4798
4799 folio_wait_writeback(folio);
4800
4801 lock_extent(io_tree, block_start, block_end, &cached_state);
4802
4803 ordered = btrfs_lookup_ordered_extent(inode, block_start);
4804 if (ordered) {
4805 unlock_extent(io_tree, block_start, block_end, &cached_state);
4806 folio_unlock(folio);
4807 folio_put(folio);
4808 btrfs_start_ordered_extent(ordered);
4809 btrfs_put_ordered_extent(ordered);
4810 goto again;
4811 }
4812
4813 clear_extent_bit(&inode->io_tree, block_start, block_end,
4814 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4815 &cached_state);
4816
4817 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,
4818 &cached_state);
4819 if (ret) {
4820 unlock_extent(io_tree, block_start, block_end, &cached_state);
4821 goto out_unlock;
4822 }
4823
4824 if (offset != blocksize) {
4825 if (!len)
4826 len = blocksize - offset;
4827 if (front)
4828 folio_zero_range(folio, block_start - folio_pos(folio),
4829 offset);
4830 else
4831 folio_zero_range(folio,
4832 (block_start - folio_pos(folio)) + offset,
4833 len);
4834 }
4835 btrfs_folio_clear_checked(fs_info, folio, block_start,
4836 block_end + 1 - block_start);
4837 btrfs_folio_set_dirty(fs_info, folio, block_start,
4838 block_end + 1 - block_start);
4839 unlock_extent(io_tree, block_start, block_end, &cached_state);
4840
4841 if (only_release_metadata)
4842 set_extent_bit(&inode->io_tree, block_start, block_end,
4843 EXTENT_NORESERVE, NULL);
4844
4845 out_unlock:
4846 if (ret) {
4847 if (only_release_metadata)
4848 btrfs_delalloc_release_metadata(inode, blocksize, true);
4849 else
4850 btrfs_delalloc_release_space(inode, data_reserved,
4851 block_start, blocksize, true);
4852 }
4853 btrfs_delalloc_release_extents(inode, blocksize);
4854 folio_unlock(folio);
4855 folio_put(folio);
4856 out:
4857 if (only_release_metadata)
4858 btrfs_check_nocow_unlock(inode);
4859 extent_changeset_free(data_reserved);
4860 return ret;
4861 }
4862
maybe_insert_hole(struct btrfs_inode * inode,u64 offset,u64 len)4863 static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len)
4864 {
4865 struct btrfs_root *root = inode->root;
4866 struct btrfs_fs_info *fs_info = root->fs_info;
4867 struct btrfs_trans_handle *trans;
4868 struct btrfs_drop_extents_args drop_args = { 0 };
4869 int ret;
4870
4871 /*
4872 * If NO_HOLES is enabled, we don't need to do anything.
4873 * Later, up in the call chain, either btrfs_set_inode_last_sub_trans()
4874 * or btrfs_update_inode() will be called, which guarantee that the next
4875 * fsync will know this inode was changed and needs to be logged.
4876 */
4877 if (btrfs_fs_incompat(fs_info, NO_HOLES))
4878 return 0;
4879
4880 /*
4881 * 1 - for the one we're dropping
4882 * 1 - for the one we're adding
4883 * 1 - for updating the inode.
4884 */
4885 trans = btrfs_start_transaction(root, 3);
4886 if (IS_ERR(trans))
4887 return PTR_ERR(trans);
4888
4889 drop_args.start = offset;
4890 drop_args.end = offset + len;
4891 drop_args.drop_cache = true;
4892
4893 ret = btrfs_drop_extents(trans, root, inode, &drop_args);
4894 if (ret) {
4895 btrfs_abort_transaction(trans, ret);
4896 btrfs_end_transaction(trans);
4897 return ret;
4898 }
4899
4900 ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len);
4901 if (ret) {
4902 btrfs_abort_transaction(trans, ret);
4903 } else {
4904 btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found);
4905 btrfs_update_inode(trans, inode);
4906 }
4907 btrfs_end_transaction(trans);
4908 return ret;
4909 }
4910
4911 /*
4912 * This function puts in dummy file extents for the area we're creating a hole
4913 * for. So if we are truncating this file to a larger size we need to insert
4914 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4915 * the range between oldsize and size
4916 */
btrfs_cont_expand(struct btrfs_inode * inode,loff_t oldsize,loff_t size)4917 int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size)
4918 {
4919 struct btrfs_root *root = inode->root;
4920 struct btrfs_fs_info *fs_info = root->fs_info;
4921 struct extent_io_tree *io_tree = &inode->io_tree;
4922 struct extent_map *em = NULL;
4923 struct extent_state *cached_state = NULL;
4924 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4925 u64 block_end = ALIGN(size, fs_info->sectorsize);
4926 u64 last_byte;
4927 u64 cur_offset;
4928 u64 hole_size;
4929 int ret = 0;
4930
4931 /*
4932 * If our size started in the middle of a block we need to zero out the
4933 * rest of the block before we expand the i_size, otherwise we could
4934 * expose stale data.
4935 */
4936 ret = btrfs_truncate_block(inode, oldsize, 0, 0);
4937 if (ret)
4938 return ret;
4939
4940 if (size <= hole_start)
4941 return 0;
4942
4943 btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1,
4944 &cached_state);
4945 cur_offset = hole_start;
4946 while (1) {
4947 em = btrfs_get_extent(inode, NULL, cur_offset, block_end - cur_offset);
4948 if (IS_ERR(em)) {
4949 ret = PTR_ERR(em);
4950 em = NULL;
4951 break;
4952 }
4953 last_byte = min(extent_map_end(em), block_end);
4954 last_byte = ALIGN(last_byte, fs_info->sectorsize);
4955 hole_size = last_byte - cur_offset;
4956
4957 if (!(em->flags & EXTENT_FLAG_PREALLOC)) {
4958 struct extent_map *hole_em;
4959
4960 ret = maybe_insert_hole(inode, cur_offset, hole_size);
4961 if (ret)
4962 break;
4963
4964 ret = btrfs_inode_set_file_extent_range(inode,
4965 cur_offset, hole_size);
4966 if (ret)
4967 break;
4968
4969 hole_em = alloc_extent_map();
4970 if (!hole_em) {
4971 btrfs_drop_extent_map_range(inode, cur_offset,
4972 cur_offset + hole_size - 1,
4973 false);
4974 btrfs_set_inode_full_sync(inode);
4975 goto next;
4976 }
4977 hole_em->start = cur_offset;
4978 hole_em->len = hole_size;
4979
4980 hole_em->disk_bytenr = EXTENT_MAP_HOLE;
4981 hole_em->disk_num_bytes = 0;
4982 hole_em->ram_bytes = hole_size;
4983 hole_em->generation = btrfs_get_fs_generation(fs_info);
4984
4985 ret = btrfs_replace_extent_map_range(inode, hole_em, true);
4986 free_extent_map(hole_em);
4987 } else {
4988 ret = btrfs_inode_set_file_extent_range(inode,
4989 cur_offset, hole_size);
4990 if (ret)
4991 break;
4992 }
4993 next:
4994 free_extent_map(em);
4995 em = NULL;
4996 cur_offset = last_byte;
4997 if (cur_offset >= block_end)
4998 break;
4999 }
5000 free_extent_map(em);
5001 unlock_extent(io_tree, hole_start, block_end - 1, &cached_state);
5002 return ret;
5003 }
5004
btrfs_setsize(struct inode * inode,struct iattr * attr)5005 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5006 {
5007 struct btrfs_root *root = BTRFS_I(inode)->root;
5008 struct btrfs_trans_handle *trans;
5009 loff_t oldsize = i_size_read(inode);
5010 loff_t newsize = attr->ia_size;
5011 int mask = attr->ia_valid;
5012 int ret;
5013
5014 /*
5015 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5016 * special case where we need to update the times despite not having
5017 * these flags set. For all other operations the VFS set these flags
5018 * explicitly if it wants a timestamp update.
5019 */
5020 if (newsize != oldsize) {
5021 inode_inc_iversion(inode);
5022 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) {
5023 inode_set_mtime_to_ts(inode,
5024 inode_set_ctime_current(inode));
5025 }
5026 }
5027
5028 if (newsize > oldsize) {
5029 /*
5030 * Don't do an expanding truncate while snapshotting is ongoing.
5031 * This is to ensure the snapshot captures a fully consistent
5032 * state of this file - if the snapshot captures this expanding
5033 * truncation, it must capture all writes that happened before
5034 * this truncation.
5035 */
5036 btrfs_drew_write_lock(&root->snapshot_lock);
5037 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize);
5038 if (ret) {
5039 btrfs_drew_write_unlock(&root->snapshot_lock);
5040 return ret;
5041 }
5042
5043 trans = btrfs_start_transaction(root, 1);
5044 if (IS_ERR(trans)) {
5045 btrfs_drew_write_unlock(&root->snapshot_lock);
5046 return PTR_ERR(trans);
5047 }
5048
5049 i_size_write(inode, newsize);
5050 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
5051 pagecache_isize_extended(inode, oldsize, newsize);
5052 ret = btrfs_update_inode(trans, BTRFS_I(inode));
5053 btrfs_drew_write_unlock(&root->snapshot_lock);
5054 btrfs_end_transaction(trans);
5055 } else {
5056 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
5057
5058 if (btrfs_is_zoned(fs_info)) {
5059 ret = btrfs_wait_ordered_range(BTRFS_I(inode),
5060 ALIGN(newsize, fs_info->sectorsize),
5061 (u64)-1);
5062 if (ret)
5063 return ret;
5064 }
5065
5066 /*
5067 * We're truncating a file that used to have good data down to
5068 * zero. Make sure any new writes to the file get on disk
5069 * on close.
5070 */
5071 if (newsize == 0)
5072 set_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
5073 &BTRFS_I(inode)->runtime_flags);
5074
5075 truncate_setsize(inode, newsize);
5076
5077 inode_dio_wait(inode);
5078
5079 ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize);
5080 if (ret && inode->i_nlink) {
5081 int err;
5082
5083 /*
5084 * Truncate failed, so fix up the in-memory size. We
5085 * adjusted disk_i_size down as we removed extents, so
5086 * wait for disk_i_size to be stable and then update the
5087 * in-memory size to match.
5088 */
5089 err = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);
5090 if (err)
5091 return err;
5092 i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5093 }
5094 }
5095
5096 return ret;
5097 }
5098
btrfs_setattr(struct mnt_idmap * idmap,struct dentry * dentry,struct iattr * attr)5099 static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
5100 struct iattr *attr)
5101 {
5102 struct inode *inode = d_inode(dentry);
5103 struct btrfs_root *root = BTRFS_I(inode)->root;
5104 int err;
5105
5106 if (btrfs_root_readonly(root))
5107 return -EROFS;
5108
5109 err = setattr_prepare(idmap, dentry, attr);
5110 if (err)
5111 return err;
5112
5113 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5114 err = btrfs_setsize(inode, attr);
5115 if (err)
5116 return err;
5117 }
5118
5119 if (attr->ia_valid) {
5120 setattr_copy(idmap, inode, attr);
5121 inode_inc_iversion(inode);
5122 err = btrfs_dirty_inode(BTRFS_I(inode));
5123
5124 if (!err && attr->ia_valid & ATTR_MODE)
5125 err = posix_acl_chmod(idmap, dentry, inode->i_mode);
5126 }
5127
5128 return err;
5129 }
5130
5131 /*
5132 * While truncating the inode pages during eviction, we get the VFS
5133 * calling btrfs_invalidate_folio() against each folio of the inode. This
5134 * is slow because the calls to btrfs_invalidate_folio() result in a
5135 * huge amount of calls to lock_extent() and clear_extent_bit(),
5136 * which keep merging and splitting extent_state structures over and over,
5137 * wasting lots of time.
5138 *
5139 * Therefore if the inode is being evicted, let btrfs_invalidate_folio()
5140 * skip all those expensive operations on a per folio basis and do only
5141 * the ordered io finishing, while we release here the extent_map and
5142 * extent_state structures, without the excessive merging and splitting.
5143 */
evict_inode_truncate_pages(struct inode * inode)5144 static void evict_inode_truncate_pages(struct inode *inode)
5145 {
5146 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5147 struct rb_node *node;
5148
5149 ASSERT(inode->i_state & I_FREEING);
5150 truncate_inode_pages_final(&inode->i_data);
5151
5152 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
5153
5154 /*
5155 * Keep looping until we have no more ranges in the io tree.
5156 * We can have ongoing bios started by readahead that have
5157 * their endio callback (extent_io.c:end_bio_extent_readpage)
5158 * still in progress (unlocked the pages in the bio but did not yet
5159 * unlocked the ranges in the io tree). Therefore this means some
5160 * ranges can still be locked and eviction started because before
5161 * submitting those bios, which are executed by a separate task (work
5162 * queue kthread), inode references (inode->i_count) were not taken
5163 * (which would be dropped in the end io callback of each bio).
5164 * Therefore here we effectively end up waiting for those bios and
5165 * anyone else holding locked ranges without having bumped the inode's
5166 * reference count - if we don't do it, when they access the inode's
5167 * io_tree to unlock a range it may be too late, leading to an
5168 * use-after-free issue.
5169 */
5170 spin_lock(&io_tree->lock);
5171 while (!RB_EMPTY_ROOT(&io_tree->state)) {
5172 struct extent_state *state;
5173 struct extent_state *cached_state = NULL;
5174 u64 start;
5175 u64 end;
5176 unsigned state_flags;
5177
5178 node = rb_first(&io_tree->state);
5179 state = rb_entry(node, struct extent_state, rb_node);
5180 start = state->start;
5181 end = state->end;
5182 state_flags = state->state;
5183 spin_unlock(&io_tree->lock);
5184
5185 lock_extent(io_tree, start, end, &cached_state);
5186
5187 /*
5188 * If still has DELALLOC flag, the extent didn't reach disk,
5189 * and its reserved space won't be freed by delayed_ref.
5190 * So we need to free its reserved space here.
5191 * (Refer to comment in btrfs_invalidate_folio, case 2)
5192 *
5193 * Note, end is the bytenr of last byte, so we need + 1 here.
5194 */
5195 if (state_flags & EXTENT_DELALLOC)
5196 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start,
5197 end - start + 1, NULL);
5198
5199 clear_extent_bit(io_tree, start, end,
5200 EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING,
5201 &cached_state);
5202
5203 cond_resched();
5204 spin_lock(&io_tree->lock);
5205 }
5206 spin_unlock(&io_tree->lock);
5207 }
5208
evict_refill_and_join(struct btrfs_root * root,struct btrfs_block_rsv * rsv)5209 static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
5210 struct btrfs_block_rsv *rsv)
5211 {
5212 struct btrfs_fs_info *fs_info = root->fs_info;
5213 struct btrfs_trans_handle *trans;
5214 u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1);
5215 int ret;
5216
5217 /*
5218 * Eviction should be taking place at some place safe because of our
5219 * delayed iputs. However the normal flushing code will run delayed
5220 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
5221 *
5222 * We reserve the delayed_refs_extra here again because we can't use
5223 * btrfs_start_transaction(root, 0) for the same deadlocky reason as
5224 * above. We reserve our extra bit here because we generate a ton of
5225 * delayed refs activity by truncating.
5226 *
5227 * BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can,
5228 * if we fail to make this reservation we can re-try without the
5229 * delayed_refs_extra so we can make some forward progress.
5230 */
5231 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size + delayed_refs_extra,
5232 BTRFS_RESERVE_FLUSH_EVICT);
5233 if (ret) {
5234 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size,
5235 BTRFS_RESERVE_FLUSH_EVICT);
5236 if (ret) {
5237 btrfs_warn(fs_info,
5238 "could not allocate space for delete; will truncate on mount");
5239 return ERR_PTR(-ENOSPC);
5240 }
5241 delayed_refs_extra = 0;
5242 }
5243
5244 trans = btrfs_join_transaction(root);
5245 if (IS_ERR(trans))
5246 return trans;
5247
5248 if (delayed_refs_extra) {
5249 trans->block_rsv = &fs_info->trans_block_rsv;
5250 trans->bytes_reserved = delayed_refs_extra;
5251 btrfs_block_rsv_migrate(rsv, trans->block_rsv,
5252 delayed_refs_extra, true);
5253 }
5254 return trans;
5255 }
5256
btrfs_evict_inode(struct inode * inode)5257 void btrfs_evict_inode(struct inode *inode)
5258 {
5259 struct btrfs_fs_info *fs_info;
5260 struct btrfs_trans_handle *trans;
5261 struct btrfs_root *root = BTRFS_I(inode)->root;
5262 struct btrfs_block_rsv *rsv = NULL;
5263 int ret;
5264
5265 trace_btrfs_inode_evict(inode);
5266
5267 if (!root) {
5268 fsverity_cleanup_inode(inode);
5269 clear_inode(inode);
5270 return;
5271 }
5272
5273 fs_info = inode_to_fs_info(inode);
5274 evict_inode_truncate_pages(inode);
5275
5276 if (inode->i_nlink &&
5277 ((btrfs_root_refs(&root->root_item) != 0 &&
5278 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) ||
5279 btrfs_is_free_space_inode(BTRFS_I(inode))))
5280 goto out;
5281
5282 if (is_bad_inode(inode))
5283 goto out;
5284
5285 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
5286 goto out;
5287
5288 if (inode->i_nlink > 0) {
5289 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5290 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID);
5291 goto out;
5292 }
5293
5294 /*
5295 * This makes sure the inode item in tree is uptodate and the space for
5296 * the inode update is released.
5297 */
5298 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5299 if (ret)
5300 goto out;
5301
5302 /*
5303 * This drops any pending insert or delete operations we have for this
5304 * inode. We could have a delayed dir index deletion queued up, but
5305 * we're removing the inode completely so that'll be taken care of in
5306 * the truncate.
5307 */
5308 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
5309
5310 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5311 if (!rsv)
5312 goto out;
5313 rsv->size = btrfs_calc_metadata_size(fs_info, 1);
5314 rsv->failfast = true;
5315
5316 btrfs_i_size_write(BTRFS_I(inode), 0);
5317
5318 while (1) {
5319 struct btrfs_truncate_control control = {
5320 .inode = BTRFS_I(inode),
5321 .ino = btrfs_ino(BTRFS_I(inode)),
5322 .new_size = 0,
5323 .min_type = 0,
5324 };
5325
5326 trans = evict_refill_and_join(root, rsv);
5327 if (IS_ERR(trans))
5328 goto out;
5329
5330 trans->block_rsv = rsv;
5331
5332 ret = btrfs_truncate_inode_items(trans, root, &control);
5333 trans->block_rsv = &fs_info->trans_block_rsv;
5334 btrfs_end_transaction(trans);
5335 /*
5336 * We have not added new delayed items for our inode after we
5337 * have flushed its delayed items, so no need to throttle on
5338 * delayed items. However we have modified extent buffers.
5339 */
5340 btrfs_btree_balance_dirty_nodelay(fs_info);
5341 if (ret && ret != -ENOSPC && ret != -EAGAIN)
5342 goto out;
5343 else if (!ret)
5344 break;
5345 }
5346
5347 /*
5348 * Errors here aren't a big deal, it just means we leave orphan items in
5349 * the tree. They will be cleaned up on the next mount. If the inode
5350 * number gets reused, cleanup deletes the orphan item without doing
5351 * anything, and unlink reuses the existing orphan item.
5352 *
5353 * If it turns out that we are dropping too many of these, we might want
5354 * to add a mechanism for retrying these after a commit.
5355 */
5356 trans = evict_refill_and_join(root, rsv);
5357 if (!IS_ERR(trans)) {
5358 trans->block_rsv = rsv;
5359 btrfs_orphan_del(trans, BTRFS_I(inode));
5360 trans->block_rsv = &fs_info->trans_block_rsv;
5361 btrfs_end_transaction(trans);
5362 }
5363
5364 out:
5365 btrfs_free_block_rsv(fs_info, rsv);
5366 /*
5367 * If we didn't successfully delete, the orphan item will still be in
5368 * the tree and we'll retry on the next mount. Again, we might also want
5369 * to retry these periodically in the future.
5370 */
5371 btrfs_remove_delayed_node(BTRFS_I(inode));
5372 fsverity_cleanup_inode(inode);
5373 clear_inode(inode);
5374 }
5375
5376 /*
5377 * Return the key found in the dir entry in the location pointer, fill @type
5378 * with BTRFS_FT_*, and return 0.
5379 *
5380 * If no dir entries were found, returns -ENOENT.
5381 * If found a corrupted location in dir entry, returns -EUCLEAN.
5382 */
btrfs_inode_by_name(struct btrfs_inode * dir,struct dentry * dentry,struct btrfs_key * location,u8 * type)5383 static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry,
5384 struct btrfs_key *location, u8 *type)
5385 {
5386 struct btrfs_dir_item *di;
5387 struct btrfs_path *path;
5388 struct btrfs_root *root = dir->root;
5389 int ret = 0;
5390 struct fscrypt_name fname;
5391
5392 path = btrfs_alloc_path();
5393 if (!path)
5394 return -ENOMEM;
5395
5396 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);
5397 if (ret < 0)
5398 goto out;
5399 /*
5400 * fscrypt_setup_filename() should never return a positive value, but
5401 * gcc on sparc/parisc thinks it can, so assert that doesn't happen.
5402 */
5403 ASSERT(ret == 0);
5404
5405 /* This needs to handle no-key deletions later on */
5406
5407 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir),
5408 &fname.disk_name, 0);
5409 if (IS_ERR_OR_NULL(di)) {
5410 ret = di ? PTR_ERR(di) : -ENOENT;
5411 goto out;
5412 }
5413
5414 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5415 if (location->type != BTRFS_INODE_ITEM_KEY &&
5416 location->type != BTRFS_ROOT_ITEM_KEY) {
5417 ret = -EUCLEAN;
5418 btrfs_warn(root->fs_info,
5419 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
5420 __func__, fname.disk_name.name, btrfs_ino(dir),
5421 location->objectid, location->type, location->offset);
5422 }
5423 if (!ret)
5424 *type = btrfs_dir_ftype(path->nodes[0], di);
5425 out:
5426 fscrypt_free_filename(&fname);
5427 btrfs_free_path(path);
5428 return ret;
5429 }
5430
5431 /*
5432 * when we hit a tree root in a directory, the btrfs part of the inode
5433 * needs to be changed to reflect the root directory of the tree root. This
5434 * is kind of like crossing a mount point.
5435 */
fixup_tree_root_location(struct btrfs_fs_info * fs_info,struct btrfs_inode * dir,struct dentry * dentry,struct btrfs_key * location,struct btrfs_root ** sub_root)5436 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5437 struct btrfs_inode *dir,
5438 struct dentry *dentry,
5439 struct btrfs_key *location,
5440 struct btrfs_root **sub_root)
5441 {
5442 struct btrfs_path *path;
5443 struct btrfs_root *new_root;
5444 struct btrfs_root_ref *ref;
5445 struct extent_buffer *leaf;
5446 struct btrfs_key key;
5447 int ret;
5448 int err = 0;
5449 struct fscrypt_name fname;
5450
5451 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname);
5452 if (ret)
5453 return ret;
5454
5455 path = btrfs_alloc_path();
5456 if (!path) {
5457 err = -ENOMEM;
5458 goto out;
5459 }
5460
5461 err = -ENOENT;
5462 key.objectid = btrfs_root_id(dir->root);
5463 key.type = BTRFS_ROOT_REF_KEY;
5464 key.offset = location->objectid;
5465
5466 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5467 if (ret) {
5468 if (ret < 0)
5469 err = ret;
5470 goto out;
5471 }
5472
5473 leaf = path->nodes[0];
5474 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5475 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) ||
5476 btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len)
5477 goto out;
5478
5479 ret = memcmp_extent_buffer(leaf, fname.disk_name.name,
5480 (unsigned long)(ref + 1), fname.disk_name.len);
5481 if (ret)
5482 goto out;
5483
5484 btrfs_release_path(path);
5485
5486 new_root = btrfs_get_fs_root(fs_info, location->objectid, true);
5487 if (IS_ERR(new_root)) {
5488 err = PTR_ERR(new_root);
5489 goto out;
5490 }
5491
5492 *sub_root = new_root;
5493 location->objectid = btrfs_root_dirid(&new_root->root_item);
5494 location->type = BTRFS_INODE_ITEM_KEY;
5495 location->offset = 0;
5496 err = 0;
5497 out:
5498 btrfs_free_path(path);
5499 fscrypt_free_filename(&fname);
5500 return err;
5501 }
5502
btrfs_add_inode_to_root(struct btrfs_inode * inode,bool prealloc)5503 static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc)
5504 {
5505 struct btrfs_root *root = inode->root;
5506 struct btrfs_inode *existing;
5507 const u64 ino = btrfs_ino(inode);
5508 int ret;
5509
5510 if (inode_unhashed(&inode->vfs_inode))
5511 return 0;
5512
5513 if (prealloc) {
5514 ret = xa_reserve(&root->inodes, ino, GFP_NOFS);
5515 if (ret)
5516 return ret;
5517 }
5518
5519 existing = xa_store(&root->inodes, ino, inode, GFP_ATOMIC);
5520
5521 if (xa_is_err(existing)) {
5522 ret = xa_err(existing);
5523 ASSERT(ret != -EINVAL);
5524 ASSERT(ret != -ENOMEM);
5525 return ret;
5526 } else if (existing) {
5527 WARN_ON(!(existing->vfs_inode.i_state & (I_WILL_FREE | I_FREEING)));
5528 }
5529
5530 return 0;
5531 }
5532
btrfs_del_inode_from_root(struct btrfs_inode * inode)5533 static void btrfs_del_inode_from_root(struct btrfs_inode *inode)
5534 {
5535 struct btrfs_root *root = inode->root;
5536 struct btrfs_inode *entry;
5537 bool empty = false;
5538
5539 xa_lock(&root->inodes);
5540 entry = __xa_erase(&root->inodes, btrfs_ino(inode));
5541 if (entry == inode)
5542 empty = xa_empty(&root->inodes);
5543 xa_unlock(&root->inodes);
5544
5545 if (empty && btrfs_root_refs(&root->root_item) == 0) {
5546 xa_lock(&root->inodes);
5547 empty = xa_empty(&root->inodes);
5548 xa_unlock(&root->inodes);
5549 if (empty)
5550 btrfs_add_dead_root(root);
5551 }
5552 }
5553
5554
btrfs_init_locked_inode(struct inode * inode,void * p)5555 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5556 {
5557 struct btrfs_iget_args *args = p;
5558
5559 btrfs_set_inode_number(BTRFS_I(inode), args->ino);
5560 BTRFS_I(inode)->root = btrfs_grab_root(args->root);
5561
5562 if (args->root && args->root == args->root->fs_info->tree_root &&
5563 args->ino != BTRFS_BTREE_INODE_OBJECTID)
5564 set_bit(BTRFS_INODE_FREE_SPACE_INODE,
5565 &BTRFS_I(inode)->runtime_flags);
5566 return 0;
5567 }
5568
btrfs_find_actor(struct inode * inode,void * opaque)5569 static int btrfs_find_actor(struct inode *inode, void *opaque)
5570 {
5571 struct btrfs_iget_args *args = opaque;
5572
5573 return args->ino == btrfs_ino(BTRFS_I(inode)) &&
5574 args->root == BTRFS_I(inode)->root;
5575 }
5576
btrfs_iget_locked(u64 ino,struct btrfs_root * root)5577 static struct inode *btrfs_iget_locked(u64 ino, struct btrfs_root *root)
5578 {
5579 struct inode *inode;
5580 struct btrfs_iget_args args;
5581 unsigned long hashval = btrfs_inode_hash(ino, root);
5582
5583 args.ino = ino;
5584 args.root = root;
5585
5586 inode = iget5_locked_rcu(root->fs_info->sb, hashval, btrfs_find_actor,
5587 btrfs_init_locked_inode,
5588 (void *)&args);
5589 return inode;
5590 }
5591
5592 /*
5593 * Get an inode object given its inode number and corresponding root.
5594 * Path can be preallocated to prevent recursing back to iget through
5595 * allocator. NULL is also valid but may require an additional allocation
5596 * later.
5597 */
btrfs_iget_path(u64 ino,struct btrfs_root * root,struct btrfs_path * path)5598 struct inode *btrfs_iget_path(u64 ino, struct btrfs_root *root,
5599 struct btrfs_path *path)
5600 {
5601 struct inode *inode;
5602 int ret;
5603
5604 inode = btrfs_iget_locked(ino, root);
5605 if (!inode)
5606 return ERR_PTR(-ENOMEM);
5607
5608 if (!(inode->i_state & I_NEW))
5609 return inode;
5610
5611 ret = btrfs_read_locked_inode(inode, path);
5612 /*
5613 * ret > 0 can come from btrfs_search_slot called by
5614 * btrfs_read_locked_inode(), this means the inode item was not found.
5615 */
5616 if (ret > 0)
5617 ret = -ENOENT;
5618 if (ret < 0)
5619 goto error;
5620
5621 ret = btrfs_add_inode_to_root(BTRFS_I(inode), true);
5622 if (ret < 0)
5623 goto error;
5624
5625 unlock_new_inode(inode);
5626
5627 return inode;
5628 error:
5629 iget_failed(inode);
5630 return ERR_PTR(ret);
5631 }
5632
btrfs_iget(u64 ino,struct btrfs_root * root)5633 struct inode *btrfs_iget(u64 ino, struct btrfs_root *root)
5634 {
5635 return btrfs_iget_path(ino, root, NULL);
5636 }
5637
new_simple_dir(struct inode * dir,struct btrfs_key * key,struct btrfs_root * root)5638 static struct inode *new_simple_dir(struct inode *dir,
5639 struct btrfs_key *key,
5640 struct btrfs_root *root)
5641 {
5642 struct timespec64 ts;
5643 struct inode *inode = new_inode(dir->i_sb);
5644
5645 if (!inode)
5646 return ERR_PTR(-ENOMEM);
5647
5648 BTRFS_I(inode)->root = btrfs_grab_root(root);
5649 BTRFS_I(inode)->ref_root_id = key->objectid;
5650 set_bit(BTRFS_INODE_ROOT_STUB, &BTRFS_I(inode)->runtime_flags);
5651 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5652
5653 btrfs_set_inode_number(BTRFS_I(inode), BTRFS_EMPTY_SUBVOL_DIR_OBJECTID);
5654 /*
5655 * We only need lookup, the rest is read-only and there's no inode
5656 * associated with the dentry
5657 */
5658 inode->i_op = &simple_dir_inode_operations;
5659 inode->i_opflags &= ~IOP_XATTR;
5660 inode->i_fop = &simple_dir_operations;
5661 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5662
5663 ts = inode_set_ctime_current(inode);
5664 inode_set_mtime_to_ts(inode, ts);
5665 inode_set_atime_to_ts(inode, inode_get_atime(dir));
5666 BTRFS_I(inode)->i_otime_sec = ts.tv_sec;
5667 BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;
5668
5669 inode->i_uid = dir->i_uid;
5670 inode->i_gid = dir->i_gid;
5671
5672 return inode;
5673 }
5674
5675 static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN);
5676 static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE);
5677 static_assert(BTRFS_FT_DIR == FT_DIR);
5678 static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV);
5679 static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV);
5680 static_assert(BTRFS_FT_FIFO == FT_FIFO);
5681 static_assert(BTRFS_FT_SOCK == FT_SOCK);
5682 static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK);
5683
btrfs_inode_type(struct inode * inode)5684 static inline u8 btrfs_inode_type(struct inode *inode)
5685 {
5686 return fs_umode_to_ftype(inode->i_mode);
5687 }
5688
btrfs_lookup_dentry(struct inode * dir,struct dentry * dentry)5689 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5690 {
5691 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
5692 struct inode *inode;
5693 struct btrfs_root *root = BTRFS_I(dir)->root;
5694 struct btrfs_root *sub_root = root;
5695 struct btrfs_key location = { 0 };
5696 u8 di_type = 0;
5697 int ret = 0;
5698
5699 if (dentry->d_name.len > BTRFS_NAME_LEN)
5700 return ERR_PTR(-ENAMETOOLONG);
5701
5702 ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type);
5703 if (ret < 0)
5704 return ERR_PTR(ret);
5705
5706 if (location.type == BTRFS_INODE_ITEM_KEY) {
5707 inode = btrfs_iget(location.objectid, root);
5708 if (IS_ERR(inode))
5709 return inode;
5710
5711 /* Do extra check against inode mode with di_type */
5712 if (btrfs_inode_type(inode) != di_type) {
5713 btrfs_crit(fs_info,
5714 "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",
5715 inode->i_mode, btrfs_inode_type(inode),
5716 di_type);
5717 iput(inode);
5718 return ERR_PTR(-EUCLEAN);
5719 }
5720 return inode;
5721 }
5722
5723 ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry,
5724 &location, &sub_root);
5725 if (ret < 0) {
5726 if (ret != -ENOENT)
5727 inode = ERR_PTR(ret);
5728 else
5729 inode = new_simple_dir(dir, &location, root);
5730 } else {
5731 inode = btrfs_iget(location.objectid, sub_root);
5732 btrfs_put_root(sub_root);
5733
5734 if (IS_ERR(inode))
5735 return inode;
5736
5737 down_read(&fs_info->cleanup_work_sem);
5738 if (!sb_rdonly(inode->i_sb))
5739 ret = btrfs_orphan_cleanup(sub_root);
5740 up_read(&fs_info->cleanup_work_sem);
5741 if (ret) {
5742 iput(inode);
5743 inode = ERR_PTR(ret);
5744 }
5745 }
5746
5747 return inode;
5748 }
5749
btrfs_dentry_delete(const struct dentry * dentry)5750 static int btrfs_dentry_delete(const struct dentry *dentry)
5751 {
5752 struct btrfs_root *root;
5753 struct inode *inode = d_inode(dentry);
5754
5755 if (!inode && !IS_ROOT(dentry))
5756 inode = d_inode(dentry->d_parent);
5757
5758 if (inode) {
5759 root = BTRFS_I(inode)->root;
5760 if (btrfs_root_refs(&root->root_item) == 0)
5761 return 1;
5762
5763 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5764 return 1;
5765 }
5766 return 0;
5767 }
5768
btrfs_lookup(struct inode * dir,struct dentry * dentry,unsigned int flags)5769 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5770 unsigned int flags)
5771 {
5772 struct inode *inode = btrfs_lookup_dentry(dir, dentry);
5773
5774 if (inode == ERR_PTR(-ENOENT))
5775 inode = NULL;
5776 return d_splice_alias(inode, dentry);
5777 }
5778
5779 /*
5780 * Find the highest existing sequence number in a directory and then set the
5781 * in-memory index_cnt variable to the first free sequence number.
5782 */
btrfs_set_inode_index_count(struct btrfs_inode * inode)5783 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
5784 {
5785 struct btrfs_root *root = inode->root;
5786 struct btrfs_key key, found_key;
5787 struct btrfs_path *path;
5788 struct extent_buffer *leaf;
5789 int ret;
5790
5791 key.objectid = btrfs_ino(inode);
5792 key.type = BTRFS_DIR_INDEX_KEY;
5793 key.offset = (u64)-1;
5794
5795 path = btrfs_alloc_path();
5796 if (!path)
5797 return -ENOMEM;
5798
5799 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5800 if (ret < 0)
5801 goto out;
5802 /* FIXME: we should be able to handle this */
5803 if (ret == 0)
5804 goto out;
5805 ret = 0;
5806
5807 if (path->slots[0] == 0) {
5808 inode->index_cnt = BTRFS_DIR_START_INDEX;
5809 goto out;
5810 }
5811
5812 path->slots[0]--;
5813
5814 leaf = path->nodes[0];
5815 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5816
5817 if (found_key.objectid != btrfs_ino(inode) ||
5818 found_key.type != BTRFS_DIR_INDEX_KEY) {
5819 inode->index_cnt = BTRFS_DIR_START_INDEX;
5820 goto out;
5821 }
5822
5823 inode->index_cnt = found_key.offset + 1;
5824 out:
5825 btrfs_free_path(path);
5826 return ret;
5827 }
5828
btrfs_get_dir_last_index(struct btrfs_inode * dir,u64 * index)5829 static int btrfs_get_dir_last_index(struct btrfs_inode *dir, u64 *index)
5830 {
5831 int ret = 0;
5832
5833 btrfs_inode_lock(dir, 0);
5834 if (dir->index_cnt == (u64)-1) {
5835 ret = btrfs_inode_delayed_dir_index_count(dir);
5836 if (ret) {
5837 ret = btrfs_set_inode_index_count(dir);
5838 if (ret)
5839 goto out;
5840 }
5841 }
5842
5843 /* index_cnt is the index number of next new entry, so decrement it. */
5844 *index = dir->index_cnt - 1;
5845 out:
5846 btrfs_inode_unlock(dir, 0);
5847
5848 return ret;
5849 }
5850
5851 /*
5852 * All this infrastructure exists because dir_emit can fault, and we are holding
5853 * the tree lock when doing readdir. For now just allocate a buffer and copy
5854 * our information into that, and then dir_emit from the buffer. This is
5855 * similar to what NFS does, only we don't keep the buffer around in pagecache
5856 * because I'm afraid I'll mess that up. Long term we need to make filldir do
5857 * copy_to_user_inatomic so we don't have to worry about page faulting under the
5858 * tree lock.
5859 */
btrfs_opendir(struct inode * inode,struct file * file)5860 static int btrfs_opendir(struct inode *inode, struct file *file)
5861 {
5862 struct btrfs_file_private *private;
5863 u64 last_index;
5864 int ret;
5865
5866 ret = btrfs_get_dir_last_index(BTRFS_I(inode), &last_index);
5867 if (ret)
5868 return ret;
5869
5870 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
5871 if (!private)
5872 return -ENOMEM;
5873 private->last_index = last_index;
5874 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
5875 if (!private->filldir_buf) {
5876 kfree(private);
5877 return -ENOMEM;
5878 }
5879 file->private_data = private;
5880 return 0;
5881 }
5882
btrfs_dir_llseek(struct file * file,loff_t offset,int whence)5883 static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence)
5884 {
5885 struct btrfs_file_private *private = file->private_data;
5886 int ret;
5887
5888 ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)),
5889 &private->last_index);
5890 if (ret)
5891 return ret;
5892
5893 return generic_file_llseek(file, offset, whence);
5894 }
5895
5896 struct dir_entry {
5897 u64 ino;
5898 u64 offset;
5899 unsigned type;
5900 int name_len;
5901 };
5902
btrfs_filldir(void * addr,int entries,struct dir_context * ctx)5903 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
5904 {
5905 while (entries--) {
5906 struct dir_entry *entry = addr;
5907 char *name = (char *)(entry + 1);
5908
5909 ctx->pos = get_unaligned(&entry->offset);
5910 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),
5911 get_unaligned(&entry->ino),
5912 get_unaligned(&entry->type)))
5913 return 1;
5914 addr += sizeof(struct dir_entry) +
5915 get_unaligned(&entry->name_len);
5916 ctx->pos++;
5917 }
5918 return 0;
5919 }
5920
btrfs_real_readdir(struct file * file,struct dir_context * ctx)5921 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5922 {
5923 struct inode *inode = file_inode(file);
5924 struct btrfs_root *root = BTRFS_I(inode)->root;
5925 struct btrfs_file_private *private = file->private_data;
5926 struct btrfs_dir_item *di;
5927 struct btrfs_key key;
5928 struct btrfs_key found_key;
5929 struct btrfs_path *path;
5930 void *addr;
5931 LIST_HEAD(ins_list);
5932 LIST_HEAD(del_list);
5933 int ret;
5934 char *name_ptr;
5935 int name_len;
5936 int entries = 0;
5937 int total_len = 0;
5938 bool put = false;
5939 struct btrfs_key location;
5940
5941 if (!dir_emit_dots(file, ctx))
5942 return 0;
5943
5944 path = btrfs_alloc_path();
5945 if (!path)
5946 return -ENOMEM;
5947
5948 addr = private->filldir_buf;
5949 path->reada = READA_FORWARD;
5950
5951 put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), private->last_index,
5952 &ins_list, &del_list);
5953
5954 again:
5955 key.type = BTRFS_DIR_INDEX_KEY;
5956 key.offset = ctx->pos;
5957 key.objectid = btrfs_ino(BTRFS_I(inode));
5958
5959 btrfs_for_each_slot(root, &key, &found_key, path, ret) {
5960 struct dir_entry *entry;
5961 struct extent_buffer *leaf = path->nodes[0];
5962 u8 ftype;
5963
5964 if (found_key.objectid != key.objectid)
5965 break;
5966 if (found_key.type != BTRFS_DIR_INDEX_KEY)
5967 break;
5968 if (found_key.offset < ctx->pos)
5969 continue;
5970 if (found_key.offset > private->last_index)
5971 break;
5972 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5973 continue;
5974 di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
5975 name_len = btrfs_dir_name_len(leaf, di);
5976 if ((total_len + sizeof(struct dir_entry) + name_len) >=
5977 PAGE_SIZE) {
5978 btrfs_release_path(path);
5979 ret = btrfs_filldir(private->filldir_buf, entries, ctx);
5980 if (ret)
5981 goto nopos;
5982 addr = private->filldir_buf;
5983 entries = 0;
5984 total_len = 0;
5985 goto again;
5986 }
5987
5988 ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di));
5989 entry = addr;
5990 name_ptr = (char *)(entry + 1);
5991 read_extent_buffer(leaf, name_ptr,
5992 (unsigned long)(di + 1), name_len);
5993 put_unaligned(name_len, &entry->name_len);
5994 put_unaligned(fs_ftype_to_dtype(ftype), &entry->type);
5995 btrfs_dir_item_key_to_cpu(leaf, di, &location);
5996 put_unaligned(location.objectid, &entry->ino);
5997 put_unaligned(found_key.offset, &entry->offset);
5998 entries++;
5999 addr += sizeof(struct dir_entry) + name_len;
6000 total_len += sizeof(struct dir_entry) + name_len;
6001 }
6002 /* Catch error encountered during iteration */
6003 if (ret < 0)
6004 goto err;
6005
6006 btrfs_release_path(path);
6007
6008 ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6009 if (ret)
6010 goto nopos;
6011
6012 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
6013 if (ret)
6014 goto nopos;
6015
6016 /*
6017 * Stop new entries from being returned after we return the last
6018 * entry.
6019 *
6020 * New directory entries are assigned a strictly increasing
6021 * offset. This means that new entries created during readdir
6022 * are *guaranteed* to be seen in the future by that readdir.
6023 * This has broken buggy programs which operate on names as
6024 * they're returned by readdir. Until we re-use freed offsets
6025 * we have this hack to stop new entries from being returned
6026 * under the assumption that they'll never reach this huge
6027 * offset.
6028 *
6029 * This is being careful not to overflow 32bit loff_t unless the
6030 * last entry requires it because doing so has broken 32bit apps
6031 * in the past.
6032 */
6033 if (ctx->pos >= INT_MAX)
6034 ctx->pos = LLONG_MAX;
6035 else
6036 ctx->pos = INT_MAX;
6037 nopos:
6038 ret = 0;
6039 err:
6040 if (put)
6041 btrfs_readdir_put_delayed_items(BTRFS_I(inode), &ins_list, &del_list);
6042 btrfs_free_path(path);
6043 return ret;
6044 }
6045
6046 /*
6047 * This is somewhat expensive, updating the tree every time the
6048 * inode changes. But, it is most likely to find the inode in cache.
6049 * FIXME, needs more benchmarking...there are no reasons other than performance
6050 * to keep or drop this code.
6051 */
btrfs_dirty_inode(struct btrfs_inode * inode)6052 static int btrfs_dirty_inode(struct btrfs_inode *inode)
6053 {
6054 struct btrfs_root *root = inode->root;
6055 struct btrfs_fs_info *fs_info = root->fs_info;
6056 struct btrfs_trans_handle *trans;
6057 int ret;
6058
6059 if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags))
6060 return 0;
6061
6062 trans = btrfs_join_transaction(root);
6063 if (IS_ERR(trans))
6064 return PTR_ERR(trans);
6065
6066 ret = btrfs_update_inode(trans, inode);
6067 if (ret == -ENOSPC || ret == -EDQUOT) {
6068 /* whoops, lets try again with the full transaction */
6069 btrfs_end_transaction(trans);
6070 trans = btrfs_start_transaction(root, 1);
6071 if (IS_ERR(trans))
6072 return PTR_ERR(trans);
6073
6074 ret = btrfs_update_inode(trans, inode);
6075 }
6076 btrfs_end_transaction(trans);
6077 if (inode->delayed_node)
6078 btrfs_balance_delayed_items(fs_info);
6079
6080 return ret;
6081 }
6082
6083 /*
6084 * This is a copy of file_update_time. We need this so we can return error on
6085 * ENOSPC for updating the inode in the case of file write and mmap writes.
6086 */
btrfs_update_time(struct inode * inode,int flags)6087 static int btrfs_update_time(struct inode *inode, int flags)
6088 {
6089 struct btrfs_root *root = BTRFS_I(inode)->root;
6090 bool dirty;
6091
6092 if (btrfs_root_readonly(root))
6093 return -EROFS;
6094
6095 dirty = inode_update_timestamps(inode, flags);
6096 return dirty ? btrfs_dirty_inode(BTRFS_I(inode)) : 0;
6097 }
6098
6099 /*
6100 * helper to find a free sequence number in a given directory. This current
6101 * code is very simple, later versions will do smarter things in the btree
6102 */
btrfs_set_inode_index(struct btrfs_inode * dir,u64 * index)6103 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6104 {
6105 int ret = 0;
6106
6107 if (dir->index_cnt == (u64)-1) {
6108 ret = btrfs_inode_delayed_dir_index_count(dir);
6109 if (ret) {
6110 ret = btrfs_set_inode_index_count(dir);
6111 if (ret)
6112 return ret;
6113 }
6114 }
6115
6116 *index = dir->index_cnt;
6117 dir->index_cnt++;
6118
6119 return ret;
6120 }
6121
btrfs_insert_inode_locked(struct inode * inode)6122 static int btrfs_insert_inode_locked(struct inode *inode)
6123 {
6124 struct btrfs_iget_args args;
6125
6126 args.ino = btrfs_ino(BTRFS_I(inode));
6127 args.root = BTRFS_I(inode)->root;
6128
6129 return insert_inode_locked4(inode,
6130 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6131 btrfs_find_actor, &args);
6132 }
6133
btrfs_new_inode_prepare(struct btrfs_new_inode_args * args,unsigned int * trans_num_items)6134 int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args,
6135 unsigned int *trans_num_items)
6136 {
6137 struct inode *dir = args->dir;
6138 struct inode *inode = args->inode;
6139 int ret;
6140
6141 if (!args->orphan) {
6142 ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0,
6143 &args->fname);
6144 if (ret)
6145 return ret;
6146 }
6147
6148 ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl);
6149 if (ret) {
6150 fscrypt_free_filename(&args->fname);
6151 return ret;
6152 }
6153
6154 /* 1 to add inode item */
6155 *trans_num_items = 1;
6156 /* 1 to add compression property */
6157 if (BTRFS_I(dir)->prop_compress)
6158 (*trans_num_items)++;
6159 /* 1 to add default ACL xattr */
6160 if (args->default_acl)
6161 (*trans_num_items)++;
6162 /* 1 to add access ACL xattr */
6163 if (args->acl)
6164 (*trans_num_items)++;
6165 #ifdef CONFIG_SECURITY
6166 /* 1 to add LSM xattr */
6167 if (dir->i_security)
6168 (*trans_num_items)++;
6169 #endif
6170 if (args->orphan) {
6171 /* 1 to add orphan item */
6172 (*trans_num_items)++;
6173 } else {
6174 /*
6175 * 1 to add dir item
6176 * 1 to add dir index
6177 * 1 to update parent inode item
6178 *
6179 * No need for 1 unit for the inode ref item because it is
6180 * inserted in a batch together with the inode item at
6181 * btrfs_create_new_inode().
6182 */
6183 *trans_num_items += 3;
6184 }
6185 return 0;
6186 }
6187
btrfs_new_inode_args_destroy(struct btrfs_new_inode_args * args)6188 void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args)
6189 {
6190 posix_acl_release(args->acl);
6191 posix_acl_release(args->default_acl);
6192 fscrypt_free_filename(&args->fname);
6193 }
6194
6195 /*
6196 * Inherit flags from the parent inode.
6197 *
6198 * Currently only the compression flags and the cow flags are inherited.
6199 */
btrfs_inherit_iflags(struct btrfs_inode * inode,struct btrfs_inode * dir)6200 static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir)
6201 {
6202 unsigned int flags;
6203
6204 flags = dir->flags;
6205
6206 if (flags & BTRFS_INODE_NOCOMPRESS) {
6207 inode->flags &= ~BTRFS_INODE_COMPRESS;
6208 inode->flags |= BTRFS_INODE_NOCOMPRESS;
6209 } else if (flags & BTRFS_INODE_COMPRESS) {
6210 inode->flags &= ~BTRFS_INODE_NOCOMPRESS;
6211 inode->flags |= BTRFS_INODE_COMPRESS;
6212 }
6213
6214 if (flags & BTRFS_INODE_NODATACOW) {
6215 inode->flags |= BTRFS_INODE_NODATACOW;
6216 if (S_ISREG(inode->vfs_inode.i_mode))
6217 inode->flags |= BTRFS_INODE_NODATASUM;
6218 }
6219
6220 btrfs_sync_inode_flags_to_i_flags(&inode->vfs_inode);
6221 }
6222
btrfs_create_new_inode(struct btrfs_trans_handle * trans,struct btrfs_new_inode_args * args)6223 int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
6224 struct btrfs_new_inode_args *args)
6225 {
6226 struct timespec64 ts;
6227 struct inode *dir = args->dir;
6228 struct inode *inode = args->inode;
6229 const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name;
6230 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6231 struct btrfs_root *root;
6232 struct btrfs_inode_item *inode_item;
6233 struct btrfs_path *path;
6234 u64 objectid;
6235 struct btrfs_inode_ref *ref;
6236 struct btrfs_key key[2];
6237 u32 sizes[2];
6238 struct btrfs_item_batch batch;
6239 unsigned long ptr;
6240 int ret;
6241 bool xa_reserved = false;
6242
6243 path = btrfs_alloc_path();
6244 if (!path)
6245 return -ENOMEM;
6246
6247 if (!args->subvol)
6248 BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root);
6249 root = BTRFS_I(inode)->root;
6250
6251 ret = btrfs_init_file_extent_tree(BTRFS_I(inode));
6252 if (ret)
6253 goto out;
6254
6255 ret = btrfs_get_free_objectid(root, &objectid);
6256 if (ret)
6257 goto out;
6258 btrfs_set_inode_number(BTRFS_I(inode), objectid);
6259
6260 ret = xa_reserve(&root->inodes, objectid, GFP_NOFS);
6261 if (ret)
6262 goto out;
6263 xa_reserved = true;
6264
6265 if (args->orphan) {
6266 /*
6267 * O_TMPFILE, set link count to 0, so that after this point, we
6268 * fill in an inode item with the correct link count.
6269 */
6270 set_nlink(inode, 0);
6271 } else {
6272 trace_btrfs_inode_request(dir);
6273
6274 ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index);
6275 if (ret)
6276 goto out;
6277 }
6278
6279 if (S_ISDIR(inode->i_mode))
6280 BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX;
6281
6282 BTRFS_I(inode)->generation = trans->transid;
6283 inode->i_generation = BTRFS_I(inode)->generation;
6284
6285 /*
6286 * We don't have any capability xattrs set here yet, shortcut any
6287 * queries for the xattrs here. If we add them later via the inode
6288 * security init path or any other path this flag will be cleared.
6289 */
6290 set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags);
6291
6292 /*
6293 * Subvolumes don't inherit flags from their parent directory.
6294 * Originally this was probably by accident, but we probably can't
6295 * change it now without compatibility issues.
6296 */
6297 if (!args->subvol)
6298 btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir));
6299
6300 if (S_ISREG(inode->i_mode)) {
6301 if (btrfs_test_opt(fs_info, NODATASUM))
6302 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6303 if (btrfs_test_opt(fs_info, NODATACOW))
6304 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6305 BTRFS_INODE_NODATASUM;
6306 }
6307
6308 ret = btrfs_insert_inode_locked(inode);
6309 if (ret < 0) {
6310 if (!args->orphan)
6311 BTRFS_I(dir)->index_cnt--;
6312 goto out;
6313 }
6314
6315 /*
6316 * We could have gotten an inode number from somebody who was fsynced
6317 * and then removed in this same transaction, so let's just set full
6318 * sync since it will be a full sync anyway and this will blow away the
6319 * old info in the log.
6320 */
6321 btrfs_set_inode_full_sync(BTRFS_I(inode));
6322
6323 key[0].objectid = objectid;
6324 key[0].type = BTRFS_INODE_ITEM_KEY;
6325 key[0].offset = 0;
6326
6327 sizes[0] = sizeof(struct btrfs_inode_item);
6328
6329 if (!args->orphan) {
6330 /*
6331 * Start new inodes with an inode_ref. This is slightly more
6332 * efficient for small numbers of hard links since they will
6333 * be packed into one item. Extended refs will kick in if we
6334 * add more hard links than can fit in the ref item.
6335 */
6336 key[1].objectid = objectid;
6337 key[1].type = BTRFS_INODE_REF_KEY;
6338 if (args->subvol) {
6339 key[1].offset = objectid;
6340 sizes[1] = 2 + sizeof(*ref);
6341 } else {
6342 key[1].offset = btrfs_ino(BTRFS_I(dir));
6343 sizes[1] = name->len + sizeof(*ref);
6344 }
6345 }
6346
6347 batch.keys = &key[0];
6348 batch.data_sizes = &sizes[0];
6349 batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]);
6350 batch.nr = args->orphan ? 1 : 2;
6351 ret = btrfs_insert_empty_items(trans, root, path, &batch);
6352 if (ret != 0) {
6353 btrfs_abort_transaction(trans, ret);
6354 goto discard;
6355 }
6356
6357 ts = simple_inode_init_ts(inode);
6358 BTRFS_I(inode)->i_otime_sec = ts.tv_sec;
6359 BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;
6360
6361 /*
6362 * We're going to fill the inode item now, so at this point the inode
6363 * must be fully initialized.
6364 */
6365
6366 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6367 struct btrfs_inode_item);
6368 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6369 sizeof(*inode_item));
6370 fill_inode_item(trans, path->nodes[0], inode_item, inode);
6371
6372 if (!args->orphan) {
6373 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6374 struct btrfs_inode_ref);
6375 ptr = (unsigned long)(ref + 1);
6376 if (args->subvol) {
6377 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2);
6378 btrfs_set_inode_ref_index(path->nodes[0], ref, 0);
6379 write_extent_buffer(path->nodes[0], "..", ptr, 2);
6380 } else {
6381 btrfs_set_inode_ref_name_len(path->nodes[0], ref,
6382 name->len);
6383 btrfs_set_inode_ref_index(path->nodes[0], ref,
6384 BTRFS_I(inode)->dir_index);
6385 write_extent_buffer(path->nodes[0], name->name, ptr,
6386 name->len);
6387 }
6388 }
6389
6390 btrfs_mark_buffer_dirty(trans, path->nodes[0]);
6391 /*
6392 * We don't need the path anymore, plus inheriting properties, adding
6393 * ACLs, security xattrs, orphan item or adding the link, will result in
6394 * allocating yet another path. So just free our path.
6395 */
6396 btrfs_free_path(path);
6397 path = NULL;
6398
6399 if (args->subvol) {
6400 struct inode *parent;
6401
6402 /*
6403 * Subvolumes inherit properties from their parent subvolume,
6404 * not the directory they were created in.
6405 */
6406 parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root);
6407 if (IS_ERR(parent)) {
6408 ret = PTR_ERR(parent);
6409 } else {
6410 ret = btrfs_inode_inherit_props(trans, inode, parent);
6411 iput(parent);
6412 }
6413 } else {
6414 ret = btrfs_inode_inherit_props(trans, inode, dir);
6415 }
6416 if (ret) {
6417 btrfs_err(fs_info,
6418 "error inheriting props for ino %llu (root %llu): %d",
6419 btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ret);
6420 }
6421
6422 /*
6423 * Subvolumes don't inherit ACLs or get passed to the LSM. This is
6424 * probably a bug.
6425 */
6426 if (!args->subvol) {
6427 ret = btrfs_init_inode_security(trans, args);
6428 if (ret) {
6429 btrfs_abort_transaction(trans, ret);
6430 goto discard;
6431 }
6432 }
6433
6434 ret = btrfs_add_inode_to_root(BTRFS_I(inode), false);
6435 if (WARN_ON(ret)) {
6436 /* Shouldn't happen, we used xa_reserve() before. */
6437 btrfs_abort_transaction(trans, ret);
6438 goto discard;
6439 }
6440
6441 trace_btrfs_inode_new(inode);
6442 btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
6443
6444 btrfs_update_root_times(trans, root);
6445
6446 if (args->orphan) {
6447 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
6448 } else {
6449 ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
6450 0, BTRFS_I(inode)->dir_index);
6451 }
6452 if (ret) {
6453 btrfs_abort_transaction(trans, ret);
6454 goto discard;
6455 }
6456
6457 return 0;
6458
6459 discard:
6460 /*
6461 * discard_new_inode() calls iput(), but the caller owns the reference
6462 * to the inode.
6463 */
6464 ihold(inode);
6465 discard_new_inode(inode);
6466 out:
6467 if (xa_reserved)
6468 xa_release(&root->inodes, objectid);
6469
6470 btrfs_free_path(path);
6471 return ret;
6472 }
6473
6474 /*
6475 * utility function to add 'inode' into 'parent_inode' with
6476 * a give name and a given sequence number.
6477 * if 'add_backref' is true, also insert a backref from the
6478 * inode to the parent directory.
6479 */
btrfs_add_link(struct btrfs_trans_handle * trans,struct btrfs_inode * parent_inode,struct btrfs_inode * inode,const struct fscrypt_str * name,int add_backref,u64 index)6480 int btrfs_add_link(struct btrfs_trans_handle *trans,
6481 struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6482 const struct fscrypt_str *name, int add_backref, u64 index)
6483 {
6484 int ret = 0;
6485 struct btrfs_key key;
6486 struct btrfs_root *root = parent_inode->root;
6487 u64 ino = btrfs_ino(inode);
6488 u64 parent_ino = btrfs_ino(parent_inode);
6489
6490 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6491 memcpy(&key, &inode->root->root_key, sizeof(key));
6492 } else {
6493 key.objectid = ino;
6494 key.type = BTRFS_INODE_ITEM_KEY;
6495 key.offset = 0;
6496 }
6497
6498 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6499 ret = btrfs_add_root_ref(trans, key.objectid,
6500 btrfs_root_id(root), parent_ino,
6501 index, name);
6502 } else if (add_backref) {
6503 ret = btrfs_insert_inode_ref(trans, root, name,
6504 ino, parent_ino, index);
6505 }
6506
6507 /* Nothing to clean up yet */
6508 if (ret)
6509 return ret;
6510
6511 ret = btrfs_insert_dir_item(trans, name, parent_inode, &key,
6512 btrfs_inode_type(&inode->vfs_inode), index);
6513 if (ret == -EEXIST || ret == -EOVERFLOW)
6514 goto fail_dir_item;
6515 else if (ret) {
6516 btrfs_abort_transaction(trans, ret);
6517 return ret;
6518 }
6519
6520 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6521 name->len * 2);
6522 inode_inc_iversion(&parent_inode->vfs_inode);
6523 /*
6524 * If we are replaying a log tree, we do not want to update the mtime
6525 * and ctime of the parent directory with the current time, since the
6526 * log replay procedure is responsible for setting them to their correct
6527 * values (the ones it had when the fsync was done).
6528 */
6529 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags))
6530 inode_set_mtime_to_ts(&parent_inode->vfs_inode,
6531 inode_set_ctime_current(&parent_inode->vfs_inode));
6532
6533 ret = btrfs_update_inode(trans, parent_inode);
6534 if (ret)
6535 btrfs_abort_transaction(trans, ret);
6536 return ret;
6537
6538 fail_dir_item:
6539 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6540 u64 local_index;
6541 int err;
6542 err = btrfs_del_root_ref(trans, key.objectid,
6543 btrfs_root_id(root), parent_ino,
6544 &local_index, name);
6545 if (err)
6546 btrfs_abort_transaction(trans, err);
6547 } else if (add_backref) {
6548 u64 local_index;
6549 int err;
6550
6551 err = btrfs_del_inode_ref(trans, root, name, ino, parent_ino,
6552 &local_index);
6553 if (err)
6554 btrfs_abort_transaction(trans, err);
6555 }
6556
6557 /* Return the original error code */
6558 return ret;
6559 }
6560
btrfs_create_common(struct inode * dir,struct dentry * dentry,struct inode * inode)6561 static int btrfs_create_common(struct inode *dir, struct dentry *dentry,
6562 struct inode *inode)
6563 {
6564 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6565 struct btrfs_root *root = BTRFS_I(dir)->root;
6566 struct btrfs_new_inode_args new_inode_args = {
6567 .dir = dir,
6568 .dentry = dentry,
6569 .inode = inode,
6570 };
6571 unsigned int trans_num_items;
6572 struct btrfs_trans_handle *trans;
6573 int err;
6574
6575 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
6576 if (err)
6577 goto out_inode;
6578
6579 trans = btrfs_start_transaction(root, trans_num_items);
6580 if (IS_ERR(trans)) {
6581 err = PTR_ERR(trans);
6582 goto out_new_inode_args;
6583 }
6584
6585 err = btrfs_create_new_inode(trans, &new_inode_args);
6586 if (!err)
6587 d_instantiate_new(dentry, inode);
6588
6589 btrfs_end_transaction(trans);
6590 btrfs_btree_balance_dirty(fs_info);
6591 out_new_inode_args:
6592 btrfs_new_inode_args_destroy(&new_inode_args);
6593 out_inode:
6594 if (err)
6595 iput(inode);
6596 return err;
6597 }
6598
btrfs_mknod(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode,dev_t rdev)6599 static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
6600 struct dentry *dentry, umode_t mode, dev_t rdev)
6601 {
6602 struct inode *inode;
6603
6604 inode = new_inode(dir->i_sb);
6605 if (!inode)
6606 return -ENOMEM;
6607 inode_init_owner(idmap, inode, dir, mode);
6608 inode->i_op = &btrfs_special_inode_operations;
6609 init_special_inode(inode, inode->i_mode, rdev);
6610 return btrfs_create_common(dir, dentry, inode);
6611 }
6612
btrfs_create(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode,bool excl)6613 static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir,
6614 struct dentry *dentry, umode_t mode, bool excl)
6615 {
6616 struct inode *inode;
6617
6618 inode = new_inode(dir->i_sb);
6619 if (!inode)
6620 return -ENOMEM;
6621 inode_init_owner(idmap, inode, dir, mode);
6622 inode->i_fop = &btrfs_file_operations;
6623 inode->i_op = &btrfs_file_inode_operations;
6624 inode->i_mapping->a_ops = &btrfs_aops;
6625 return btrfs_create_common(dir, dentry, inode);
6626 }
6627
btrfs_link(struct dentry * old_dentry,struct inode * dir,struct dentry * dentry)6628 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6629 struct dentry *dentry)
6630 {
6631 struct btrfs_trans_handle *trans = NULL;
6632 struct btrfs_root *root = BTRFS_I(dir)->root;
6633 struct inode *inode = d_inode(old_dentry);
6634 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
6635 struct fscrypt_name fname;
6636 u64 index;
6637 int err;
6638 int drop_inode = 0;
6639
6640 /* do not allow sys_link's with other subvols of the same device */
6641 if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root))
6642 return -EXDEV;
6643
6644 if (inode->i_nlink >= BTRFS_LINK_MAX)
6645 return -EMLINK;
6646
6647 err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname);
6648 if (err)
6649 goto fail;
6650
6651 err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6652 if (err)
6653 goto fail;
6654
6655 /*
6656 * 2 items for inode and inode ref
6657 * 2 items for dir items
6658 * 1 item for parent inode
6659 * 1 item for orphan item deletion if O_TMPFILE
6660 */
6661 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6);
6662 if (IS_ERR(trans)) {
6663 err = PTR_ERR(trans);
6664 trans = NULL;
6665 goto fail;
6666 }
6667
6668 /* There are several dir indexes for this inode, clear the cache. */
6669 BTRFS_I(inode)->dir_index = 0ULL;
6670 inc_nlink(inode);
6671 inode_inc_iversion(inode);
6672 inode_set_ctime_current(inode);
6673 ihold(inode);
6674 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6675
6676 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6677 &fname.disk_name, 1, index);
6678
6679 if (err) {
6680 drop_inode = 1;
6681 } else {
6682 struct dentry *parent = dentry->d_parent;
6683
6684 err = btrfs_update_inode(trans, BTRFS_I(inode));
6685 if (err)
6686 goto fail;
6687 if (inode->i_nlink == 1) {
6688 /*
6689 * If new hard link count is 1, it's a file created
6690 * with open(2) O_TMPFILE flag.
6691 */
6692 err = btrfs_orphan_del(trans, BTRFS_I(inode));
6693 if (err)
6694 goto fail;
6695 }
6696 d_instantiate(dentry, inode);
6697 btrfs_log_new_name(trans, old_dentry, NULL, 0, parent);
6698 }
6699
6700 fail:
6701 fscrypt_free_filename(&fname);
6702 if (trans)
6703 btrfs_end_transaction(trans);
6704 if (drop_inode) {
6705 inode_dec_link_count(inode);
6706 iput(inode);
6707 }
6708 btrfs_btree_balance_dirty(fs_info);
6709 return err;
6710 }
6711
btrfs_mkdir(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode)6712 static int btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
6713 struct dentry *dentry, umode_t mode)
6714 {
6715 struct inode *inode;
6716
6717 inode = new_inode(dir->i_sb);
6718 if (!inode)
6719 return -ENOMEM;
6720 inode_init_owner(idmap, inode, dir, S_IFDIR | mode);
6721 inode->i_op = &btrfs_dir_inode_operations;
6722 inode->i_fop = &btrfs_dir_file_operations;
6723 return btrfs_create_common(dir, dentry, inode);
6724 }
6725
uncompress_inline(struct btrfs_path * path,struct folio * folio,struct btrfs_file_extent_item * item)6726 static noinline int uncompress_inline(struct btrfs_path *path,
6727 struct folio *folio,
6728 struct btrfs_file_extent_item *item)
6729 {
6730 int ret;
6731 struct extent_buffer *leaf = path->nodes[0];
6732 char *tmp;
6733 size_t max_size;
6734 unsigned long inline_size;
6735 unsigned long ptr;
6736 int compress_type;
6737
6738 compress_type = btrfs_file_extent_compression(leaf, item);
6739 max_size = btrfs_file_extent_ram_bytes(leaf, item);
6740 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
6741 tmp = kmalloc(inline_size, GFP_NOFS);
6742 if (!tmp)
6743 return -ENOMEM;
6744 ptr = btrfs_file_extent_inline_start(item);
6745
6746 read_extent_buffer(leaf, tmp, ptr, inline_size);
6747
6748 max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6749 ret = btrfs_decompress(compress_type, tmp, folio, 0, inline_size,
6750 max_size);
6751
6752 /*
6753 * decompression code contains a memset to fill in any space between the end
6754 * of the uncompressed data and the end of max_size in case the decompressed
6755 * data ends up shorter than ram_bytes. That doesn't cover the hole between
6756 * the end of an inline extent and the beginning of the next block, so we
6757 * cover that region here.
6758 */
6759
6760 if (max_size < PAGE_SIZE)
6761 folio_zero_range(folio, max_size, PAGE_SIZE - max_size);
6762 kfree(tmp);
6763 return ret;
6764 }
6765
read_inline_extent(struct btrfs_inode * inode,struct btrfs_path * path,struct folio * folio)6766 static int read_inline_extent(struct btrfs_inode *inode, struct btrfs_path *path,
6767 struct folio *folio)
6768 {
6769 struct btrfs_file_extent_item *fi;
6770 void *kaddr;
6771 size_t copy_size;
6772
6773 if (!folio || folio_test_uptodate(folio))
6774 return 0;
6775
6776 ASSERT(folio_pos(folio) == 0);
6777
6778 fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
6779 struct btrfs_file_extent_item);
6780 if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE)
6781 return uncompress_inline(path, folio, fi);
6782
6783 copy_size = min_t(u64, PAGE_SIZE,
6784 btrfs_file_extent_ram_bytes(path->nodes[0], fi));
6785 kaddr = kmap_local_folio(folio, 0);
6786 read_extent_buffer(path->nodes[0], kaddr,
6787 btrfs_file_extent_inline_start(fi), copy_size);
6788 kunmap_local(kaddr);
6789 if (copy_size < PAGE_SIZE)
6790 folio_zero_range(folio, copy_size, PAGE_SIZE - copy_size);
6791 return 0;
6792 }
6793
6794 /*
6795 * Lookup the first extent overlapping a range in a file.
6796 *
6797 * @inode: file to search in
6798 * @page: page to read extent data into if the extent is inline
6799 * @start: file offset
6800 * @len: length of range starting at @start
6801 *
6802 * Return the first &struct extent_map which overlaps the given range, reading
6803 * it from the B-tree and caching it if necessary. Note that there may be more
6804 * extents which overlap the given range after the returned extent_map.
6805 *
6806 * If @page is not NULL and the extent is inline, this also reads the extent
6807 * data directly into the page and marks the extent up to date in the io_tree.
6808 *
6809 * Return: ERR_PTR on error, non-NULL extent_map on success.
6810 */
btrfs_get_extent(struct btrfs_inode * inode,struct folio * folio,u64 start,u64 len)6811 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
6812 struct folio *folio, u64 start, u64 len)
6813 {
6814 struct btrfs_fs_info *fs_info = inode->root->fs_info;
6815 int ret = 0;
6816 u64 extent_start = 0;
6817 u64 extent_end = 0;
6818 u64 objectid = btrfs_ino(inode);
6819 int extent_type = -1;
6820 struct btrfs_path *path = NULL;
6821 struct btrfs_root *root = inode->root;
6822 struct btrfs_file_extent_item *item;
6823 struct extent_buffer *leaf;
6824 struct btrfs_key found_key;
6825 struct extent_map *em = NULL;
6826 struct extent_map_tree *em_tree = &inode->extent_tree;
6827
6828 read_lock(&em_tree->lock);
6829 em = lookup_extent_mapping(em_tree, start, len);
6830 read_unlock(&em_tree->lock);
6831
6832 if (em) {
6833 if (em->start > start || em->start + em->len <= start)
6834 free_extent_map(em);
6835 else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio)
6836 free_extent_map(em);
6837 else
6838 goto out;
6839 }
6840 em = alloc_extent_map();
6841 if (!em) {
6842 ret = -ENOMEM;
6843 goto out;
6844 }
6845 em->start = EXTENT_MAP_HOLE;
6846 em->disk_bytenr = EXTENT_MAP_HOLE;
6847 em->len = (u64)-1;
6848
6849 path = btrfs_alloc_path();
6850 if (!path) {
6851 ret = -ENOMEM;
6852 goto out;
6853 }
6854
6855 /* Chances are we'll be called again, so go ahead and do readahead */
6856 path->reada = READA_FORWARD;
6857
6858 /*
6859 * The same explanation in load_free_space_cache applies here as well,
6860 * we only read when we're loading the free space cache, and at that
6861 * point the commit_root has everything we need.
6862 */
6863 if (btrfs_is_free_space_inode(inode)) {
6864 path->search_commit_root = 1;
6865 path->skip_locking = 1;
6866 }
6867
6868 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
6869 if (ret < 0) {
6870 goto out;
6871 } else if (ret > 0) {
6872 if (path->slots[0] == 0)
6873 goto not_found;
6874 path->slots[0]--;
6875 ret = 0;
6876 }
6877
6878 leaf = path->nodes[0];
6879 item = btrfs_item_ptr(leaf, path->slots[0],
6880 struct btrfs_file_extent_item);
6881 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6882 if (found_key.objectid != objectid ||
6883 found_key.type != BTRFS_EXTENT_DATA_KEY) {
6884 /*
6885 * If we backup past the first extent we want to move forward
6886 * and see if there is an extent in front of us, otherwise we'll
6887 * say there is a hole for our whole search range which can
6888 * cause problems.
6889 */
6890 extent_end = start;
6891 goto next;
6892 }
6893
6894 extent_type = btrfs_file_extent_type(leaf, item);
6895 extent_start = found_key.offset;
6896 extent_end = btrfs_file_extent_end(path);
6897 if (extent_type == BTRFS_FILE_EXTENT_REG ||
6898 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
6899 /* Only regular file could have regular/prealloc extent */
6900 if (!S_ISREG(inode->vfs_inode.i_mode)) {
6901 ret = -EUCLEAN;
6902 btrfs_crit(fs_info,
6903 "regular/prealloc extent found for non-regular inode %llu",
6904 btrfs_ino(inode));
6905 goto out;
6906 }
6907 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
6908 extent_start);
6909 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
6910 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
6911 path->slots[0],
6912 extent_start);
6913 }
6914 next:
6915 if (start >= extent_end) {
6916 path->slots[0]++;
6917 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
6918 ret = btrfs_next_leaf(root, path);
6919 if (ret < 0)
6920 goto out;
6921 else if (ret > 0)
6922 goto not_found;
6923
6924 leaf = path->nodes[0];
6925 }
6926 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6927 if (found_key.objectid != objectid ||
6928 found_key.type != BTRFS_EXTENT_DATA_KEY)
6929 goto not_found;
6930 if (start + len <= found_key.offset)
6931 goto not_found;
6932 if (start > found_key.offset)
6933 goto next;
6934
6935 /* New extent overlaps with existing one */
6936 em->start = start;
6937 em->len = found_key.offset - start;
6938 em->disk_bytenr = EXTENT_MAP_HOLE;
6939 goto insert;
6940 }
6941
6942 btrfs_extent_item_to_extent_map(inode, path, item, em);
6943
6944 if (extent_type == BTRFS_FILE_EXTENT_REG ||
6945 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
6946 goto insert;
6947 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
6948 /*
6949 * Inline extent can only exist at file offset 0. This is
6950 * ensured by tree-checker and inline extent creation path.
6951 * Thus all members representing file offsets should be zero.
6952 */
6953 ASSERT(extent_start == 0);
6954 ASSERT(em->start == 0);
6955
6956 /*
6957 * btrfs_extent_item_to_extent_map() should have properly
6958 * initialized em members already.
6959 *
6960 * Other members are not utilized for inline extents.
6961 */
6962 ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE);
6963 ASSERT(em->len == fs_info->sectorsize);
6964
6965 ret = read_inline_extent(inode, path, folio);
6966 if (ret < 0)
6967 goto out;
6968 goto insert;
6969 }
6970 not_found:
6971 em->start = start;
6972 em->len = len;
6973 em->disk_bytenr = EXTENT_MAP_HOLE;
6974 insert:
6975 ret = 0;
6976 btrfs_release_path(path);
6977 if (em->start > start || extent_map_end(em) <= start) {
6978 btrfs_err(fs_info,
6979 "bad extent! em: [%llu %llu] passed [%llu %llu]",
6980 em->start, em->len, start, len);
6981 ret = -EIO;
6982 goto out;
6983 }
6984
6985 write_lock(&em_tree->lock);
6986 ret = btrfs_add_extent_mapping(inode, &em, start, len);
6987 write_unlock(&em_tree->lock);
6988 out:
6989 btrfs_free_path(path);
6990
6991 trace_btrfs_get_extent(root, inode, em);
6992
6993 if (ret) {
6994 free_extent_map(em);
6995 return ERR_PTR(ret);
6996 }
6997 return em;
6998 }
6999
btrfs_extent_readonly(struct btrfs_fs_info * fs_info,u64 bytenr)7000 static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
7001 {
7002 struct btrfs_block_group *block_group;
7003 bool readonly = false;
7004
7005 block_group = btrfs_lookup_block_group(fs_info, bytenr);
7006 if (!block_group || block_group->ro)
7007 readonly = true;
7008 if (block_group)
7009 btrfs_put_block_group(block_group);
7010 return readonly;
7011 }
7012
7013 /*
7014 * Check if we can do nocow write into the range [@offset, @offset + @len)
7015 *
7016 * @offset: File offset
7017 * @len: The length to write, will be updated to the nocow writeable
7018 * range
7019 * @orig_start: (optional) Return the original file offset of the file extent
7020 * @orig_len: (optional) Return the original on-disk length of the file extent
7021 * @ram_bytes: (optional) Return the ram_bytes of the file extent
7022 * @strict: if true, omit optimizations that might force us into unnecessary
7023 * cow. e.g., don't trust generation number.
7024 *
7025 * Return:
7026 * >0 and update @len if we can do nocow write
7027 * 0 if we can't do nocow write
7028 * <0 if error happened
7029 *
7030 * NOTE: This only checks the file extents, caller is responsible to wait for
7031 * any ordered extents.
7032 */
can_nocow_extent(struct inode * inode,u64 offset,u64 * len,struct btrfs_file_extent * file_extent,bool nowait,bool strict)7033 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7034 struct btrfs_file_extent *file_extent,
7035 bool nowait, bool strict)
7036 {
7037 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
7038 struct can_nocow_file_extent_args nocow_args = { 0 };
7039 struct btrfs_path *path;
7040 int ret;
7041 struct extent_buffer *leaf;
7042 struct btrfs_root *root = BTRFS_I(inode)->root;
7043 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7044 struct btrfs_file_extent_item *fi;
7045 struct btrfs_key key;
7046 int found_type;
7047
7048 path = btrfs_alloc_path();
7049 if (!path)
7050 return -ENOMEM;
7051 path->nowait = nowait;
7052
7053 ret = btrfs_lookup_file_extent(NULL, root, path,
7054 btrfs_ino(BTRFS_I(inode)), offset, 0);
7055 if (ret < 0)
7056 goto out;
7057
7058 if (ret == 1) {
7059 if (path->slots[0] == 0) {
7060 /* can't find the item, must cow */
7061 ret = 0;
7062 goto out;
7063 }
7064 path->slots[0]--;
7065 }
7066 ret = 0;
7067 leaf = path->nodes[0];
7068 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
7069 if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7070 key.type != BTRFS_EXTENT_DATA_KEY) {
7071 /* not our file or wrong item type, must cow */
7072 goto out;
7073 }
7074
7075 if (key.offset > offset) {
7076 /* Wrong offset, must cow */
7077 goto out;
7078 }
7079
7080 if (btrfs_file_extent_end(path) <= offset)
7081 goto out;
7082
7083 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
7084 found_type = btrfs_file_extent_type(leaf, fi);
7085
7086 nocow_args.start = offset;
7087 nocow_args.end = offset + *len - 1;
7088 nocow_args.strict = strict;
7089 nocow_args.free_path = true;
7090
7091 ret = can_nocow_file_extent(path, &key, BTRFS_I(inode), &nocow_args);
7092 /* can_nocow_file_extent() has freed the path. */
7093 path = NULL;
7094
7095 if (ret != 1) {
7096 /* Treat errors as not being able to NOCOW. */
7097 ret = 0;
7098 goto out;
7099 }
7100
7101 ret = 0;
7102 if (btrfs_extent_readonly(fs_info,
7103 nocow_args.file_extent.disk_bytenr +
7104 nocow_args.file_extent.offset))
7105 goto out;
7106
7107 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7108 found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7109 u64 range_end;
7110
7111 range_end = round_up(offset + nocow_args.file_extent.num_bytes,
7112 root->fs_info->sectorsize) - 1;
7113 ret = test_range_bit_exists(io_tree, offset, range_end, EXTENT_DELALLOC);
7114 if (ret) {
7115 ret = -EAGAIN;
7116 goto out;
7117 }
7118 }
7119
7120 if (file_extent)
7121 memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent));
7122
7123 *len = nocow_args.file_extent.num_bytes;
7124 ret = 1;
7125 out:
7126 btrfs_free_path(path);
7127 return ret;
7128 }
7129
7130 /* The callers of this must take lock_extent() */
btrfs_create_io_em(struct btrfs_inode * inode,u64 start,const struct btrfs_file_extent * file_extent,int type)7131 struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start,
7132 const struct btrfs_file_extent *file_extent,
7133 int type)
7134 {
7135 struct extent_map *em;
7136 int ret;
7137
7138 /*
7139 * Note the missing NOCOW type.
7140 *
7141 * For pure NOCOW writes, we should not create an io extent map, but
7142 * just reusing the existing one.
7143 * Only PREALLOC writes (NOCOW write into preallocated range) can
7144 * create an io extent map.
7145 */
7146 ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7147 type == BTRFS_ORDERED_COMPRESSED ||
7148 type == BTRFS_ORDERED_REGULAR);
7149
7150 switch (type) {
7151 case BTRFS_ORDERED_PREALLOC:
7152 /* We're only referring part of a larger preallocated extent. */
7153 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7154 break;
7155 case BTRFS_ORDERED_REGULAR:
7156 /* COW results a new extent matching our file extent size. */
7157 ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes);
7158 ASSERT(file_extent->ram_bytes == file_extent->num_bytes);
7159
7160 /* Since it's a new extent, we should not have any offset. */
7161 ASSERT(file_extent->offset == 0);
7162 break;
7163 case BTRFS_ORDERED_COMPRESSED:
7164 /* Must be compressed. */
7165 ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE);
7166
7167 /*
7168 * Encoded write can make us to refer to part of the
7169 * uncompressed extent.
7170 */
7171 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7172 break;
7173 }
7174
7175 em = alloc_extent_map();
7176 if (!em)
7177 return ERR_PTR(-ENOMEM);
7178
7179 em->start = start;
7180 em->len = file_extent->num_bytes;
7181 em->disk_bytenr = file_extent->disk_bytenr;
7182 em->disk_num_bytes = file_extent->disk_num_bytes;
7183 em->ram_bytes = file_extent->ram_bytes;
7184 em->generation = -1;
7185 em->offset = file_extent->offset;
7186 em->flags |= EXTENT_FLAG_PINNED;
7187 if (type == BTRFS_ORDERED_COMPRESSED)
7188 extent_map_set_compression(em, file_extent->compression);
7189
7190 ret = btrfs_replace_extent_map_range(inode, em, true);
7191 if (ret) {
7192 free_extent_map(em);
7193 return ERR_PTR(ret);
7194 }
7195
7196 /* em got 2 refs now, callers needs to do free_extent_map once. */
7197 return em;
7198 }
7199
7200 /*
7201 * For release_folio() and invalidate_folio() we have a race window where
7202 * folio_end_writeback() is called but the subpage spinlock is not yet released.
7203 * If we continue to release/invalidate the page, we could cause use-after-free
7204 * for subpage spinlock. So this function is to spin and wait for subpage
7205 * spinlock.
7206 */
wait_subpage_spinlock(struct folio * folio)7207 static void wait_subpage_spinlock(struct folio *folio)
7208 {
7209 struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
7210 struct btrfs_subpage *subpage;
7211
7212 if (!btrfs_is_subpage(fs_info, folio->mapping))
7213 return;
7214
7215 ASSERT(folio_test_private(folio) && folio_get_private(folio));
7216 subpage = folio_get_private(folio);
7217
7218 /*
7219 * This may look insane as we just acquire the spinlock and release it,
7220 * without doing anything. But we just want to make sure no one is
7221 * still holding the subpage spinlock.
7222 * And since the page is not dirty nor writeback, and we have page
7223 * locked, the only possible way to hold a spinlock is from the endio
7224 * function to clear page writeback.
7225 *
7226 * Here we just acquire the spinlock so that all existing callers
7227 * should exit and we're safe to release/invalidate the page.
7228 */
7229 spin_lock_irq(&subpage->lock);
7230 spin_unlock_irq(&subpage->lock);
7231 }
7232
btrfs_launder_folio(struct folio * folio)7233 static int btrfs_launder_folio(struct folio *folio)
7234 {
7235 return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, folio_pos(folio),
7236 PAGE_SIZE, NULL);
7237 }
7238
__btrfs_release_folio(struct folio * folio,gfp_t gfp_flags)7239 static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7240 {
7241 if (try_release_extent_mapping(folio, gfp_flags)) {
7242 wait_subpage_spinlock(folio);
7243 clear_folio_extent_mapped(folio);
7244 return true;
7245 }
7246 return false;
7247 }
7248
btrfs_release_folio(struct folio * folio,gfp_t gfp_flags)7249 static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7250 {
7251 if (folio_test_writeback(folio) || folio_test_dirty(folio))
7252 return false;
7253 return __btrfs_release_folio(folio, gfp_flags);
7254 }
7255
7256 #ifdef CONFIG_MIGRATION
btrfs_migrate_folio(struct address_space * mapping,struct folio * dst,struct folio * src,enum migrate_mode mode)7257 static int btrfs_migrate_folio(struct address_space *mapping,
7258 struct folio *dst, struct folio *src,
7259 enum migrate_mode mode)
7260 {
7261 int ret = filemap_migrate_folio(mapping, dst, src, mode);
7262
7263 if (ret != MIGRATEPAGE_SUCCESS)
7264 return ret;
7265
7266 if (folio_test_ordered(src)) {
7267 folio_clear_ordered(src);
7268 folio_set_ordered(dst);
7269 }
7270
7271 return MIGRATEPAGE_SUCCESS;
7272 }
7273 #else
7274 #define btrfs_migrate_folio NULL
7275 #endif
7276
btrfs_invalidate_folio(struct folio * folio,size_t offset,size_t length)7277 static void btrfs_invalidate_folio(struct folio *folio, size_t offset,
7278 size_t length)
7279 {
7280 struct btrfs_inode *inode = folio_to_inode(folio);
7281 struct btrfs_fs_info *fs_info = inode->root->fs_info;
7282 struct extent_io_tree *tree = &inode->io_tree;
7283 struct extent_state *cached_state = NULL;
7284 u64 page_start = folio_pos(folio);
7285 u64 page_end = page_start + folio_size(folio) - 1;
7286 u64 cur;
7287 int inode_evicting = inode->vfs_inode.i_state & I_FREEING;
7288
7289 /*
7290 * We have folio locked so no new ordered extent can be created on this
7291 * page, nor bio can be submitted for this folio.
7292 *
7293 * But already submitted bio can still be finished on this folio.
7294 * Furthermore, endio function won't skip folio which has Ordered
7295 * (Private2) already cleared, so it's possible for endio and
7296 * invalidate_folio to do the same ordered extent accounting twice
7297 * on one folio.
7298 *
7299 * So here we wait for any submitted bios to finish, so that we won't
7300 * do double ordered extent accounting on the same folio.
7301 */
7302 folio_wait_writeback(folio);
7303 wait_subpage_spinlock(folio);
7304
7305 /*
7306 * For subpage case, we have call sites like
7307 * btrfs_punch_hole_lock_range() which passes range not aligned to
7308 * sectorsize.
7309 * If the range doesn't cover the full folio, we don't need to and
7310 * shouldn't clear page extent mapped, as folio->private can still
7311 * record subpage dirty bits for other part of the range.
7312 *
7313 * For cases that invalidate the full folio even the range doesn't
7314 * cover the full folio, like invalidating the last folio, we're
7315 * still safe to wait for ordered extent to finish.
7316 */
7317 if (!(offset == 0 && length == folio_size(folio))) {
7318 btrfs_release_folio(folio, GFP_NOFS);
7319 return;
7320 }
7321
7322 if (!inode_evicting)
7323 lock_extent(tree, page_start, page_end, &cached_state);
7324
7325 cur = page_start;
7326 while (cur < page_end) {
7327 struct btrfs_ordered_extent *ordered;
7328 u64 range_end;
7329 u32 range_len;
7330 u32 extra_flags = 0;
7331
7332 ordered = btrfs_lookup_first_ordered_range(inode, cur,
7333 page_end + 1 - cur);
7334 if (!ordered) {
7335 range_end = page_end;
7336 /*
7337 * No ordered extent covering this range, we are safe
7338 * to delete all extent states in the range.
7339 */
7340 extra_flags = EXTENT_CLEAR_ALL_BITS;
7341 goto next;
7342 }
7343 if (ordered->file_offset > cur) {
7344 /*
7345 * There is a range between [cur, oe->file_offset) not
7346 * covered by any ordered extent.
7347 * We are safe to delete all extent states, and handle
7348 * the ordered extent in the next iteration.
7349 */
7350 range_end = ordered->file_offset - 1;
7351 extra_flags = EXTENT_CLEAR_ALL_BITS;
7352 goto next;
7353 }
7354
7355 range_end = min(ordered->file_offset + ordered->num_bytes - 1,
7356 page_end);
7357 ASSERT(range_end + 1 - cur < U32_MAX);
7358 range_len = range_end + 1 - cur;
7359 if (!btrfs_folio_test_ordered(fs_info, folio, cur, range_len)) {
7360 /*
7361 * If Ordered (Private2) is cleared, it means endio has
7362 * already been executed for the range.
7363 * We can't delete the extent states as
7364 * btrfs_finish_ordered_io() may still use some of them.
7365 */
7366 goto next;
7367 }
7368 btrfs_folio_clear_ordered(fs_info, folio, cur, range_len);
7369
7370 /*
7371 * IO on this page will never be started, so we need to account
7372 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW
7373 * here, must leave that up for the ordered extent completion.
7374 *
7375 * This will also unlock the range for incoming
7376 * btrfs_finish_ordered_io().
7377 */
7378 if (!inode_evicting)
7379 clear_extent_bit(tree, cur, range_end,
7380 EXTENT_DELALLOC |
7381 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
7382 EXTENT_DEFRAG, &cached_state);
7383
7384 spin_lock_irq(&inode->ordered_tree_lock);
7385 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
7386 ordered->truncated_len = min(ordered->truncated_len,
7387 cur - ordered->file_offset);
7388 spin_unlock_irq(&inode->ordered_tree_lock);
7389
7390 /*
7391 * If the ordered extent has finished, we're safe to delete all
7392 * the extent states of the range, otherwise
7393 * btrfs_finish_ordered_io() will get executed by endio for
7394 * other pages, so we can't delete extent states.
7395 */
7396 if (btrfs_dec_test_ordered_pending(inode, &ordered,
7397 cur, range_end + 1 - cur)) {
7398 btrfs_finish_ordered_io(ordered);
7399 /*
7400 * The ordered extent has finished, now we're again
7401 * safe to delete all extent states of the range.
7402 */
7403 extra_flags = EXTENT_CLEAR_ALL_BITS;
7404 }
7405 next:
7406 if (ordered)
7407 btrfs_put_ordered_extent(ordered);
7408 /*
7409 * Qgroup reserved space handler
7410 * Sector(s) here will be either:
7411 *
7412 * 1) Already written to disk or bio already finished
7413 * Then its QGROUP_RESERVED bit in io_tree is already cleared.
7414 * Qgroup will be handled by its qgroup_record then.
7415 * btrfs_qgroup_free_data() call will do nothing here.
7416 *
7417 * 2) Not written to disk yet
7418 * Then btrfs_qgroup_free_data() call will clear the
7419 * QGROUP_RESERVED bit of its io_tree, and free the qgroup
7420 * reserved data space.
7421 * Since the IO will never happen for this page.
7422 */
7423 btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur, NULL);
7424 if (!inode_evicting) {
7425 clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED |
7426 EXTENT_DELALLOC | EXTENT_UPTODATE |
7427 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG |
7428 extra_flags, &cached_state);
7429 }
7430 cur = range_end + 1;
7431 }
7432 /*
7433 * We have iterated through all ordered extents of the page, the page
7434 * should not have Ordered (Private2) anymore, or the above iteration
7435 * did something wrong.
7436 */
7437 ASSERT(!folio_test_ordered(folio));
7438 btrfs_folio_clear_checked(fs_info, folio, folio_pos(folio), folio_size(folio));
7439 if (!inode_evicting)
7440 __btrfs_release_folio(folio, GFP_NOFS);
7441 clear_folio_extent_mapped(folio);
7442 }
7443
btrfs_truncate(struct btrfs_inode * inode,bool skip_writeback)7444 static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback)
7445 {
7446 struct btrfs_truncate_control control = {
7447 .inode = inode,
7448 .ino = btrfs_ino(inode),
7449 .min_type = BTRFS_EXTENT_DATA_KEY,
7450 .clear_extent_range = true,
7451 };
7452 struct btrfs_root *root = inode->root;
7453 struct btrfs_fs_info *fs_info = root->fs_info;
7454 struct btrfs_block_rsv *rsv;
7455 int ret;
7456 struct btrfs_trans_handle *trans;
7457 u64 mask = fs_info->sectorsize - 1;
7458 const u64 min_size = btrfs_calc_metadata_size(fs_info, 1);
7459
7460 if (!skip_writeback) {
7461 ret = btrfs_wait_ordered_range(inode,
7462 inode->vfs_inode.i_size & (~mask),
7463 (u64)-1);
7464 if (ret)
7465 return ret;
7466 }
7467
7468 /*
7469 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of
7470 * things going on here:
7471 *
7472 * 1) We need to reserve space to update our inode.
7473 *
7474 * 2) We need to have something to cache all the space that is going to
7475 * be free'd up by the truncate operation, but also have some slack
7476 * space reserved in case it uses space during the truncate (thank you
7477 * very much snapshotting).
7478 *
7479 * And we need these to be separate. The fact is we can use a lot of
7480 * space doing the truncate, and we have no earthly idea how much space
7481 * we will use, so we need the truncate reservation to be separate so it
7482 * doesn't end up using space reserved for updating the inode. We also
7483 * need to be able to stop the transaction and start a new one, which
7484 * means we need to be able to update the inode several times, and we
7485 * have no idea of knowing how many times that will be, so we can't just
7486 * reserve 1 item for the entirety of the operation, so that has to be
7487 * done separately as well.
7488 *
7489 * So that leaves us with
7490 *
7491 * 1) rsv - for the truncate reservation, which we will steal from the
7492 * transaction reservation.
7493 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for
7494 * updating the inode.
7495 */
7496 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
7497 if (!rsv)
7498 return -ENOMEM;
7499 rsv->size = min_size;
7500 rsv->failfast = true;
7501
7502 /*
7503 * 1 for the truncate slack space
7504 * 1 for updating the inode.
7505 */
7506 trans = btrfs_start_transaction(root, 2);
7507 if (IS_ERR(trans)) {
7508 ret = PTR_ERR(trans);
7509 goto out;
7510 }
7511
7512 /* Migrate the slack space for the truncate to our reserve */
7513 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
7514 min_size, false);
7515 /*
7516 * We have reserved 2 metadata units when we started the transaction and
7517 * min_size matches 1 unit, so this should never fail, but if it does,
7518 * it's not critical we just fail truncation.
7519 */
7520 if (WARN_ON(ret)) {
7521 btrfs_end_transaction(trans);
7522 goto out;
7523 }
7524
7525 trans->block_rsv = rsv;
7526
7527 while (1) {
7528 struct extent_state *cached_state = NULL;
7529 const u64 new_size = inode->vfs_inode.i_size;
7530 const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize);
7531
7532 control.new_size = new_size;
7533 lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);
7534 /*
7535 * We want to drop from the next block forward in case this new
7536 * size is not block aligned since we will be keeping the last
7537 * block of the extent just the way it is.
7538 */
7539 btrfs_drop_extent_map_range(inode,
7540 ALIGN(new_size, fs_info->sectorsize),
7541 (u64)-1, false);
7542
7543 ret = btrfs_truncate_inode_items(trans, root, &control);
7544
7545 inode_sub_bytes(&inode->vfs_inode, control.sub_bytes);
7546 btrfs_inode_safe_disk_i_size_write(inode, control.last_size);
7547
7548 unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);
7549
7550 trans->block_rsv = &fs_info->trans_block_rsv;
7551 if (ret != -ENOSPC && ret != -EAGAIN)
7552 break;
7553
7554 ret = btrfs_update_inode(trans, inode);
7555 if (ret)
7556 break;
7557
7558 btrfs_end_transaction(trans);
7559 btrfs_btree_balance_dirty(fs_info);
7560
7561 trans = btrfs_start_transaction(root, 2);
7562 if (IS_ERR(trans)) {
7563 ret = PTR_ERR(trans);
7564 trans = NULL;
7565 break;
7566 }
7567
7568 btrfs_block_rsv_release(fs_info, rsv, -1, NULL);
7569 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
7570 rsv, min_size, false);
7571 /*
7572 * We have reserved 2 metadata units when we started the
7573 * transaction and min_size matches 1 unit, so this should never
7574 * fail, but if it does, it's not critical we just fail truncation.
7575 */
7576 if (WARN_ON(ret))
7577 break;
7578
7579 trans->block_rsv = rsv;
7580 }
7581
7582 /*
7583 * We can't call btrfs_truncate_block inside a trans handle as we could
7584 * deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we
7585 * know we've truncated everything except the last little bit, and can
7586 * do btrfs_truncate_block and then update the disk_i_size.
7587 */
7588 if (ret == BTRFS_NEED_TRUNCATE_BLOCK) {
7589 btrfs_end_transaction(trans);
7590 btrfs_btree_balance_dirty(fs_info);
7591
7592 ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size, 0, 0);
7593 if (ret)
7594 goto out;
7595 trans = btrfs_start_transaction(root, 1);
7596 if (IS_ERR(trans)) {
7597 ret = PTR_ERR(trans);
7598 goto out;
7599 }
7600 btrfs_inode_safe_disk_i_size_write(inode, 0);
7601 }
7602
7603 if (trans) {
7604 int ret2;
7605
7606 trans->block_rsv = &fs_info->trans_block_rsv;
7607 ret2 = btrfs_update_inode(trans, inode);
7608 if (ret2 && !ret)
7609 ret = ret2;
7610
7611 ret2 = btrfs_end_transaction(trans);
7612 if (ret2 && !ret)
7613 ret = ret2;
7614 btrfs_btree_balance_dirty(fs_info);
7615 }
7616 out:
7617 btrfs_free_block_rsv(fs_info, rsv);
7618 /*
7619 * So if we truncate and then write and fsync we normally would just
7620 * write the extents that changed, which is a problem if we need to
7621 * first truncate that entire inode. So set this flag so we write out
7622 * all of the extents in the inode to the sync log so we're completely
7623 * safe.
7624 *
7625 * If no extents were dropped or trimmed we don't need to force the next
7626 * fsync to truncate all the inode's items from the log and re-log them
7627 * all. This means the truncate operation did not change the file size,
7628 * or changed it to a smaller size but there was only an implicit hole
7629 * between the old i_size and the new i_size, and there were no prealloc
7630 * extents beyond i_size to drop.
7631 */
7632 if (control.extents_found > 0)
7633 btrfs_set_inode_full_sync(inode);
7634
7635 return ret;
7636 }
7637
btrfs_new_subvol_inode(struct mnt_idmap * idmap,struct inode * dir)7638 struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap,
7639 struct inode *dir)
7640 {
7641 struct inode *inode;
7642
7643 inode = new_inode(dir->i_sb);
7644 if (inode) {
7645 /*
7646 * Subvolumes don't inherit the sgid bit or the parent's gid if
7647 * the parent's sgid bit is set. This is probably a bug.
7648 */
7649 inode_init_owner(idmap, inode, NULL,
7650 S_IFDIR | (~current_umask() & S_IRWXUGO));
7651 inode->i_op = &btrfs_dir_inode_operations;
7652 inode->i_fop = &btrfs_dir_file_operations;
7653 }
7654 return inode;
7655 }
7656
btrfs_alloc_inode(struct super_block * sb)7657 struct inode *btrfs_alloc_inode(struct super_block *sb)
7658 {
7659 struct btrfs_fs_info *fs_info = btrfs_sb(sb);
7660 struct btrfs_inode *ei;
7661 struct inode *inode;
7662
7663 ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL);
7664 if (!ei)
7665 return NULL;
7666
7667 ei->root = NULL;
7668 ei->generation = 0;
7669 ei->last_trans = 0;
7670 ei->last_sub_trans = 0;
7671 ei->logged_trans = 0;
7672 ei->delalloc_bytes = 0;
7673 ei->new_delalloc_bytes = 0;
7674 ei->defrag_bytes = 0;
7675 ei->disk_i_size = 0;
7676 ei->flags = 0;
7677 ei->ro_flags = 0;
7678 /*
7679 * ->index_cnt will be properly initialized later when creating a new
7680 * inode (btrfs_create_new_inode()) or when reading an existing inode
7681 * from disk (btrfs_read_locked_inode()).
7682 */
7683 ei->csum_bytes = 0;
7684 ei->dir_index = 0;
7685 ei->last_unlink_trans = 0;
7686 ei->last_reflink_trans = 0;
7687 ei->last_log_commit = 0;
7688
7689 spin_lock_init(&ei->lock);
7690 ei->outstanding_extents = 0;
7691 if (sb->s_magic != BTRFS_TEST_MAGIC)
7692 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
7693 BTRFS_BLOCK_RSV_DELALLOC);
7694 ei->runtime_flags = 0;
7695 ei->prop_compress = BTRFS_COMPRESS_NONE;
7696 ei->defrag_compress = BTRFS_COMPRESS_NONE;
7697
7698 ei->delayed_node = NULL;
7699
7700 ei->i_otime_sec = 0;
7701 ei->i_otime_nsec = 0;
7702
7703 inode = &ei->vfs_inode;
7704 extent_map_tree_init(&ei->extent_tree);
7705
7706 /* This io tree sets the valid inode. */
7707 extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO);
7708 ei->io_tree.inode = ei;
7709
7710 ei->file_extent_tree = NULL;
7711
7712 mutex_init(&ei->log_mutex);
7713 spin_lock_init(&ei->ordered_tree_lock);
7714 ei->ordered_tree = RB_ROOT;
7715 ei->ordered_tree_last = NULL;
7716 INIT_LIST_HEAD(&ei->delalloc_inodes);
7717 INIT_LIST_HEAD(&ei->delayed_iput);
7718 init_rwsem(&ei->i_mmap_lock);
7719
7720 return inode;
7721 }
7722
7723 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
btrfs_test_destroy_inode(struct inode * inode)7724 void btrfs_test_destroy_inode(struct inode *inode)
7725 {
7726 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
7727 kfree(BTRFS_I(inode)->file_extent_tree);
7728 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
7729 }
7730 #endif
7731
btrfs_free_inode(struct inode * inode)7732 void btrfs_free_inode(struct inode *inode)
7733 {
7734 kfree(BTRFS_I(inode)->file_extent_tree);
7735 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
7736 }
7737
btrfs_destroy_inode(struct inode * vfs_inode)7738 void btrfs_destroy_inode(struct inode *vfs_inode)
7739 {
7740 struct btrfs_ordered_extent *ordered;
7741 struct btrfs_inode *inode = BTRFS_I(vfs_inode);
7742 struct btrfs_root *root = inode->root;
7743 bool freespace_inode;
7744
7745 WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
7746 WARN_ON(vfs_inode->i_data.nrpages);
7747 WARN_ON(inode->block_rsv.reserved);
7748 WARN_ON(inode->block_rsv.size);
7749 WARN_ON(inode->outstanding_extents);
7750 if (!S_ISDIR(vfs_inode->i_mode)) {
7751 WARN_ON(inode->delalloc_bytes);
7752 WARN_ON(inode->new_delalloc_bytes);
7753 WARN_ON(inode->csum_bytes);
7754 }
7755 if (!root || !btrfs_is_data_reloc_root(root))
7756 WARN_ON(inode->defrag_bytes);
7757
7758 /*
7759 * This can happen where we create an inode, but somebody else also
7760 * created the same inode and we need to destroy the one we already
7761 * created.
7762 */
7763 if (!root)
7764 return;
7765
7766 /*
7767 * If this is a free space inode do not take the ordered extents lockdep
7768 * map.
7769 */
7770 freespace_inode = btrfs_is_free_space_inode(inode);
7771
7772 while (1) {
7773 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
7774 if (!ordered)
7775 break;
7776 else {
7777 btrfs_err(root->fs_info,
7778 "found ordered extent %llu %llu on inode cleanup",
7779 ordered->file_offset, ordered->num_bytes);
7780
7781 if (!freespace_inode)
7782 btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
7783
7784 btrfs_remove_ordered_extent(inode, ordered);
7785 btrfs_put_ordered_extent(ordered);
7786 btrfs_put_ordered_extent(ordered);
7787 }
7788 }
7789 btrfs_qgroup_check_reserved_leak(inode);
7790 btrfs_del_inode_from_root(inode);
7791 btrfs_drop_extent_map_range(inode, 0, (u64)-1, false);
7792 btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1);
7793 btrfs_put_root(inode->root);
7794 }
7795
btrfs_drop_inode(struct inode * inode)7796 int btrfs_drop_inode(struct inode *inode)
7797 {
7798 struct btrfs_root *root = BTRFS_I(inode)->root;
7799
7800 if (root == NULL)
7801 return 1;
7802
7803 /* the snap/subvol tree is on deleting */
7804 if (btrfs_root_refs(&root->root_item) == 0)
7805 return 1;
7806 else
7807 return generic_drop_inode(inode);
7808 }
7809
init_once(void * foo)7810 static void init_once(void *foo)
7811 {
7812 struct btrfs_inode *ei = foo;
7813
7814 inode_init_once(&ei->vfs_inode);
7815 }
7816
btrfs_destroy_cachep(void)7817 void __cold btrfs_destroy_cachep(void)
7818 {
7819 /*
7820 * Make sure all delayed rcu free inodes are flushed before we
7821 * destroy cache.
7822 */
7823 rcu_barrier();
7824 kmem_cache_destroy(btrfs_inode_cachep);
7825 }
7826
btrfs_init_cachep(void)7827 int __init btrfs_init_cachep(void)
7828 {
7829 btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
7830 sizeof(struct btrfs_inode), 0,
7831 SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
7832 init_once);
7833 if (!btrfs_inode_cachep)
7834 return -ENOMEM;
7835
7836 return 0;
7837 }
7838
btrfs_getattr(struct mnt_idmap * idmap,const struct path * path,struct kstat * stat,u32 request_mask,unsigned int flags)7839 static int btrfs_getattr(struct mnt_idmap *idmap,
7840 const struct path *path, struct kstat *stat,
7841 u32 request_mask, unsigned int flags)
7842 {
7843 u64 delalloc_bytes;
7844 u64 inode_bytes;
7845 struct inode *inode = d_inode(path->dentry);
7846 u32 blocksize = btrfs_sb(inode->i_sb)->sectorsize;
7847 u32 bi_flags = BTRFS_I(inode)->flags;
7848 u32 bi_ro_flags = BTRFS_I(inode)->ro_flags;
7849
7850 stat->result_mask |= STATX_BTIME;
7851 stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec;
7852 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec;
7853 if (bi_flags & BTRFS_INODE_APPEND)
7854 stat->attributes |= STATX_ATTR_APPEND;
7855 if (bi_flags & BTRFS_INODE_COMPRESS)
7856 stat->attributes |= STATX_ATTR_COMPRESSED;
7857 if (bi_flags & BTRFS_INODE_IMMUTABLE)
7858 stat->attributes |= STATX_ATTR_IMMUTABLE;
7859 if (bi_flags & BTRFS_INODE_NODUMP)
7860 stat->attributes |= STATX_ATTR_NODUMP;
7861 if (bi_ro_flags & BTRFS_INODE_RO_VERITY)
7862 stat->attributes |= STATX_ATTR_VERITY;
7863
7864 stat->attributes_mask |= (STATX_ATTR_APPEND |
7865 STATX_ATTR_COMPRESSED |
7866 STATX_ATTR_IMMUTABLE |
7867 STATX_ATTR_NODUMP);
7868
7869 generic_fillattr(idmap, request_mask, inode, stat);
7870 stat->dev = BTRFS_I(inode)->root->anon_dev;
7871
7872 stat->subvol = BTRFS_I(inode)->root->root_key.objectid;
7873 stat->result_mask |= STATX_SUBVOL;
7874
7875 spin_lock(&BTRFS_I(inode)->lock);
7876 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
7877 inode_bytes = inode_get_bytes(inode);
7878 spin_unlock(&BTRFS_I(inode)->lock);
7879 stat->blocks = (ALIGN(inode_bytes, blocksize) +
7880 ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT;
7881 return 0;
7882 }
7883
btrfs_rename_exchange(struct inode * old_dir,struct dentry * old_dentry,struct inode * new_dir,struct dentry * new_dentry)7884 static int btrfs_rename_exchange(struct inode *old_dir,
7885 struct dentry *old_dentry,
7886 struct inode *new_dir,
7887 struct dentry *new_dentry)
7888 {
7889 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
7890 struct btrfs_trans_handle *trans;
7891 unsigned int trans_num_items;
7892 struct btrfs_root *root = BTRFS_I(old_dir)->root;
7893 struct btrfs_root *dest = BTRFS_I(new_dir)->root;
7894 struct inode *new_inode = new_dentry->d_inode;
7895 struct inode *old_inode = old_dentry->d_inode;
7896 struct btrfs_rename_ctx old_rename_ctx;
7897 struct btrfs_rename_ctx new_rename_ctx;
7898 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
7899 u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
7900 u64 old_idx = 0;
7901 u64 new_idx = 0;
7902 int ret;
7903 int ret2;
7904 bool need_abort = false;
7905 struct fscrypt_name old_fname, new_fname;
7906 struct fscrypt_str *old_name, *new_name;
7907
7908 /*
7909 * For non-subvolumes allow exchange only within one subvolume, in the
7910 * same inode namespace. Two subvolumes (represented as directory) can
7911 * be exchanged as they're a logical link and have a fixed inode number.
7912 */
7913 if (root != dest &&
7914 (old_ino != BTRFS_FIRST_FREE_OBJECTID ||
7915 new_ino != BTRFS_FIRST_FREE_OBJECTID))
7916 return -EXDEV;
7917
7918 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);
7919 if (ret)
7920 return ret;
7921
7922 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);
7923 if (ret) {
7924 fscrypt_free_filename(&old_fname);
7925 return ret;
7926 }
7927
7928 old_name = &old_fname.disk_name;
7929 new_name = &new_fname.disk_name;
7930
7931 /* close the race window with snapshot create/destroy ioctl */
7932 if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||
7933 new_ino == BTRFS_FIRST_FREE_OBJECTID)
7934 down_read(&fs_info->subvol_sem);
7935
7936 /*
7937 * For each inode:
7938 * 1 to remove old dir item
7939 * 1 to remove old dir index
7940 * 1 to add new dir item
7941 * 1 to add new dir index
7942 * 1 to update parent inode
7943 *
7944 * If the parents are the same, we only need to account for one
7945 */
7946 trans_num_items = (old_dir == new_dir ? 9 : 10);
7947 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
7948 /*
7949 * 1 to remove old root ref
7950 * 1 to remove old root backref
7951 * 1 to add new root ref
7952 * 1 to add new root backref
7953 */
7954 trans_num_items += 4;
7955 } else {
7956 /*
7957 * 1 to update inode item
7958 * 1 to remove old inode ref
7959 * 1 to add new inode ref
7960 */
7961 trans_num_items += 3;
7962 }
7963 if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
7964 trans_num_items += 4;
7965 else
7966 trans_num_items += 3;
7967 trans = btrfs_start_transaction(root, trans_num_items);
7968 if (IS_ERR(trans)) {
7969 ret = PTR_ERR(trans);
7970 goto out_notrans;
7971 }
7972
7973 if (dest != root) {
7974 ret = btrfs_record_root_in_trans(trans, dest);
7975 if (ret)
7976 goto out_fail;
7977 }
7978
7979 /*
7980 * We need to find a free sequence number both in the source and
7981 * in the destination directory for the exchange.
7982 */
7983 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
7984 if (ret)
7985 goto out_fail;
7986 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
7987 if (ret)
7988 goto out_fail;
7989
7990 BTRFS_I(old_inode)->dir_index = 0ULL;
7991 BTRFS_I(new_inode)->dir_index = 0ULL;
7992
7993 /* Reference for the source. */
7994 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
7995 /* force full log commit if subvolume involved. */
7996 btrfs_set_log_full_commit(trans);
7997 } else {
7998 ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino,
7999 btrfs_ino(BTRFS_I(new_dir)),
8000 old_idx);
8001 if (ret)
8002 goto out_fail;
8003 need_abort = true;
8004 }
8005
8006 /* And now for the dest. */
8007 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8008 /* force full log commit if subvolume involved. */
8009 btrfs_set_log_full_commit(trans);
8010 } else {
8011 ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino,
8012 btrfs_ino(BTRFS_I(old_dir)),
8013 new_idx);
8014 if (ret) {
8015 if (need_abort)
8016 btrfs_abort_transaction(trans, ret);
8017 goto out_fail;
8018 }
8019 }
8020
8021 /* Update inode version and ctime/mtime. */
8022 inode_inc_iversion(old_dir);
8023 inode_inc_iversion(new_dir);
8024 inode_inc_iversion(old_inode);
8025 inode_inc_iversion(new_inode);
8026 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8027
8028 if (old_dentry->d_parent != new_dentry->d_parent) {
8029 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8030 BTRFS_I(old_inode), true);
8031 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
8032 BTRFS_I(new_inode), true);
8033 }
8034
8035 /* src is a subvolume */
8036 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8037 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);
8038 } else { /* src is an inode */
8039 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8040 BTRFS_I(old_dentry->d_inode),
8041 old_name, &old_rename_ctx);
8042 if (!ret)
8043 ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8044 }
8045 if (ret) {
8046 btrfs_abort_transaction(trans, ret);
8047 goto out_fail;
8048 }
8049
8050 /* dest is a subvolume */
8051 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8052 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);
8053 } else { /* dest is an inode */
8054 ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8055 BTRFS_I(new_dentry->d_inode),
8056 new_name, &new_rename_ctx);
8057 if (!ret)
8058 ret = btrfs_update_inode(trans, BTRFS_I(new_inode));
8059 }
8060 if (ret) {
8061 btrfs_abort_transaction(trans, ret);
8062 goto out_fail;
8063 }
8064
8065 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8066 new_name, 0, old_idx);
8067 if (ret) {
8068 btrfs_abort_transaction(trans, ret);
8069 goto out_fail;
8070 }
8071
8072 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
8073 old_name, 0, new_idx);
8074 if (ret) {
8075 btrfs_abort_transaction(trans, ret);
8076 goto out_fail;
8077 }
8078
8079 if (old_inode->i_nlink == 1)
8080 BTRFS_I(old_inode)->dir_index = old_idx;
8081 if (new_inode->i_nlink == 1)
8082 BTRFS_I(new_inode)->dir_index = new_idx;
8083
8084 /*
8085 * Now pin the logs of the roots. We do it to ensure that no other task
8086 * can sync the logs while we are in progress with the rename, because
8087 * that could result in an inconsistency in case any of the inodes that
8088 * are part of this rename operation were logged before.
8089 */
8090 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)
8091 btrfs_pin_log_trans(root);
8092 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)
8093 btrfs_pin_log_trans(dest);
8094
8095 /* Do the log updates for all inodes. */
8096 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)
8097 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8098 old_rename_ctx.index, new_dentry->d_parent);
8099 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)
8100 btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir),
8101 new_rename_ctx.index, old_dentry->d_parent);
8102
8103 /* Now unpin the logs. */
8104 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)
8105 btrfs_end_log_trans(root);
8106 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)
8107 btrfs_end_log_trans(dest);
8108 out_fail:
8109 ret2 = btrfs_end_transaction(trans);
8110 ret = ret ? ret : ret2;
8111 out_notrans:
8112 if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||
8113 old_ino == BTRFS_FIRST_FREE_OBJECTID)
8114 up_read(&fs_info->subvol_sem);
8115
8116 fscrypt_free_filename(&new_fname);
8117 fscrypt_free_filename(&old_fname);
8118 return ret;
8119 }
8120
new_whiteout_inode(struct mnt_idmap * idmap,struct inode * dir)8121 static struct inode *new_whiteout_inode(struct mnt_idmap *idmap,
8122 struct inode *dir)
8123 {
8124 struct inode *inode;
8125
8126 inode = new_inode(dir->i_sb);
8127 if (inode) {
8128 inode_init_owner(idmap, inode, dir,
8129 S_IFCHR | WHITEOUT_MODE);
8130 inode->i_op = &btrfs_special_inode_operations;
8131 init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);
8132 }
8133 return inode;
8134 }
8135
btrfs_rename(struct mnt_idmap * idmap,struct inode * old_dir,struct dentry * old_dentry,struct inode * new_dir,struct dentry * new_dentry,unsigned int flags)8136 static int btrfs_rename(struct mnt_idmap *idmap,
8137 struct inode *old_dir, struct dentry *old_dentry,
8138 struct inode *new_dir, struct dentry *new_dentry,
8139 unsigned int flags)
8140 {
8141 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
8142 struct btrfs_new_inode_args whiteout_args = {
8143 .dir = old_dir,
8144 .dentry = old_dentry,
8145 };
8146 struct btrfs_trans_handle *trans;
8147 unsigned int trans_num_items;
8148 struct btrfs_root *root = BTRFS_I(old_dir)->root;
8149 struct btrfs_root *dest = BTRFS_I(new_dir)->root;
8150 struct inode *new_inode = d_inode(new_dentry);
8151 struct inode *old_inode = d_inode(old_dentry);
8152 struct btrfs_rename_ctx rename_ctx;
8153 u64 index = 0;
8154 int ret;
8155 int ret2;
8156 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
8157 struct fscrypt_name old_fname, new_fname;
8158
8159 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
8160 return -EPERM;
8161
8162 /* we only allow rename subvolume link between subvolumes */
8163 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
8164 return -EXDEV;
8165
8166 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
8167 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
8168 return -ENOTEMPTY;
8169
8170 if (S_ISDIR(old_inode->i_mode) && new_inode &&
8171 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
8172 return -ENOTEMPTY;
8173
8174 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);
8175 if (ret)
8176 return ret;
8177
8178 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);
8179 if (ret) {
8180 fscrypt_free_filename(&old_fname);
8181 return ret;
8182 }
8183
8184 /* check for collisions, even if the name isn't there */
8185 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name);
8186 if (ret) {
8187 if (ret == -EEXIST) {
8188 /* we shouldn't get
8189 * eexist without a new_inode */
8190 if (WARN_ON(!new_inode)) {
8191 goto out_fscrypt_names;
8192 }
8193 } else {
8194 /* maybe -EOVERFLOW */
8195 goto out_fscrypt_names;
8196 }
8197 }
8198 ret = 0;
8199
8200 /*
8201 * we're using rename to replace one file with another. Start IO on it
8202 * now so we don't add too much work to the end of the transaction
8203 */
8204 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
8205 filemap_flush(old_inode->i_mapping);
8206
8207 if (flags & RENAME_WHITEOUT) {
8208 whiteout_args.inode = new_whiteout_inode(idmap, old_dir);
8209 if (!whiteout_args.inode) {
8210 ret = -ENOMEM;
8211 goto out_fscrypt_names;
8212 }
8213 ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items);
8214 if (ret)
8215 goto out_whiteout_inode;
8216 } else {
8217 /* 1 to update the old parent inode. */
8218 trans_num_items = 1;
8219 }
8220
8221 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8222 /* Close the race window with snapshot create/destroy ioctl */
8223 down_read(&fs_info->subvol_sem);
8224 /*
8225 * 1 to remove old root ref
8226 * 1 to remove old root backref
8227 * 1 to add new root ref
8228 * 1 to add new root backref
8229 */
8230 trans_num_items += 4;
8231 } else {
8232 /*
8233 * 1 to update inode
8234 * 1 to remove old inode ref
8235 * 1 to add new inode ref
8236 */
8237 trans_num_items += 3;
8238 }
8239 /*
8240 * 1 to remove old dir item
8241 * 1 to remove old dir index
8242 * 1 to add new dir item
8243 * 1 to add new dir index
8244 */
8245 trans_num_items += 4;
8246 /* 1 to update new parent inode if it's not the same as the old parent */
8247 if (new_dir != old_dir)
8248 trans_num_items++;
8249 if (new_inode) {
8250 /*
8251 * 1 to update inode
8252 * 1 to remove inode ref
8253 * 1 to remove dir item
8254 * 1 to remove dir index
8255 * 1 to possibly add orphan item
8256 */
8257 trans_num_items += 5;
8258 }
8259 trans = btrfs_start_transaction(root, trans_num_items);
8260 if (IS_ERR(trans)) {
8261 ret = PTR_ERR(trans);
8262 goto out_notrans;
8263 }
8264
8265 if (dest != root) {
8266 ret = btrfs_record_root_in_trans(trans, dest);
8267 if (ret)
8268 goto out_fail;
8269 }
8270
8271 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
8272 if (ret)
8273 goto out_fail;
8274
8275 BTRFS_I(old_inode)->dir_index = 0ULL;
8276 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8277 /* force full log commit if subvolume involved. */
8278 btrfs_set_log_full_commit(trans);
8279 } else {
8280 ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name,
8281 old_ino, btrfs_ino(BTRFS_I(new_dir)),
8282 index);
8283 if (ret)
8284 goto out_fail;
8285 }
8286
8287 inode_inc_iversion(old_dir);
8288 inode_inc_iversion(new_dir);
8289 inode_inc_iversion(old_inode);
8290 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8291
8292 if (old_dentry->d_parent != new_dentry->d_parent)
8293 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8294 BTRFS_I(old_inode), true);
8295
8296 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8297 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);
8298 } else {
8299 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8300 BTRFS_I(d_inode(old_dentry)),
8301 &old_fname.disk_name, &rename_ctx);
8302 if (!ret)
8303 ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8304 }
8305 if (ret) {
8306 btrfs_abort_transaction(trans, ret);
8307 goto out_fail;
8308 }
8309
8310 if (new_inode) {
8311 inode_inc_iversion(new_inode);
8312 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
8313 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
8314 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);
8315 BUG_ON(new_inode->i_nlink == 0);
8316 } else {
8317 ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8318 BTRFS_I(d_inode(new_dentry)),
8319 &new_fname.disk_name);
8320 }
8321 if (!ret && new_inode->i_nlink == 0)
8322 ret = btrfs_orphan_add(trans,
8323 BTRFS_I(d_inode(new_dentry)));
8324 if (ret) {
8325 btrfs_abort_transaction(trans, ret);
8326 goto out_fail;
8327 }
8328 }
8329
8330 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8331 &new_fname.disk_name, 0, index);
8332 if (ret) {
8333 btrfs_abort_transaction(trans, ret);
8334 goto out_fail;
8335 }
8336
8337 if (old_inode->i_nlink == 1)
8338 BTRFS_I(old_inode)->dir_index = index;
8339
8340 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)
8341 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8342 rename_ctx.index, new_dentry->d_parent);
8343
8344 if (flags & RENAME_WHITEOUT) {
8345 ret = btrfs_create_new_inode(trans, &whiteout_args);
8346 if (ret) {
8347 btrfs_abort_transaction(trans, ret);
8348 goto out_fail;
8349 } else {
8350 unlock_new_inode(whiteout_args.inode);
8351 iput(whiteout_args.inode);
8352 whiteout_args.inode = NULL;
8353 }
8354 }
8355 out_fail:
8356 ret2 = btrfs_end_transaction(trans);
8357 ret = ret ? ret : ret2;
8358 out_notrans:
8359 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
8360 up_read(&fs_info->subvol_sem);
8361 if (flags & RENAME_WHITEOUT)
8362 btrfs_new_inode_args_destroy(&whiteout_args);
8363 out_whiteout_inode:
8364 if (flags & RENAME_WHITEOUT)
8365 iput(whiteout_args.inode);
8366 out_fscrypt_names:
8367 fscrypt_free_filename(&old_fname);
8368 fscrypt_free_filename(&new_fname);
8369 return ret;
8370 }
8371
btrfs_rename2(struct mnt_idmap * idmap,struct inode * old_dir,struct dentry * old_dentry,struct inode * new_dir,struct dentry * new_dentry,unsigned int flags)8372 static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
8373 struct dentry *old_dentry, struct inode *new_dir,
8374 struct dentry *new_dentry, unsigned int flags)
8375 {
8376 int ret;
8377
8378 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
8379 return -EINVAL;
8380
8381 if (flags & RENAME_EXCHANGE)
8382 ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir,
8383 new_dentry);
8384 else
8385 ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir,
8386 new_dentry, flags);
8387
8388 btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info);
8389
8390 return ret;
8391 }
8392
8393 struct btrfs_delalloc_work {
8394 struct inode *inode;
8395 struct completion completion;
8396 struct list_head list;
8397 struct btrfs_work work;
8398 };
8399
btrfs_run_delalloc_work(struct btrfs_work * work)8400 static void btrfs_run_delalloc_work(struct btrfs_work *work)
8401 {
8402 struct btrfs_delalloc_work *delalloc_work;
8403 struct inode *inode;
8404
8405 delalloc_work = container_of(work, struct btrfs_delalloc_work,
8406 work);
8407 inode = delalloc_work->inode;
8408 filemap_flush(inode->i_mapping);
8409 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8410 &BTRFS_I(inode)->runtime_flags))
8411 filemap_flush(inode->i_mapping);
8412
8413 iput(inode);
8414 complete(&delalloc_work->completion);
8415 }
8416
btrfs_alloc_delalloc_work(struct inode * inode)8417 static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode)
8418 {
8419 struct btrfs_delalloc_work *work;
8420
8421 work = kmalloc(sizeof(*work), GFP_NOFS);
8422 if (!work)
8423 return NULL;
8424
8425 init_completion(&work->completion);
8426 INIT_LIST_HEAD(&work->list);
8427 work->inode = inode;
8428 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL);
8429
8430 return work;
8431 }
8432
8433 /*
8434 * some fairly slow code that needs optimization. This walks the list
8435 * of all the inodes with pending delalloc and forces them to disk.
8436 */
start_delalloc_inodes(struct btrfs_root * root,struct writeback_control * wbc,bool snapshot,bool in_reclaim_context)8437 static int start_delalloc_inodes(struct btrfs_root *root,
8438 struct writeback_control *wbc, bool snapshot,
8439 bool in_reclaim_context)
8440 {
8441 struct btrfs_inode *binode;
8442 struct inode *inode;
8443 struct btrfs_delalloc_work *work, *next;
8444 LIST_HEAD(works);
8445 LIST_HEAD(splice);
8446 int ret = 0;
8447 bool full_flush = wbc->nr_to_write == LONG_MAX;
8448
8449 mutex_lock(&root->delalloc_mutex);
8450 spin_lock(&root->delalloc_lock);
8451 list_splice_init(&root->delalloc_inodes, &splice);
8452 while (!list_empty(&splice)) {
8453 binode = list_entry(splice.next, struct btrfs_inode,
8454 delalloc_inodes);
8455
8456 list_move_tail(&binode->delalloc_inodes,
8457 &root->delalloc_inodes);
8458
8459 if (in_reclaim_context &&
8460 test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &binode->runtime_flags))
8461 continue;
8462
8463 inode = igrab(&binode->vfs_inode);
8464 if (!inode) {
8465 cond_resched_lock(&root->delalloc_lock);
8466 continue;
8467 }
8468 spin_unlock(&root->delalloc_lock);
8469
8470 if (snapshot)
8471 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
8472 &binode->runtime_flags);
8473 if (full_flush) {
8474 work = btrfs_alloc_delalloc_work(inode);
8475 if (!work) {
8476 iput(inode);
8477 ret = -ENOMEM;
8478 goto out;
8479 }
8480 list_add_tail(&work->list, &works);
8481 btrfs_queue_work(root->fs_info->flush_workers,
8482 &work->work);
8483 } else {
8484 ret = filemap_fdatawrite_wbc(inode->i_mapping, wbc);
8485 btrfs_add_delayed_iput(BTRFS_I(inode));
8486 if (ret || wbc->nr_to_write <= 0)
8487 goto out;
8488 }
8489 cond_resched();
8490 spin_lock(&root->delalloc_lock);
8491 }
8492 spin_unlock(&root->delalloc_lock);
8493
8494 out:
8495 list_for_each_entry_safe(work, next, &works, list) {
8496 list_del_init(&work->list);
8497 wait_for_completion(&work->completion);
8498 kfree(work);
8499 }
8500
8501 if (!list_empty(&splice)) {
8502 spin_lock(&root->delalloc_lock);
8503 list_splice_tail(&splice, &root->delalloc_inodes);
8504 spin_unlock(&root->delalloc_lock);
8505 }
8506 mutex_unlock(&root->delalloc_mutex);
8507 return ret;
8508 }
8509
btrfs_start_delalloc_snapshot(struct btrfs_root * root,bool in_reclaim_context)8510 int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context)
8511 {
8512 struct writeback_control wbc = {
8513 .nr_to_write = LONG_MAX,
8514 .sync_mode = WB_SYNC_NONE,
8515 .range_start = 0,
8516 .range_end = LLONG_MAX,
8517 };
8518 struct btrfs_fs_info *fs_info = root->fs_info;
8519
8520 if (BTRFS_FS_ERROR(fs_info))
8521 return -EROFS;
8522
8523 return start_delalloc_inodes(root, &wbc, true, in_reclaim_context);
8524 }
8525
btrfs_start_delalloc_roots(struct btrfs_fs_info * fs_info,long nr,bool in_reclaim_context)8526 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr,
8527 bool in_reclaim_context)
8528 {
8529 struct writeback_control wbc = {
8530 .nr_to_write = nr,
8531 .sync_mode = WB_SYNC_NONE,
8532 .range_start = 0,
8533 .range_end = LLONG_MAX,
8534 };
8535 struct btrfs_root *root;
8536 LIST_HEAD(splice);
8537 int ret;
8538
8539 if (BTRFS_FS_ERROR(fs_info))
8540 return -EROFS;
8541
8542 mutex_lock(&fs_info->delalloc_root_mutex);
8543 spin_lock(&fs_info->delalloc_root_lock);
8544 list_splice_init(&fs_info->delalloc_roots, &splice);
8545 while (!list_empty(&splice)) {
8546 /*
8547 * Reset nr_to_write here so we know that we're doing a full
8548 * flush.
8549 */
8550 if (nr == LONG_MAX)
8551 wbc.nr_to_write = LONG_MAX;
8552
8553 root = list_first_entry(&splice, struct btrfs_root,
8554 delalloc_root);
8555 root = btrfs_grab_root(root);
8556 BUG_ON(!root);
8557 list_move_tail(&root->delalloc_root,
8558 &fs_info->delalloc_roots);
8559 spin_unlock(&fs_info->delalloc_root_lock);
8560
8561 ret = start_delalloc_inodes(root, &wbc, false, in_reclaim_context);
8562 btrfs_put_root(root);
8563 if (ret < 0 || wbc.nr_to_write <= 0)
8564 goto out;
8565 spin_lock(&fs_info->delalloc_root_lock);
8566 }
8567 spin_unlock(&fs_info->delalloc_root_lock);
8568
8569 ret = 0;
8570 out:
8571 if (!list_empty(&splice)) {
8572 spin_lock(&fs_info->delalloc_root_lock);
8573 list_splice_tail(&splice, &fs_info->delalloc_roots);
8574 spin_unlock(&fs_info->delalloc_root_lock);
8575 }
8576 mutex_unlock(&fs_info->delalloc_root_mutex);
8577 return ret;
8578 }
8579
btrfs_symlink(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,const char * symname)8580 static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
8581 struct dentry *dentry, const char *symname)
8582 {
8583 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
8584 struct btrfs_trans_handle *trans;
8585 struct btrfs_root *root = BTRFS_I(dir)->root;
8586 struct btrfs_path *path;
8587 struct btrfs_key key;
8588 struct inode *inode;
8589 struct btrfs_new_inode_args new_inode_args = {
8590 .dir = dir,
8591 .dentry = dentry,
8592 };
8593 unsigned int trans_num_items;
8594 int err;
8595 int name_len;
8596 int datasize;
8597 unsigned long ptr;
8598 struct btrfs_file_extent_item *ei;
8599 struct extent_buffer *leaf;
8600
8601 name_len = strlen(symname);
8602 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
8603 return -ENAMETOOLONG;
8604
8605 inode = new_inode(dir->i_sb);
8606 if (!inode)
8607 return -ENOMEM;
8608 inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO);
8609 inode->i_op = &btrfs_symlink_inode_operations;
8610 inode_nohighmem(inode);
8611 inode->i_mapping->a_ops = &btrfs_aops;
8612 btrfs_i_size_write(BTRFS_I(inode), name_len);
8613 inode_set_bytes(inode, name_len);
8614
8615 new_inode_args.inode = inode;
8616 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
8617 if (err)
8618 goto out_inode;
8619 /* 1 additional item for the inline extent */
8620 trans_num_items++;
8621
8622 trans = btrfs_start_transaction(root, trans_num_items);
8623 if (IS_ERR(trans)) {
8624 err = PTR_ERR(trans);
8625 goto out_new_inode_args;
8626 }
8627
8628 err = btrfs_create_new_inode(trans, &new_inode_args);
8629 if (err)
8630 goto out;
8631
8632 path = btrfs_alloc_path();
8633 if (!path) {
8634 err = -ENOMEM;
8635 btrfs_abort_transaction(trans, err);
8636 discard_new_inode(inode);
8637 inode = NULL;
8638 goto out;
8639 }
8640 key.objectid = btrfs_ino(BTRFS_I(inode));
8641 key.offset = 0;
8642 key.type = BTRFS_EXTENT_DATA_KEY;
8643 datasize = btrfs_file_extent_calc_inline_size(name_len);
8644 err = btrfs_insert_empty_item(trans, root, path, &key,
8645 datasize);
8646 if (err) {
8647 btrfs_abort_transaction(trans, err);
8648 btrfs_free_path(path);
8649 discard_new_inode(inode);
8650 inode = NULL;
8651 goto out;
8652 }
8653 leaf = path->nodes[0];
8654 ei = btrfs_item_ptr(leaf, path->slots[0],
8655 struct btrfs_file_extent_item);
8656 btrfs_set_file_extent_generation(leaf, ei, trans->transid);
8657 btrfs_set_file_extent_type(leaf, ei,
8658 BTRFS_FILE_EXTENT_INLINE);
8659 btrfs_set_file_extent_encryption(leaf, ei, 0);
8660 btrfs_set_file_extent_compression(leaf, ei, 0);
8661 btrfs_set_file_extent_other_encoding(leaf, ei, 0);
8662 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
8663
8664 ptr = btrfs_file_extent_inline_start(ei);
8665 write_extent_buffer(leaf, symname, ptr, name_len);
8666 btrfs_mark_buffer_dirty(trans, leaf);
8667 btrfs_free_path(path);
8668
8669 d_instantiate_new(dentry, inode);
8670 err = 0;
8671 out:
8672 btrfs_end_transaction(trans);
8673 btrfs_btree_balance_dirty(fs_info);
8674 out_new_inode_args:
8675 btrfs_new_inode_args_destroy(&new_inode_args);
8676 out_inode:
8677 if (err)
8678 iput(inode);
8679 return err;
8680 }
8681
insert_prealloc_file_extent(struct btrfs_trans_handle * trans_in,struct btrfs_inode * inode,struct btrfs_key * ins,u64 file_offset)8682 static struct btrfs_trans_handle *insert_prealloc_file_extent(
8683 struct btrfs_trans_handle *trans_in,
8684 struct btrfs_inode *inode,
8685 struct btrfs_key *ins,
8686 u64 file_offset)
8687 {
8688 struct btrfs_file_extent_item stack_fi;
8689 struct btrfs_replace_extent_info extent_info;
8690 struct btrfs_trans_handle *trans = trans_in;
8691 struct btrfs_path *path;
8692 u64 start = ins->objectid;
8693 u64 len = ins->offset;
8694 u64 qgroup_released = 0;
8695 int ret;
8696
8697 memset(&stack_fi, 0, sizeof(stack_fi));
8698
8699 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC);
8700 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start);
8701 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len);
8702 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len);
8703 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len);
8704 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE);
8705 /* Encryption and other encoding is reserved and all 0 */
8706
8707 ret = btrfs_qgroup_release_data(inode, file_offset, len, &qgroup_released);
8708 if (ret < 0)
8709 return ERR_PTR(ret);
8710
8711 if (trans) {
8712 ret = insert_reserved_file_extent(trans, inode,
8713 file_offset, &stack_fi,
8714 true, qgroup_released);
8715 if (ret)
8716 goto free_qgroup;
8717 return trans;
8718 }
8719
8720 extent_info.disk_offset = start;
8721 extent_info.disk_len = len;
8722 extent_info.data_offset = 0;
8723 extent_info.data_len = len;
8724 extent_info.file_offset = file_offset;
8725 extent_info.extent_buf = (char *)&stack_fi;
8726 extent_info.is_new_extent = true;
8727 extent_info.update_times = true;
8728 extent_info.qgroup_reserved = qgroup_released;
8729 extent_info.insertions = 0;
8730
8731 path = btrfs_alloc_path();
8732 if (!path) {
8733 ret = -ENOMEM;
8734 goto free_qgroup;
8735 }
8736
8737 ret = btrfs_replace_file_extents(inode, path, file_offset,
8738 file_offset + len - 1, &extent_info,
8739 &trans);
8740 btrfs_free_path(path);
8741 if (ret)
8742 goto free_qgroup;
8743 return trans;
8744
8745 free_qgroup:
8746 /*
8747 * We have released qgroup data range at the beginning of the function,
8748 * and normally qgroup_released bytes will be freed when committing
8749 * transaction.
8750 * But if we error out early, we have to free what we have released
8751 * or we leak qgroup data reservation.
8752 */
8753 btrfs_qgroup_free_refroot(inode->root->fs_info,
8754 btrfs_root_id(inode->root), qgroup_released,
8755 BTRFS_QGROUP_RSV_DATA);
8756 return ERR_PTR(ret);
8757 }
8758
__btrfs_prealloc_file_range(struct inode * inode,int mode,u64 start,u64 num_bytes,u64 min_size,loff_t actual_len,u64 * alloc_hint,struct btrfs_trans_handle * trans)8759 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
8760 u64 start, u64 num_bytes, u64 min_size,
8761 loff_t actual_len, u64 *alloc_hint,
8762 struct btrfs_trans_handle *trans)
8763 {
8764 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
8765 struct extent_map *em;
8766 struct btrfs_root *root = BTRFS_I(inode)->root;
8767 struct btrfs_key ins;
8768 u64 cur_offset = start;
8769 u64 clear_offset = start;
8770 u64 i_size;
8771 u64 cur_bytes;
8772 u64 last_alloc = (u64)-1;
8773 int ret = 0;
8774 bool own_trans = true;
8775 u64 end = start + num_bytes - 1;
8776
8777 if (trans)
8778 own_trans = false;
8779 while (num_bytes > 0) {
8780 cur_bytes = min_t(u64, num_bytes, SZ_256M);
8781 cur_bytes = max(cur_bytes, min_size);
8782 /*
8783 * If we are severely fragmented we could end up with really
8784 * small allocations, so if the allocator is returning small
8785 * chunks lets make its job easier by only searching for those
8786 * sized chunks.
8787 */
8788 cur_bytes = min(cur_bytes, last_alloc);
8789 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
8790 min_size, 0, *alloc_hint, &ins, 1, 0);
8791 if (ret)
8792 break;
8793
8794 /*
8795 * We've reserved this space, and thus converted it from
8796 * ->bytes_may_use to ->bytes_reserved. Any error that happens
8797 * from here on out we will only need to clear our reservation
8798 * for the remaining unreserved area, so advance our
8799 * clear_offset by our extent size.
8800 */
8801 clear_offset += ins.offset;
8802
8803 last_alloc = ins.offset;
8804 trans = insert_prealloc_file_extent(trans, BTRFS_I(inode),
8805 &ins, cur_offset);
8806 /*
8807 * Now that we inserted the prealloc extent we can finally
8808 * decrement the number of reservations in the block group.
8809 * If we did it before, we could race with relocation and have
8810 * relocation miss the reserved extent, making it fail later.
8811 */
8812 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
8813 if (IS_ERR(trans)) {
8814 ret = PTR_ERR(trans);
8815 btrfs_free_reserved_extent(fs_info, ins.objectid,
8816 ins.offset, 0);
8817 break;
8818 }
8819
8820 em = alloc_extent_map();
8821 if (!em) {
8822 btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset,
8823 cur_offset + ins.offset - 1, false);
8824 btrfs_set_inode_full_sync(BTRFS_I(inode));
8825 goto next;
8826 }
8827
8828 em->start = cur_offset;
8829 em->len = ins.offset;
8830 em->disk_bytenr = ins.objectid;
8831 em->offset = 0;
8832 em->disk_num_bytes = ins.offset;
8833 em->ram_bytes = ins.offset;
8834 em->flags |= EXTENT_FLAG_PREALLOC;
8835 em->generation = trans->transid;
8836
8837 ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true);
8838 free_extent_map(em);
8839 next:
8840 num_bytes -= ins.offset;
8841 cur_offset += ins.offset;
8842 *alloc_hint = ins.objectid + ins.offset;
8843
8844 inode_inc_iversion(inode);
8845 inode_set_ctime_current(inode);
8846 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
8847 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
8848 (actual_len > inode->i_size) &&
8849 (cur_offset > inode->i_size)) {
8850 if (cur_offset > actual_len)
8851 i_size = actual_len;
8852 else
8853 i_size = cur_offset;
8854 i_size_write(inode, i_size);
8855 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
8856 }
8857
8858 ret = btrfs_update_inode(trans, BTRFS_I(inode));
8859
8860 if (ret) {
8861 btrfs_abort_transaction(trans, ret);
8862 if (own_trans)
8863 btrfs_end_transaction(trans);
8864 break;
8865 }
8866
8867 if (own_trans) {
8868 btrfs_end_transaction(trans);
8869 trans = NULL;
8870 }
8871 }
8872 if (clear_offset < end)
8873 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,
8874 end - clear_offset + 1);
8875 return ret;
8876 }
8877
btrfs_prealloc_file_range(struct inode * inode,int mode,u64 start,u64 num_bytes,u64 min_size,loff_t actual_len,u64 * alloc_hint)8878 int btrfs_prealloc_file_range(struct inode *inode, int mode,
8879 u64 start, u64 num_bytes, u64 min_size,
8880 loff_t actual_len, u64 *alloc_hint)
8881 {
8882 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
8883 min_size, actual_len, alloc_hint,
8884 NULL);
8885 }
8886
btrfs_prealloc_file_range_trans(struct inode * inode,struct btrfs_trans_handle * trans,int mode,u64 start,u64 num_bytes,u64 min_size,loff_t actual_len,u64 * alloc_hint)8887 int btrfs_prealloc_file_range_trans(struct inode *inode,
8888 struct btrfs_trans_handle *trans, int mode,
8889 u64 start, u64 num_bytes, u64 min_size,
8890 loff_t actual_len, u64 *alloc_hint)
8891 {
8892 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
8893 min_size, actual_len, alloc_hint, trans);
8894 }
8895
btrfs_permission(struct mnt_idmap * idmap,struct inode * inode,int mask)8896 static int btrfs_permission(struct mnt_idmap *idmap,
8897 struct inode *inode, int mask)
8898 {
8899 struct btrfs_root *root = BTRFS_I(inode)->root;
8900 umode_t mode = inode->i_mode;
8901
8902 if (mask & MAY_WRITE &&
8903 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
8904 if (btrfs_root_readonly(root))
8905 return -EROFS;
8906 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
8907 return -EACCES;
8908 }
8909 return generic_permission(idmap, inode, mask);
8910 }
8911
btrfs_tmpfile(struct mnt_idmap * idmap,struct inode * dir,struct file * file,umode_t mode)8912 static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
8913 struct file *file, umode_t mode)
8914 {
8915 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
8916 struct btrfs_trans_handle *trans;
8917 struct btrfs_root *root = BTRFS_I(dir)->root;
8918 struct inode *inode;
8919 struct btrfs_new_inode_args new_inode_args = {
8920 .dir = dir,
8921 .dentry = file->f_path.dentry,
8922 .orphan = true,
8923 };
8924 unsigned int trans_num_items;
8925 int ret;
8926
8927 inode = new_inode(dir->i_sb);
8928 if (!inode)
8929 return -ENOMEM;
8930 inode_init_owner(idmap, inode, dir, mode);
8931 inode->i_fop = &btrfs_file_operations;
8932 inode->i_op = &btrfs_file_inode_operations;
8933 inode->i_mapping->a_ops = &btrfs_aops;
8934
8935 new_inode_args.inode = inode;
8936 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
8937 if (ret)
8938 goto out_inode;
8939
8940 trans = btrfs_start_transaction(root, trans_num_items);
8941 if (IS_ERR(trans)) {
8942 ret = PTR_ERR(trans);
8943 goto out_new_inode_args;
8944 }
8945
8946 ret = btrfs_create_new_inode(trans, &new_inode_args);
8947
8948 /*
8949 * We set number of links to 0 in btrfs_create_new_inode(), and here we
8950 * set it to 1 because d_tmpfile() will issue a warning if the count is
8951 * 0, through:
8952 *
8953 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
8954 */
8955 set_nlink(inode, 1);
8956
8957 if (!ret) {
8958 d_tmpfile(file, inode);
8959 unlock_new_inode(inode);
8960 mark_inode_dirty(inode);
8961 }
8962
8963 btrfs_end_transaction(trans);
8964 btrfs_btree_balance_dirty(fs_info);
8965 out_new_inode_args:
8966 btrfs_new_inode_args_destroy(&new_inode_args);
8967 out_inode:
8968 if (ret)
8969 iput(inode);
8970 return finish_open_simple(file, ret);
8971 }
8972
btrfs_set_range_writeback(struct btrfs_inode * inode,u64 start,u64 end)8973 void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end)
8974 {
8975 struct btrfs_fs_info *fs_info = inode->root->fs_info;
8976 unsigned long index = start >> PAGE_SHIFT;
8977 unsigned long end_index = end >> PAGE_SHIFT;
8978 struct folio *folio;
8979 u32 len;
8980
8981 ASSERT(end + 1 - start <= U32_MAX);
8982 len = end + 1 - start;
8983 while (index <= end_index) {
8984 folio = __filemap_get_folio(inode->vfs_inode.i_mapping, index, 0, 0);
8985 ASSERT(!IS_ERR(folio)); /* folios should be in the extent_io_tree */
8986
8987 /* This is for data, which doesn't yet support larger folio. */
8988 ASSERT(folio_order(folio) == 0);
8989 btrfs_folio_set_writeback(fs_info, folio, start, len);
8990 folio_put(folio);
8991 index++;
8992 }
8993 }
8994
btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info * fs_info,int compress_type)8995 int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
8996 int compress_type)
8997 {
8998 switch (compress_type) {
8999 case BTRFS_COMPRESS_NONE:
9000 return BTRFS_ENCODED_IO_COMPRESSION_NONE;
9001 case BTRFS_COMPRESS_ZLIB:
9002 return BTRFS_ENCODED_IO_COMPRESSION_ZLIB;
9003 case BTRFS_COMPRESS_LZO:
9004 /*
9005 * The LZO format depends on the sector size. 64K is the maximum
9006 * sector size that we support.
9007 */
9008 if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K)
9009 return -EINVAL;
9010 return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K +
9011 (fs_info->sectorsize_bits - 12);
9012 case BTRFS_COMPRESS_ZSTD:
9013 return BTRFS_ENCODED_IO_COMPRESSION_ZSTD;
9014 default:
9015 return -EUCLEAN;
9016 }
9017 }
9018
btrfs_encoded_read_inline(struct kiocb * iocb,struct iov_iter * iter,u64 start,u64 lockend,struct extent_state ** cached_state,u64 extent_start,size_t count,struct btrfs_ioctl_encoded_io_args * encoded,bool * unlocked)9019 static ssize_t btrfs_encoded_read_inline(
9020 struct kiocb *iocb,
9021 struct iov_iter *iter, u64 start,
9022 u64 lockend,
9023 struct extent_state **cached_state,
9024 u64 extent_start, size_t count,
9025 struct btrfs_ioctl_encoded_io_args *encoded,
9026 bool *unlocked)
9027 {
9028 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9029 struct btrfs_root *root = inode->root;
9030 struct btrfs_fs_info *fs_info = root->fs_info;
9031 struct extent_io_tree *io_tree = &inode->io_tree;
9032 struct btrfs_path *path;
9033 struct extent_buffer *leaf;
9034 struct btrfs_file_extent_item *item;
9035 u64 ram_bytes;
9036 unsigned long ptr;
9037 void *tmp;
9038 ssize_t ret;
9039
9040 path = btrfs_alloc_path();
9041 if (!path) {
9042 ret = -ENOMEM;
9043 goto out;
9044 }
9045 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
9046 extent_start, 0);
9047 if (ret) {
9048 if (ret > 0) {
9049 /* The extent item disappeared? */
9050 ret = -EIO;
9051 }
9052 goto out;
9053 }
9054 leaf = path->nodes[0];
9055 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
9056
9057 ram_bytes = btrfs_file_extent_ram_bytes(leaf, item);
9058 ptr = btrfs_file_extent_inline_start(item);
9059
9060 encoded->len = min_t(u64, extent_start + ram_bytes,
9061 inode->vfs_inode.i_size) - iocb->ki_pos;
9062 ret = btrfs_encoded_io_compression_from_extent(fs_info,
9063 btrfs_file_extent_compression(leaf, item));
9064 if (ret < 0)
9065 goto out;
9066 encoded->compression = ret;
9067 if (encoded->compression) {
9068 size_t inline_size;
9069
9070 inline_size = btrfs_file_extent_inline_item_len(leaf,
9071 path->slots[0]);
9072 if (inline_size > count) {
9073 ret = -ENOBUFS;
9074 goto out;
9075 }
9076 count = inline_size;
9077 encoded->unencoded_len = ram_bytes;
9078 encoded->unencoded_offset = iocb->ki_pos - extent_start;
9079 } else {
9080 count = min_t(u64, count, encoded->len);
9081 encoded->len = count;
9082 encoded->unencoded_len = count;
9083 ptr += iocb->ki_pos - extent_start;
9084 }
9085
9086 tmp = kmalloc(count, GFP_NOFS);
9087 if (!tmp) {
9088 ret = -ENOMEM;
9089 goto out;
9090 }
9091 read_extent_buffer(leaf, tmp, ptr, count);
9092 btrfs_release_path(path);
9093 unlock_extent(io_tree, start, lockend, cached_state);
9094 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9095 *unlocked = true;
9096
9097 ret = copy_to_iter(tmp, count, iter);
9098 if (ret != count)
9099 ret = -EFAULT;
9100 kfree(tmp);
9101 out:
9102 btrfs_free_path(path);
9103 return ret;
9104 }
9105
9106 struct btrfs_encoded_read_private {
9107 wait_queue_head_t wait;
9108 atomic_t pending;
9109 blk_status_t status;
9110 };
9111
btrfs_encoded_read_endio(struct btrfs_bio * bbio)9112 static void btrfs_encoded_read_endio(struct btrfs_bio *bbio)
9113 {
9114 struct btrfs_encoded_read_private *priv = bbio->private;
9115
9116 if (bbio->bio.bi_status) {
9117 /*
9118 * The memory barrier implied by the atomic_dec_return() here
9119 * pairs with the memory barrier implied by the
9120 * atomic_dec_return() or io_wait_event() in
9121 * btrfs_encoded_read_regular_fill_pages() to ensure that this
9122 * write is observed before the load of status in
9123 * btrfs_encoded_read_regular_fill_pages().
9124 */
9125 WRITE_ONCE(priv->status, bbio->bio.bi_status);
9126 }
9127 if (!atomic_dec_return(&priv->pending))
9128 wake_up(&priv->wait);
9129 bio_put(&bbio->bio);
9130 }
9131
btrfs_encoded_read_regular_fill_pages(struct btrfs_inode * inode,u64 file_offset,u64 disk_bytenr,u64 disk_io_size,struct page ** pages)9132 int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
9133 u64 file_offset, u64 disk_bytenr,
9134 u64 disk_io_size, struct page **pages)
9135 {
9136 struct btrfs_fs_info *fs_info = inode->root->fs_info;
9137 struct btrfs_encoded_read_private priv = {
9138 .pending = ATOMIC_INIT(1),
9139 };
9140 unsigned long i = 0;
9141 struct btrfs_bio *bbio;
9142
9143 init_waitqueue_head(&priv.wait);
9144
9145 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,
9146 btrfs_encoded_read_endio, &priv);
9147 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9148 bbio->inode = inode;
9149
9150 do {
9151 size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE);
9152
9153 if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) {
9154 atomic_inc(&priv.pending);
9155 btrfs_submit_bbio(bbio, 0);
9156
9157 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,
9158 btrfs_encoded_read_endio, &priv);
9159 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9160 bbio->inode = inode;
9161 continue;
9162 }
9163
9164 i++;
9165 disk_bytenr += bytes;
9166 disk_io_size -= bytes;
9167 } while (disk_io_size);
9168
9169 atomic_inc(&priv.pending);
9170 btrfs_submit_bbio(bbio, 0);
9171
9172 if (atomic_dec_return(&priv.pending))
9173 io_wait_event(priv.wait, !atomic_read(&priv.pending));
9174 /* See btrfs_encoded_read_endio() for ordering. */
9175 return blk_status_to_errno(READ_ONCE(priv.status));
9176 }
9177
btrfs_encoded_read_regular(struct kiocb * iocb,struct iov_iter * iter,u64 start,u64 lockend,struct extent_state ** cached_state,u64 disk_bytenr,u64 disk_io_size,size_t count,bool compressed,bool * unlocked)9178 static ssize_t btrfs_encoded_read_regular(struct kiocb *iocb,
9179 struct iov_iter *iter,
9180 u64 start, u64 lockend,
9181 struct extent_state **cached_state,
9182 u64 disk_bytenr, u64 disk_io_size,
9183 size_t count, bool compressed,
9184 bool *unlocked)
9185 {
9186 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9187 struct extent_io_tree *io_tree = &inode->io_tree;
9188 struct page **pages;
9189 unsigned long nr_pages, i;
9190 u64 cur;
9191 size_t page_offset;
9192 ssize_t ret;
9193
9194 nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE);
9195 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
9196 if (!pages)
9197 return -ENOMEM;
9198 ret = btrfs_alloc_page_array(nr_pages, pages, false);
9199 if (ret) {
9200 ret = -ENOMEM;
9201 goto out;
9202 }
9203
9204 ret = btrfs_encoded_read_regular_fill_pages(inode, start, disk_bytenr,
9205 disk_io_size, pages);
9206 if (ret)
9207 goto out;
9208
9209 unlock_extent(io_tree, start, lockend, cached_state);
9210 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9211 *unlocked = true;
9212
9213 if (compressed) {
9214 i = 0;
9215 page_offset = 0;
9216 } else {
9217 i = (iocb->ki_pos - start) >> PAGE_SHIFT;
9218 page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1);
9219 }
9220 cur = 0;
9221 while (cur < count) {
9222 size_t bytes = min_t(size_t, count - cur,
9223 PAGE_SIZE - page_offset);
9224
9225 if (copy_page_to_iter(pages[i], page_offset, bytes,
9226 iter) != bytes) {
9227 ret = -EFAULT;
9228 goto out;
9229 }
9230 i++;
9231 cur += bytes;
9232 page_offset = 0;
9233 }
9234 ret = count;
9235 out:
9236 for (i = 0; i < nr_pages; i++) {
9237 if (pages[i])
9238 __free_page(pages[i]);
9239 }
9240 kfree(pages);
9241 return ret;
9242 }
9243
btrfs_encoded_read(struct kiocb * iocb,struct iov_iter * iter,struct btrfs_ioctl_encoded_io_args * encoded)9244 ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter,
9245 struct btrfs_ioctl_encoded_io_args *encoded)
9246 {
9247 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9248 struct btrfs_fs_info *fs_info = inode->root->fs_info;
9249 struct extent_io_tree *io_tree = &inode->io_tree;
9250 ssize_t ret;
9251 size_t count = iov_iter_count(iter);
9252 u64 start, lockend, disk_bytenr, disk_io_size;
9253 struct extent_state *cached_state = NULL;
9254 struct extent_map *em;
9255 bool unlocked = false;
9256
9257 file_accessed(iocb->ki_filp);
9258
9259 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
9260
9261 if (iocb->ki_pos >= inode->vfs_inode.i_size) {
9262 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9263 return 0;
9264 }
9265 start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize);
9266 /*
9267 * We don't know how long the extent containing iocb->ki_pos is, but if
9268 * it's compressed we know that it won't be longer than this.
9269 */
9270 lockend = start + BTRFS_MAX_UNCOMPRESSED - 1;
9271
9272 for (;;) {
9273 struct btrfs_ordered_extent *ordered;
9274
9275 ret = btrfs_wait_ordered_range(inode, start,
9276 lockend - start + 1);
9277 if (ret)
9278 goto out_unlock_inode;
9279 lock_extent(io_tree, start, lockend, &cached_state);
9280 ordered = btrfs_lookup_ordered_range(inode, start,
9281 lockend - start + 1);
9282 if (!ordered)
9283 break;
9284 btrfs_put_ordered_extent(ordered);
9285 unlock_extent(io_tree, start, lockend, &cached_state);
9286 cond_resched();
9287 }
9288
9289 em = btrfs_get_extent(inode, NULL, start, lockend - start + 1);
9290 if (IS_ERR(em)) {
9291 ret = PTR_ERR(em);
9292 goto out_unlock_extent;
9293 }
9294
9295 if (em->disk_bytenr == EXTENT_MAP_INLINE) {
9296 u64 extent_start = em->start;
9297
9298 /*
9299 * For inline extents we get everything we need out of the
9300 * extent item.
9301 */
9302 free_extent_map(em);
9303 em = NULL;
9304 ret = btrfs_encoded_read_inline(iocb, iter, start, lockend,
9305 &cached_state, extent_start,
9306 count, encoded, &unlocked);
9307 goto out;
9308 }
9309
9310 /*
9311 * We only want to return up to EOF even if the extent extends beyond
9312 * that.
9313 */
9314 encoded->len = min_t(u64, extent_map_end(em),
9315 inode->vfs_inode.i_size) - iocb->ki_pos;
9316 if (em->disk_bytenr == EXTENT_MAP_HOLE ||
9317 (em->flags & EXTENT_FLAG_PREALLOC)) {
9318 disk_bytenr = EXTENT_MAP_HOLE;
9319 count = min_t(u64, count, encoded->len);
9320 encoded->len = count;
9321 encoded->unencoded_len = count;
9322 } else if (extent_map_is_compressed(em)) {
9323 disk_bytenr = em->disk_bytenr;
9324 /*
9325 * Bail if the buffer isn't large enough to return the whole
9326 * compressed extent.
9327 */
9328 if (em->disk_num_bytes > count) {
9329 ret = -ENOBUFS;
9330 goto out_em;
9331 }
9332 disk_io_size = em->disk_num_bytes;
9333 count = em->disk_num_bytes;
9334 encoded->unencoded_len = em->ram_bytes;
9335 encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset);
9336 ret = btrfs_encoded_io_compression_from_extent(fs_info,
9337 extent_map_compression(em));
9338 if (ret < 0)
9339 goto out_em;
9340 encoded->compression = ret;
9341 } else {
9342 disk_bytenr = extent_map_block_start(em) + (start - em->start);
9343 if (encoded->len > count)
9344 encoded->len = count;
9345 /*
9346 * Don't read beyond what we locked. This also limits the page
9347 * allocations that we'll do.
9348 */
9349 disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start;
9350 count = start + disk_io_size - iocb->ki_pos;
9351 encoded->len = count;
9352 encoded->unencoded_len = count;
9353 disk_io_size = ALIGN(disk_io_size, fs_info->sectorsize);
9354 }
9355 free_extent_map(em);
9356 em = NULL;
9357
9358 if (disk_bytenr == EXTENT_MAP_HOLE) {
9359 unlock_extent(io_tree, start, lockend, &cached_state);
9360 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9361 unlocked = true;
9362 ret = iov_iter_zero(count, iter);
9363 if (ret != count)
9364 ret = -EFAULT;
9365 } else {
9366 ret = btrfs_encoded_read_regular(iocb, iter, start, lockend,
9367 &cached_state, disk_bytenr,
9368 disk_io_size, count,
9369 encoded->compression,
9370 &unlocked);
9371 }
9372
9373 out:
9374 if (ret >= 0)
9375 iocb->ki_pos += encoded->len;
9376 out_em:
9377 free_extent_map(em);
9378 out_unlock_extent:
9379 if (!unlocked)
9380 unlock_extent(io_tree, start, lockend, &cached_state);
9381 out_unlock_inode:
9382 if (!unlocked)
9383 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9384 return ret;
9385 }
9386
btrfs_do_encoded_write(struct kiocb * iocb,struct iov_iter * from,const struct btrfs_ioctl_encoded_io_args * encoded)9387 ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
9388 const struct btrfs_ioctl_encoded_io_args *encoded)
9389 {
9390 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9391 struct btrfs_root *root = inode->root;
9392 struct btrfs_fs_info *fs_info = root->fs_info;
9393 struct extent_io_tree *io_tree = &inode->io_tree;
9394 struct extent_changeset *data_reserved = NULL;
9395 struct extent_state *cached_state = NULL;
9396 struct btrfs_ordered_extent *ordered;
9397 struct btrfs_file_extent file_extent;
9398 int compression;
9399 size_t orig_count;
9400 u64 start, end;
9401 u64 num_bytes, ram_bytes, disk_num_bytes;
9402 unsigned long nr_folios, i;
9403 struct folio **folios;
9404 struct btrfs_key ins;
9405 bool extent_reserved = false;
9406 struct extent_map *em;
9407 ssize_t ret;
9408
9409 switch (encoded->compression) {
9410 case BTRFS_ENCODED_IO_COMPRESSION_ZLIB:
9411 compression = BTRFS_COMPRESS_ZLIB;
9412 break;
9413 case BTRFS_ENCODED_IO_COMPRESSION_ZSTD:
9414 compression = BTRFS_COMPRESS_ZSTD;
9415 break;
9416 case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K:
9417 case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K:
9418 case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K:
9419 case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K:
9420 case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K:
9421 /* The sector size must match for LZO. */
9422 if (encoded->compression -
9423 BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 !=
9424 fs_info->sectorsize_bits)
9425 return -EINVAL;
9426 compression = BTRFS_COMPRESS_LZO;
9427 break;
9428 default:
9429 return -EINVAL;
9430 }
9431 if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE)
9432 return -EINVAL;
9433
9434 /*
9435 * Compressed extents should always have checksums, so error out if we
9436 * have a NOCOW file or inode was created while mounted with NODATASUM.
9437 */
9438 if (inode->flags & BTRFS_INODE_NODATASUM)
9439 return -EINVAL;
9440
9441 orig_count = iov_iter_count(from);
9442
9443 /* The extent size must be sane. */
9444 if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED ||
9445 orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0)
9446 return -EINVAL;
9447
9448 /*
9449 * The compressed data must be smaller than the decompressed data.
9450 *
9451 * It's of course possible for data to compress to larger or the same
9452 * size, but the buffered I/O path falls back to no compression for such
9453 * data, and we don't want to break any assumptions by creating these
9454 * extents.
9455 *
9456 * Note that this is less strict than the current check we have that the
9457 * compressed data must be at least one sector smaller than the
9458 * decompressed data. We only want to enforce the weaker requirement
9459 * from old kernels that it is at least one byte smaller.
9460 */
9461 if (orig_count >= encoded->unencoded_len)
9462 return -EINVAL;
9463
9464 /* The extent must start on a sector boundary. */
9465 start = iocb->ki_pos;
9466 if (!IS_ALIGNED(start, fs_info->sectorsize))
9467 return -EINVAL;
9468
9469 /*
9470 * The extent must end on a sector boundary. However, we allow a write
9471 * which ends at or extends i_size to have an unaligned length; we round
9472 * up the extent size and set i_size to the unaligned end.
9473 */
9474 if (start + encoded->len < inode->vfs_inode.i_size &&
9475 !IS_ALIGNED(start + encoded->len, fs_info->sectorsize))
9476 return -EINVAL;
9477
9478 /* Finally, the offset in the unencoded data must be sector-aligned. */
9479 if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize))
9480 return -EINVAL;
9481
9482 num_bytes = ALIGN(encoded->len, fs_info->sectorsize);
9483 ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize);
9484 end = start + num_bytes - 1;
9485
9486 /*
9487 * If the extent cannot be inline, the compressed data on disk must be
9488 * sector-aligned. For convenience, we extend it with zeroes if it
9489 * isn't.
9490 */
9491 disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize);
9492 nr_folios = DIV_ROUND_UP(disk_num_bytes, PAGE_SIZE);
9493 folios = kvcalloc(nr_folios, sizeof(struct page *), GFP_KERNEL_ACCOUNT);
9494 if (!folios)
9495 return -ENOMEM;
9496 for (i = 0; i < nr_folios; i++) {
9497 size_t bytes = min_t(size_t, PAGE_SIZE, iov_iter_count(from));
9498 char *kaddr;
9499
9500 folios[i] = folio_alloc(GFP_KERNEL_ACCOUNT, 0);
9501 if (!folios[i]) {
9502 ret = -ENOMEM;
9503 goto out_folios;
9504 }
9505 kaddr = kmap_local_folio(folios[i], 0);
9506 if (copy_from_iter(kaddr, bytes, from) != bytes) {
9507 kunmap_local(kaddr);
9508 ret = -EFAULT;
9509 goto out_folios;
9510 }
9511 if (bytes < PAGE_SIZE)
9512 memset(kaddr + bytes, 0, PAGE_SIZE - bytes);
9513 kunmap_local(kaddr);
9514 }
9515
9516 for (;;) {
9517 struct btrfs_ordered_extent *ordered;
9518
9519 ret = btrfs_wait_ordered_range(inode, start, num_bytes);
9520 if (ret)
9521 goto out_folios;
9522 ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping,
9523 start >> PAGE_SHIFT,
9524 end >> PAGE_SHIFT);
9525 if (ret)
9526 goto out_folios;
9527 lock_extent(io_tree, start, end, &cached_state);
9528 ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);
9529 if (!ordered &&
9530 !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end))
9531 break;
9532 if (ordered)
9533 btrfs_put_ordered_extent(ordered);
9534 unlock_extent(io_tree, start, end, &cached_state);
9535 cond_resched();
9536 }
9537
9538 /*
9539 * We don't use the higher-level delalloc space functions because our
9540 * num_bytes and disk_num_bytes are different.
9541 */
9542 ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes);
9543 if (ret)
9544 goto out_unlock;
9545 ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes);
9546 if (ret)
9547 goto out_free_data_space;
9548 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes,
9549 false);
9550 if (ret)
9551 goto out_qgroup_free_data;
9552
9553 /* Try an inline extent first. */
9554 if (encoded->unencoded_len == encoded->len &&
9555 encoded->unencoded_offset == 0 &&
9556 can_cow_file_range_inline(inode, start, encoded->len, orig_count)) {
9557 ret = __cow_file_range_inline(inode, start, encoded->len,
9558 orig_count, compression, folios[0],
9559 true);
9560 if (ret <= 0) {
9561 if (ret == 0)
9562 ret = orig_count;
9563 goto out_delalloc_release;
9564 }
9565 }
9566
9567 ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes,
9568 disk_num_bytes, 0, 0, &ins, 1, 1);
9569 if (ret)
9570 goto out_delalloc_release;
9571 extent_reserved = true;
9572
9573 file_extent.disk_bytenr = ins.objectid;
9574 file_extent.disk_num_bytes = ins.offset;
9575 file_extent.num_bytes = num_bytes;
9576 file_extent.ram_bytes = ram_bytes;
9577 file_extent.offset = encoded->unencoded_offset;
9578 file_extent.compression = compression;
9579 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
9580 if (IS_ERR(em)) {
9581 ret = PTR_ERR(em);
9582 goto out_free_reserved;
9583 }
9584 free_extent_map(em);
9585
9586 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
9587 (1 << BTRFS_ORDERED_ENCODED) |
9588 (1 << BTRFS_ORDERED_COMPRESSED));
9589 if (IS_ERR(ordered)) {
9590 btrfs_drop_extent_map_range(inode, start, end, false);
9591 ret = PTR_ERR(ordered);
9592 goto out_free_reserved;
9593 }
9594 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
9595
9596 if (start + encoded->len > inode->vfs_inode.i_size)
9597 i_size_write(&inode->vfs_inode, start + encoded->len);
9598
9599 unlock_extent(io_tree, start, end, &cached_state);
9600
9601 btrfs_delalloc_release_extents(inode, num_bytes);
9602
9603 btrfs_submit_compressed_write(ordered, folios, nr_folios, 0, false);
9604 ret = orig_count;
9605 goto out;
9606
9607 out_free_reserved:
9608 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
9609 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
9610 out_delalloc_release:
9611 btrfs_delalloc_release_extents(inode, num_bytes);
9612 btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0);
9613 out_qgroup_free_data:
9614 if (ret < 0)
9615 btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes, NULL);
9616 out_free_data_space:
9617 /*
9618 * If btrfs_reserve_extent() succeeded, then we already decremented
9619 * bytes_may_use.
9620 */
9621 if (!extent_reserved)
9622 btrfs_free_reserved_data_space_noquota(fs_info, disk_num_bytes);
9623 out_unlock:
9624 unlock_extent(io_tree, start, end, &cached_state);
9625 out_folios:
9626 for (i = 0; i < nr_folios; i++) {
9627 if (folios[i])
9628 folio_put(folios[i]);
9629 }
9630 kvfree(folios);
9631 out:
9632 if (ret >= 0)
9633 iocb->ki_pos += encoded->len;
9634 return ret;
9635 }
9636
9637 #ifdef CONFIG_SWAP
9638 /*
9639 * Add an entry indicating a block group or device which is pinned by a
9640 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
9641 * negative errno on failure.
9642 */
btrfs_add_swapfile_pin(struct inode * inode,void * ptr,bool is_block_group)9643 static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr,
9644 bool is_block_group)
9645 {
9646 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
9647 struct btrfs_swapfile_pin *sp, *entry;
9648 struct rb_node **p;
9649 struct rb_node *parent = NULL;
9650
9651 sp = kmalloc(sizeof(*sp), GFP_NOFS);
9652 if (!sp)
9653 return -ENOMEM;
9654 sp->ptr = ptr;
9655 sp->inode = inode;
9656 sp->is_block_group = is_block_group;
9657 sp->bg_extent_count = 1;
9658
9659 spin_lock(&fs_info->swapfile_pins_lock);
9660 p = &fs_info->swapfile_pins.rb_node;
9661 while (*p) {
9662 parent = *p;
9663 entry = rb_entry(parent, struct btrfs_swapfile_pin, node);
9664 if (sp->ptr < entry->ptr ||
9665 (sp->ptr == entry->ptr && sp->inode < entry->inode)) {
9666 p = &(*p)->rb_left;
9667 } else if (sp->ptr > entry->ptr ||
9668 (sp->ptr == entry->ptr && sp->inode > entry->inode)) {
9669 p = &(*p)->rb_right;
9670 } else {
9671 if (is_block_group)
9672 entry->bg_extent_count++;
9673 spin_unlock(&fs_info->swapfile_pins_lock);
9674 kfree(sp);
9675 return 1;
9676 }
9677 }
9678 rb_link_node(&sp->node, parent, p);
9679 rb_insert_color(&sp->node, &fs_info->swapfile_pins);
9680 spin_unlock(&fs_info->swapfile_pins_lock);
9681 return 0;
9682 }
9683
9684 /* Free all of the entries pinned by this swapfile. */
btrfs_free_swapfile_pins(struct inode * inode)9685 static void btrfs_free_swapfile_pins(struct inode *inode)
9686 {
9687 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
9688 struct btrfs_swapfile_pin *sp;
9689 struct rb_node *node, *next;
9690
9691 spin_lock(&fs_info->swapfile_pins_lock);
9692 node = rb_first(&fs_info->swapfile_pins);
9693 while (node) {
9694 next = rb_next(node);
9695 sp = rb_entry(node, struct btrfs_swapfile_pin, node);
9696 if (sp->inode == inode) {
9697 rb_erase(&sp->node, &fs_info->swapfile_pins);
9698 if (sp->is_block_group) {
9699 btrfs_dec_block_group_swap_extents(sp->ptr,
9700 sp->bg_extent_count);
9701 btrfs_put_block_group(sp->ptr);
9702 }
9703 kfree(sp);
9704 }
9705 node = next;
9706 }
9707 spin_unlock(&fs_info->swapfile_pins_lock);
9708 }
9709
9710 struct btrfs_swap_info {
9711 u64 start;
9712 u64 block_start;
9713 u64 block_len;
9714 u64 lowest_ppage;
9715 u64 highest_ppage;
9716 unsigned long nr_pages;
9717 int nr_extents;
9718 };
9719
btrfs_add_swap_extent(struct swap_info_struct * sis,struct btrfs_swap_info * bsi)9720 static int btrfs_add_swap_extent(struct swap_info_struct *sis,
9721 struct btrfs_swap_info *bsi)
9722 {
9723 unsigned long nr_pages;
9724 unsigned long max_pages;
9725 u64 first_ppage, first_ppage_reported, next_ppage;
9726 int ret;
9727
9728 /*
9729 * Our swapfile may have had its size extended after the swap header was
9730 * written. In that case activating the swapfile should not go beyond
9731 * the max size set in the swap header.
9732 */
9733 if (bsi->nr_pages >= sis->max)
9734 return 0;
9735
9736 max_pages = sis->max - bsi->nr_pages;
9737 first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT;
9738 next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT;
9739
9740 if (first_ppage >= next_ppage)
9741 return 0;
9742 nr_pages = next_ppage - first_ppage;
9743 nr_pages = min(nr_pages, max_pages);
9744
9745 first_ppage_reported = first_ppage;
9746 if (bsi->start == 0)
9747 first_ppage_reported++;
9748 if (bsi->lowest_ppage > first_ppage_reported)
9749 bsi->lowest_ppage = first_ppage_reported;
9750 if (bsi->highest_ppage < (next_ppage - 1))
9751 bsi->highest_ppage = next_ppage - 1;
9752
9753 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage);
9754 if (ret < 0)
9755 return ret;
9756 bsi->nr_extents += ret;
9757 bsi->nr_pages += nr_pages;
9758 return 0;
9759 }
9760
btrfs_swap_deactivate(struct file * file)9761 static void btrfs_swap_deactivate(struct file *file)
9762 {
9763 struct inode *inode = file_inode(file);
9764
9765 btrfs_free_swapfile_pins(inode);
9766 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles);
9767 }
9768
btrfs_swap_activate(struct swap_info_struct * sis,struct file * file,sector_t * span)9769 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
9770 sector_t *span)
9771 {
9772 struct inode *inode = file_inode(file);
9773 struct btrfs_root *root = BTRFS_I(inode)->root;
9774 struct btrfs_fs_info *fs_info = root->fs_info;
9775 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
9776 struct extent_state *cached_state = NULL;
9777 struct extent_map *em = NULL;
9778 struct btrfs_chunk_map *map = NULL;
9779 struct btrfs_device *device = NULL;
9780 struct btrfs_swap_info bsi = {
9781 .lowest_ppage = (sector_t)-1ULL,
9782 };
9783 int ret = 0;
9784 u64 isize;
9785 u64 start;
9786
9787 /*
9788 * If the swap file was just created, make sure delalloc is done. If the
9789 * file changes again after this, the user is doing something stupid and
9790 * we don't really care.
9791 */
9792 ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);
9793 if (ret)
9794 return ret;
9795
9796 /*
9797 * The inode is locked, so these flags won't change after we check them.
9798 */
9799 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {
9800 btrfs_warn(fs_info, "swapfile must not be compressed");
9801 return -EINVAL;
9802 }
9803 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {
9804 btrfs_warn(fs_info, "swapfile must not be copy-on-write");
9805 return -EINVAL;
9806 }
9807 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
9808 btrfs_warn(fs_info, "swapfile must not be checksummed");
9809 return -EINVAL;
9810 }
9811
9812 /*
9813 * Balance or device remove/replace/resize can move stuff around from
9814 * under us. The exclop protection makes sure they aren't running/won't
9815 * run concurrently while we are mapping the swap extents, and
9816 * fs_info->swapfile_pins prevents them from running while the swap
9817 * file is active and moving the extents. Note that this also prevents
9818 * a concurrent device add which isn't actually necessary, but it's not
9819 * really worth the trouble to allow it.
9820 */
9821 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) {
9822 btrfs_warn(fs_info,
9823 "cannot activate swapfile while exclusive operation is running");
9824 return -EBUSY;
9825 }
9826
9827 /*
9828 * Prevent snapshot creation while we are activating the swap file.
9829 * We do not want to race with snapshot creation. If snapshot creation
9830 * already started before we bumped nr_swapfiles from 0 to 1 and
9831 * completes before the first write into the swap file after it is
9832 * activated, than that write would fallback to COW.
9833 */
9834 if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) {
9835 btrfs_exclop_finish(fs_info);
9836 btrfs_warn(fs_info,
9837 "cannot activate swapfile because snapshot creation is in progress");
9838 return -EINVAL;
9839 }
9840 /*
9841 * Snapshots can create extents which require COW even if NODATACOW is
9842 * set. We use this counter to prevent snapshots. We must increment it
9843 * before walking the extents because we don't want a concurrent
9844 * snapshot to run after we've already checked the extents.
9845 *
9846 * It is possible that subvolume is marked for deletion but still not
9847 * removed yet. To prevent this race, we check the root status before
9848 * activating the swapfile.
9849 */
9850 spin_lock(&root->root_item_lock);
9851 if (btrfs_root_dead(root)) {
9852 spin_unlock(&root->root_item_lock);
9853
9854 btrfs_exclop_finish(fs_info);
9855 btrfs_warn(fs_info,
9856 "cannot activate swapfile because subvolume %llu is being deleted",
9857 btrfs_root_id(root));
9858 return -EPERM;
9859 }
9860 atomic_inc(&root->nr_swapfiles);
9861 spin_unlock(&root->root_item_lock);
9862
9863 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
9864
9865 lock_extent(io_tree, 0, isize - 1, &cached_state);
9866 start = 0;
9867 while (start < isize) {
9868 u64 logical_block_start, physical_block_start;
9869 struct btrfs_block_group *bg;
9870 u64 len = isize - start;
9871
9872 em = btrfs_get_extent(BTRFS_I(inode), NULL, start, len);
9873 if (IS_ERR(em)) {
9874 ret = PTR_ERR(em);
9875 goto out;
9876 }
9877
9878 if (em->disk_bytenr == EXTENT_MAP_HOLE) {
9879 btrfs_warn(fs_info, "swapfile must not have holes");
9880 ret = -EINVAL;
9881 goto out;
9882 }
9883 if (em->disk_bytenr == EXTENT_MAP_INLINE) {
9884 /*
9885 * It's unlikely we'll ever actually find ourselves
9886 * here, as a file small enough to fit inline won't be
9887 * big enough to store more than the swap header, but in
9888 * case something changes in the future, let's catch it
9889 * here rather than later.
9890 */
9891 btrfs_warn(fs_info, "swapfile must not be inline");
9892 ret = -EINVAL;
9893 goto out;
9894 }
9895 if (extent_map_is_compressed(em)) {
9896 btrfs_warn(fs_info, "swapfile must not be compressed");
9897 ret = -EINVAL;
9898 goto out;
9899 }
9900
9901 logical_block_start = extent_map_block_start(em) + (start - em->start);
9902 len = min(len, em->len - (start - em->start));
9903 free_extent_map(em);
9904 em = NULL;
9905
9906 ret = can_nocow_extent(inode, start, &len, NULL, false, true);
9907 if (ret < 0) {
9908 goto out;
9909 } else if (ret) {
9910 ret = 0;
9911 } else {
9912 btrfs_warn(fs_info,
9913 "swapfile must not be copy-on-write");
9914 ret = -EINVAL;
9915 goto out;
9916 }
9917
9918 map = btrfs_get_chunk_map(fs_info, logical_block_start, len);
9919 if (IS_ERR(map)) {
9920 ret = PTR_ERR(map);
9921 goto out;
9922 }
9923
9924 if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
9925 btrfs_warn(fs_info,
9926 "swapfile must have single data profile");
9927 ret = -EINVAL;
9928 goto out;
9929 }
9930
9931 if (device == NULL) {
9932 device = map->stripes[0].dev;
9933 ret = btrfs_add_swapfile_pin(inode, device, false);
9934 if (ret == 1)
9935 ret = 0;
9936 else if (ret)
9937 goto out;
9938 } else if (device != map->stripes[0].dev) {
9939 btrfs_warn(fs_info, "swapfile must be on one device");
9940 ret = -EINVAL;
9941 goto out;
9942 }
9943
9944 physical_block_start = (map->stripes[0].physical +
9945 (logical_block_start - map->start));
9946 len = min(len, map->chunk_len - (logical_block_start - map->start));
9947 btrfs_free_chunk_map(map);
9948 map = NULL;
9949
9950 bg = btrfs_lookup_block_group(fs_info, logical_block_start);
9951 if (!bg) {
9952 btrfs_warn(fs_info,
9953 "could not find block group containing swapfile");
9954 ret = -EINVAL;
9955 goto out;
9956 }
9957
9958 if (!btrfs_inc_block_group_swap_extents(bg)) {
9959 btrfs_warn(fs_info,
9960 "block group for swapfile at %llu is read-only%s",
9961 bg->start,
9962 atomic_read(&fs_info->scrubs_running) ?
9963 " (scrub running)" : "");
9964 btrfs_put_block_group(bg);
9965 ret = -EINVAL;
9966 goto out;
9967 }
9968
9969 ret = btrfs_add_swapfile_pin(inode, bg, true);
9970 if (ret) {
9971 btrfs_put_block_group(bg);
9972 if (ret == 1)
9973 ret = 0;
9974 else
9975 goto out;
9976 }
9977
9978 if (bsi.block_len &&
9979 bsi.block_start + bsi.block_len == physical_block_start) {
9980 bsi.block_len += len;
9981 } else {
9982 if (bsi.block_len) {
9983 ret = btrfs_add_swap_extent(sis, &bsi);
9984 if (ret)
9985 goto out;
9986 }
9987 bsi.start = start;
9988 bsi.block_start = physical_block_start;
9989 bsi.block_len = len;
9990 }
9991
9992 start += len;
9993 }
9994
9995 if (bsi.block_len)
9996 ret = btrfs_add_swap_extent(sis, &bsi);
9997
9998 out:
9999 if (!IS_ERR_OR_NULL(em))
10000 free_extent_map(em);
10001 if (!IS_ERR_OR_NULL(map))
10002 btrfs_free_chunk_map(map);
10003
10004 unlock_extent(io_tree, 0, isize - 1, &cached_state);
10005
10006 if (ret)
10007 btrfs_swap_deactivate(file);
10008
10009 btrfs_drew_write_unlock(&root->snapshot_lock);
10010
10011 btrfs_exclop_finish(fs_info);
10012
10013 if (ret)
10014 return ret;
10015
10016 if (device)
10017 sis->bdev = device->bdev;
10018 *span = bsi.highest_ppage - bsi.lowest_ppage + 1;
10019 sis->max = bsi.nr_pages;
10020 sis->pages = bsi.nr_pages - 1;
10021 sis->highest_bit = bsi.nr_pages - 1;
10022 return bsi.nr_extents;
10023 }
10024 #else
btrfs_swap_deactivate(struct file * file)10025 static void btrfs_swap_deactivate(struct file *file)
10026 {
10027 }
10028
btrfs_swap_activate(struct swap_info_struct * sis,struct file * file,sector_t * span)10029 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
10030 sector_t *span)
10031 {
10032 return -EOPNOTSUPP;
10033 }
10034 #endif
10035
10036 /*
10037 * Update the number of bytes used in the VFS' inode. When we replace extents in
10038 * a range (clone, dedupe, fallocate's zero range), we must update the number of
10039 * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls
10040 * always get a correct value.
10041 */
btrfs_update_inode_bytes(struct btrfs_inode * inode,const u64 add_bytes,const u64 del_bytes)10042 void btrfs_update_inode_bytes(struct btrfs_inode *inode,
10043 const u64 add_bytes,
10044 const u64 del_bytes)
10045 {
10046 if (add_bytes == del_bytes)
10047 return;
10048
10049 spin_lock(&inode->lock);
10050 if (del_bytes > 0)
10051 inode_sub_bytes(&inode->vfs_inode, del_bytes);
10052 if (add_bytes > 0)
10053 inode_add_bytes(&inode->vfs_inode, add_bytes);
10054 spin_unlock(&inode->lock);
10055 }
10056
10057 /*
10058 * Verify that there are no ordered extents for a given file range.
10059 *
10060 * @inode: The target inode.
10061 * @start: Start offset of the file range, should be sector size aligned.
10062 * @end: End offset (inclusive) of the file range, its value +1 should be
10063 * sector size aligned.
10064 *
10065 * This should typically be used for cases where we locked an inode's VFS lock in
10066 * exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode,
10067 * we have flushed all delalloc in the range, we have waited for all ordered
10068 * extents in the range to complete and finally we have locked the file range in
10069 * the inode's io_tree.
10070 */
btrfs_assert_inode_range_clean(struct btrfs_inode * inode,u64 start,u64 end)10071 void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end)
10072 {
10073 struct btrfs_root *root = inode->root;
10074 struct btrfs_ordered_extent *ordered;
10075
10076 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
10077 return;
10078
10079 ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);
10080 if (ordered) {
10081 btrfs_err(root->fs_info,
10082 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])",
10083 start, end, btrfs_ino(inode), btrfs_root_id(root),
10084 ordered->file_offset,
10085 ordered->file_offset + ordered->num_bytes - 1);
10086 btrfs_put_ordered_extent(ordered);
10087 }
10088
10089 ASSERT(ordered == NULL);
10090 }
10091
10092 /*
10093 * Find the first inode with a minimum number.
10094 *
10095 * @root: The root to search for.
10096 * @min_ino: The minimum inode number.
10097 *
10098 * Find the first inode in the @root with a number >= @min_ino and return it.
10099 * Returns NULL if no such inode found.
10100 */
btrfs_find_first_inode(struct btrfs_root * root,u64 min_ino)10101 struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino)
10102 {
10103 struct btrfs_inode *inode;
10104 unsigned long from = min_ino;
10105
10106 xa_lock(&root->inodes);
10107 while (true) {
10108 inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT);
10109 if (!inode)
10110 break;
10111 if (igrab(&inode->vfs_inode))
10112 break;
10113
10114 from = btrfs_ino(inode) + 1;
10115 cond_resched_lock(&root->inodes.xa_lock);
10116 }
10117 xa_unlock(&root->inodes);
10118
10119 return inode;
10120 }
10121
10122 static const struct inode_operations btrfs_dir_inode_operations = {
10123 .getattr = btrfs_getattr,
10124 .lookup = btrfs_lookup,
10125 .create = btrfs_create,
10126 .unlink = btrfs_unlink,
10127 .link = btrfs_link,
10128 .mkdir = btrfs_mkdir,
10129 .rmdir = btrfs_rmdir,
10130 .rename = btrfs_rename2,
10131 .symlink = btrfs_symlink,
10132 .setattr = btrfs_setattr,
10133 .mknod = btrfs_mknod,
10134 .listxattr = btrfs_listxattr,
10135 .permission = btrfs_permission,
10136 .get_inode_acl = btrfs_get_acl,
10137 .set_acl = btrfs_set_acl,
10138 .update_time = btrfs_update_time,
10139 .tmpfile = btrfs_tmpfile,
10140 .fileattr_get = btrfs_fileattr_get,
10141 .fileattr_set = btrfs_fileattr_set,
10142 };
10143
10144 static const struct file_operations btrfs_dir_file_operations = {
10145 .llseek = btrfs_dir_llseek,
10146 .read = generic_read_dir,
10147 .iterate_shared = btrfs_real_readdir,
10148 .open = btrfs_opendir,
10149 .unlocked_ioctl = btrfs_ioctl,
10150 #ifdef CONFIG_COMPAT
10151 .compat_ioctl = btrfs_compat_ioctl,
10152 #endif
10153 .release = btrfs_release_file,
10154 .fsync = btrfs_sync_file,
10155 };
10156
10157 /*
10158 * btrfs doesn't support the bmap operation because swapfiles
10159 * use bmap to make a mapping of extents in the file. They assume
10160 * these extents won't change over the life of the file and they
10161 * use the bmap result to do IO directly to the drive.
10162 *
10163 * the btrfs bmap call would return logical addresses that aren't
10164 * suitable for IO and they also will change frequently as COW
10165 * operations happen. So, swapfile + btrfs == corruption.
10166 *
10167 * For now we're avoiding this by dropping bmap.
10168 */
10169 static const struct address_space_operations btrfs_aops = {
10170 .read_folio = btrfs_read_folio,
10171 .writepages = btrfs_writepages,
10172 .readahead = btrfs_readahead,
10173 .invalidate_folio = btrfs_invalidate_folio,
10174 .launder_folio = btrfs_launder_folio,
10175 .release_folio = btrfs_release_folio,
10176 .migrate_folio = btrfs_migrate_folio,
10177 .dirty_folio = filemap_dirty_folio,
10178 .error_remove_folio = generic_error_remove_folio,
10179 .swap_activate = btrfs_swap_activate,
10180 .swap_deactivate = btrfs_swap_deactivate,
10181 };
10182
10183 static const struct inode_operations btrfs_file_inode_operations = {
10184 .getattr = btrfs_getattr,
10185 .setattr = btrfs_setattr,
10186 .listxattr = btrfs_listxattr,
10187 .permission = btrfs_permission,
10188 .fiemap = btrfs_fiemap,
10189 .get_inode_acl = btrfs_get_acl,
10190 .set_acl = btrfs_set_acl,
10191 .update_time = btrfs_update_time,
10192 .fileattr_get = btrfs_fileattr_get,
10193 .fileattr_set = btrfs_fileattr_set,
10194 };
10195 static const struct inode_operations btrfs_special_inode_operations = {
10196 .getattr = btrfs_getattr,
10197 .setattr = btrfs_setattr,
10198 .permission = btrfs_permission,
10199 .listxattr = btrfs_listxattr,
10200 .get_inode_acl = btrfs_get_acl,
10201 .set_acl = btrfs_set_acl,
10202 .update_time = btrfs_update_time,
10203 };
10204 static const struct inode_operations btrfs_symlink_inode_operations = {
10205 .get_link = page_get_link,
10206 .getattr = btrfs_getattr,
10207 .setattr = btrfs_setattr,
10208 .permission = btrfs_permission,
10209 .listxattr = btrfs_listxattr,
10210 .update_time = btrfs_update_time,
10211 };
10212
10213 const struct dentry_operations btrfs_dentry_operations = {
10214 .d_delete = btrfs_dentry_delete,
10215 };
10216