inode.c (e2877c2a032d821ca339dc333401e0c63bf67d5c) inode.c (9aa29a20b70097213d10e03a452366ceea72fc02)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6#include <crypto/hash.h>
7#include <linux/kernel.h>
8#include <linux/bio.h>

--- 63 unchanged lines hidden (view full) ---

72#include "raid-stripe-tree.h"
73#include "fiemap.h"
74
75struct btrfs_iget_args {
76 u64 ino;
77 struct btrfs_root *root;
78};
79
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6#include <crypto/hash.h>
7#include <linux/kernel.h>
8#include <linux/bio.h>

--- 63 unchanged lines hidden (view full) ---

72#include "raid-stripe-tree.h"
73#include "fiemap.h"
74
75struct btrfs_iget_args {
76 u64 ino;
77 struct btrfs_root *root;
78};
79
80struct btrfs_dio_data {
81 ssize_t submitted;
82 struct extent_changeset *data_reserved;
83 struct btrfs_ordered_extent *ordered;
84 bool data_space_reserved;
85 bool nocow_done;
86};
87
88struct btrfs_dio_private {
89 /* Range of I/O */
90 u64 file_offset;
91 u32 bytes;
92
93 /* This must be last */
94 struct btrfs_bio bbio;
95};
96
97static struct bio_set btrfs_dio_bioset;
98
99struct btrfs_rename_ctx {
100 /* Output field. Stores the index number of the old directory entry. */
101 u64 index;
102};
103
104/*
105 * Used by data_reloc_print_warning_inode() to pass needed info for filename
106 * resolution and output of error message.

--- 26 unchanged lines hidden (view full) ---

133
134static int btrfs_setsize(struct inode *inode, struct iattr *attr);
135static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);
136
137static noinline int run_delalloc_cow(struct btrfs_inode *inode,
138 struct page *locked_page, u64 start,
139 u64 end, struct writeback_control *wbc,
140 bool pages_dirty);
80struct btrfs_rename_ctx {
81 /* Output field. Stores the index number of the old directory entry. */
82 u64 index;
83};
84
85/*
86 * Used by data_reloc_print_warning_inode() to pass needed info for filename
87 * resolution and output of error message.

--- 26 unchanged lines hidden (view full) ---

114
115static int btrfs_setsize(struct inode *inode, struct iattr *attr);
116static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);
117
118static noinline int run_delalloc_cow(struct btrfs_inode *inode,
119 struct page *locked_page, u64 start,
120 u64 end, struct writeback_control *wbc,
121 bool pages_dirty);
141static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start,
142 const struct btrfs_file_extent *file_extent,
143 int type);
144
145static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
146 u64 root, void *warn_ctx)
147{
148 struct data_reloc_warn *warn = warn_ctx;
149 struct btrfs_fs_info *fs_info = warn->fs_info;
150 struct extent_buffer *eb;
151 struct btrfs_inode_item *inode_item;

--- 1048 unchanged lines hidden (view full) ---

1200 /* Here we're doing allocation and writeback of the compressed pages */
1201 file_extent.disk_bytenr = ins.objectid;
1202 file_extent.disk_num_bytes = ins.offset;
1203 file_extent.ram_bytes = async_extent->ram_size;
1204 file_extent.num_bytes = async_extent->ram_size;
1205 file_extent.offset = 0;
1206 file_extent.compression = async_extent->compress_type;
1207
122
123static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
124 u64 root, void *warn_ctx)
125{
126 struct data_reloc_warn *warn = warn_ctx;
127 struct btrfs_fs_info *fs_info = warn->fs_info;
128 struct extent_buffer *eb;
129 struct btrfs_inode_item *inode_item;

--- 1048 unchanged lines hidden (view full) ---

1178 /* Here we're doing allocation and writeback of the compressed pages */
1179 file_extent.disk_bytenr = ins.objectid;
1180 file_extent.disk_num_bytes = ins.offset;
1181 file_extent.ram_bytes = async_extent->ram_size;
1182 file_extent.num_bytes = async_extent->ram_size;
1183 file_extent.offset = 0;
1184 file_extent.compression = async_extent->compress_type;
1185
1208 em = create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
1186 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
1209 if (IS_ERR(em)) {
1210 ret = PTR_ERR(em);
1211 goto out_free_reserve;
1212 }
1213 free_extent_map(em);
1214
1215 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
1216 1 << BTRFS_ORDERED_COMPRESSED);

--- 35 unchanged lines hidden (view full) ---

1252 kthread_associate_blkcg(NULL);
1253 btrfs_debug(fs_info,
1254"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",
1255 btrfs_root_id(root), btrfs_ino(inode), start,
1256 async_extent->ram_size, ret);
1257 kfree(async_extent);
1258}
1259
1187 if (IS_ERR(em)) {
1188 ret = PTR_ERR(em);
1189 goto out_free_reserve;
1190 }
1191 free_extent_map(em);
1192
1193 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
1194 1 << BTRFS_ORDERED_COMPRESSED);

--- 35 unchanged lines hidden (view full) ---

1230 kthread_associate_blkcg(NULL);
1231 btrfs_debug(fs_info,
1232"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",
1233 btrfs_root_id(root), btrfs_ino(inode), start,
1234 async_extent->ram_size, ret);
1235 kfree(async_extent);
1236}
1237
1260static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
1261 u64 num_bytes)
1238u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
1239 u64 num_bytes)
1262{
1263 struct extent_map_tree *em_tree = &inode->extent_tree;
1264 struct extent_map *em;
1265 u64 alloc_hint = 0;
1266
1267 read_lock(&em_tree->lock);
1268 em = search_extent_mapping(em_tree, start, num_bytes);
1269 if (em) {

--- 93 unchanged lines hidden (view full) ---

1363 * already done the cleanup.
1364 */
1365 if (ret == 0)
1366 ret = 1;
1367 goto done;
1368 }
1369 }
1370
1240{
1241 struct extent_map_tree *em_tree = &inode->extent_tree;
1242 struct extent_map *em;
1243 u64 alloc_hint = 0;
1244
1245 read_lock(&em_tree->lock);
1246 em = search_extent_mapping(em_tree, start, num_bytes);
1247 if (em) {

--- 93 unchanged lines hidden (view full) ---

1341 * already done the cleanup.
1342 */
1343 if (ret == 0)
1344 ret = 1;
1345 goto done;
1346 }
1347 }
1348
1371 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
1349 alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);
1372
1373 /*
1374 * Relocation relies on the relocated extents to have exactly the same
1375 * size as the original extents. Normally writeback for relocation data
1376 * extents follows a NOCOW path because relocation preallocates the
1377 * extents. However, due to an operation such as scrub turning a block
1378 * group to RO mode, it may fallback to COW mode, so we must make sure
1379 * an extent allocated during COW has exactly the requested size and can

--- 50 unchanged lines hidden (view full) ---

1430 file_extent.num_bytes = ins.offset;
1431 file_extent.ram_bytes = ins.offset;
1432 file_extent.offset = 0;
1433 file_extent.compression = BTRFS_COMPRESS_NONE;
1434
1435 lock_extent(&inode->io_tree, start, start + ram_size - 1,
1436 &cached);
1437
1350
1351 /*
1352 * Relocation relies on the relocated extents to have exactly the same
1353 * size as the original extents. Normally writeback for relocation data
1354 * extents follows a NOCOW path because relocation preallocates the
1355 * extents. However, due to an operation such as scrub turning a block
1356 * group to RO mode, it may fallback to COW mode, so we must make sure
1357 * an extent allocated during COW has exactly the requested size and can

--- 50 unchanged lines hidden (view full) ---

1408 file_extent.num_bytes = ins.offset;
1409 file_extent.ram_bytes = ins.offset;
1410 file_extent.offset = 0;
1411 file_extent.compression = BTRFS_COMPRESS_NONE;
1412
1413 lock_extent(&inode->io_tree, start, start + ram_size - 1,
1414 &cached);
1415
1438 em = create_io_em(inode, start, &file_extent, BTRFS_ORDERED_REGULAR);
1416 em = btrfs_create_io_em(inode, start, &file_extent,
1417 BTRFS_ORDERED_REGULAR);
1439 if (IS_ERR(em)) {
1440 unlock_extent(&inode->io_tree, start,
1441 start + ram_size - 1, &cached);
1442 ret = PTR_ERR(em);
1443 goto out_reserve;
1444 }
1445 free_extent_map(em);
1446

--- 700 unchanged lines hidden (view full) ---

2147
2148 nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;
2149 lock_extent(&inode->io_tree, cur_offset, nocow_end, &cached_state);
2150
2151 is_prealloc = extent_type == BTRFS_FILE_EXTENT_PREALLOC;
2152 if (is_prealloc) {
2153 struct extent_map *em;
2154
1418 if (IS_ERR(em)) {
1419 unlock_extent(&inode->io_tree, start,
1420 start + ram_size - 1, &cached);
1421 ret = PTR_ERR(em);
1422 goto out_reserve;
1423 }
1424 free_extent_map(em);
1425

--- 700 unchanged lines hidden (view full) ---

2126
2127 nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;
2128 lock_extent(&inode->io_tree, cur_offset, nocow_end, &cached_state);
2129
2130 is_prealloc = extent_type == BTRFS_FILE_EXTENT_PREALLOC;
2131 if (is_prealloc) {
2132 struct extent_map *em;
2133
2155 em = create_io_em(inode, cur_offset, &nocow_args.file_extent,
2156 BTRFS_ORDERED_PREALLOC);
2134 em = btrfs_create_io_em(inode, cur_offset,
2135 &nocow_args.file_extent,
2136 BTRFS_ORDERED_PREALLOC);
2157 if (IS_ERR(em)) {
2158 unlock_extent(&inode->io_tree, cur_offset,
2159 nocow_end, &cached_state);
2160 btrfs_dec_nocow_writers(nocow_bg);
2161 ret = PTR_ERR(em);
2162 goto error;
2163 }
2164 free_extent_map(em);

--- 412 unchanged lines hidden (view full) ---

2577 ASSERT(inode->new_delalloc_bytes >= len);
2578 inode->new_delalloc_bytes -= len;
2579 if (bits & EXTENT_ADD_INODE_BYTES)
2580 inode_add_bytes(&inode->vfs_inode, len);
2581 spin_unlock(&inode->lock);
2582 }
2583}
2584
2137 if (IS_ERR(em)) {
2138 unlock_extent(&inode->io_tree, cur_offset,
2139 nocow_end, &cached_state);
2140 btrfs_dec_nocow_writers(nocow_bg);
2141 ret = PTR_ERR(em);
2142 goto error;
2143 }
2144 free_extent_map(em);

--- 412 unchanged lines hidden (view full) ---

2557 ASSERT(inode->new_delalloc_bytes >= len);
2558 inode->new_delalloc_bytes -= len;
2559 if (bits & EXTENT_ADD_INODE_BYTES)
2560 inode_add_bytes(&inode->vfs_inode, len);
2561 spin_unlock(&inode->lock);
2562 }
2563}
2564
2585static int btrfs_extract_ordered_extent(struct btrfs_bio *bbio,
2586 struct btrfs_ordered_extent *ordered)
2587{
2588 u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
2589 u64 len = bbio->bio.bi_iter.bi_size;
2590 struct btrfs_ordered_extent *new;
2591 int ret;
2592
2593 /* Must always be called for the beginning of an ordered extent. */
2594 if (WARN_ON_ONCE(start != ordered->disk_bytenr))
2595 return -EINVAL;
2596
2597 /* No need to split if the ordered extent covers the entire bio. */
2598 if (ordered->disk_num_bytes == len) {
2599 refcount_inc(&ordered->refs);
2600 bbio->ordered = ordered;
2601 return 0;
2602 }
2603
2604 /*
2605 * Don't split the extent_map for NOCOW extents, as we're writing into
2606 * a pre-existing one.
2607 */
2608 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) {
2609 ret = split_extent_map(bbio->inode, bbio->file_offset,
2610 ordered->num_bytes, len,
2611 ordered->disk_bytenr);
2612 if (ret)
2613 return ret;
2614 }
2615
2616 new = btrfs_split_ordered_extent(ordered, len);
2617 if (IS_ERR(new))
2618 return PTR_ERR(new);
2619 bbio->ordered = new;
2620 return 0;
2621}
2622
2623/*
2624 * given a list of ordered sums record them in the inode. This happens
2625 * at IO completion time based on sums calculated at bio submission time.
2626 */
2627static int add_pending_csums(struct btrfs_trans_handle *trans,
2628 struct list_head *list)
2629{
2630 struct btrfs_ordered_sum *sum;

--- 4359 unchanged lines hidden (view full) ---

6990
6991 if (ret) {
6992 free_extent_map(em);
6993 return ERR_PTR(ret);
6994 }
6995 return em;
6996}
6997
2565/*
2566 * given a list of ordered sums record them in the inode. This happens
2567 * at IO completion time based on sums calculated at bio submission time.
2568 */
2569static int add_pending_csums(struct btrfs_trans_handle *trans,
2570 struct list_head *list)
2571{
2572 struct btrfs_ordered_sum *sum;

--- 4359 unchanged lines hidden (view full) ---

6932
6933 if (ret) {
6934 free_extent_map(em);
6935 return ERR_PTR(ret);
6936 }
6937 return em;
6938}
6939
6998static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
6999 struct btrfs_dio_data *dio_data,
7000 const u64 start,
7001 const struct btrfs_file_extent *file_extent,
7002 const int type)
7003{
7004 struct extent_map *em = NULL;
7005 struct btrfs_ordered_extent *ordered;
7006
7007 if (type != BTRFS_ORDERED_NOCOW) {
7008 em = create_io_em(inode, start, file_extent, type);
7009 if (IS_ERR(em))
7010 goto out;
7011 }
7012
7013 ordered = btrfs_alloc_ordered_extent(inode, start, file_extent,
7014 (1 << type) |
7015 (1 << BTRFS_ORDERED_DIRECT));
7016 if (IS_ERR(ordered)) {
7017 if (em) {
7018 free_extent_map(em);
7019 btrfs_drop_extent_map_range(inode, start,
7020 start + file_extent->num_bytes - 1, false);
7021 }
7022 em = ERR_CAST(ordered);
7023 } else {
7024 ASSERT(!dio_data->ordered);
7025 dio_data->ordered = ordered;
7026 }
7027 out:
7028
7029 return em;
7030}
7031
7032static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode,
7033 struct btrfs_dio_data *dio_data,
7034 u64 start, u64 len)
7035{
7036 struct btrfs_root *root = inode->root;
7037 struct btrfs_fs_info *fs_info = root->fs_info;
7038 struct btrfs_file_extent file_extent;
7039 struct extent_map *em;
7040 struct btrfs_key ins;
7041 u64 alloc_hint;
7042 int ret;
7043
7044 alloc_hint = get_extent_allocation_hint(inode, start, len);
7045again:
7046 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7047 0, alloc_hint, &ins, 1, 1);
7048 if (ret == -EAGAIN) {
7049 ASSERT(btrfs_is_zoned(fs_info));
7050 wait_on_bit_io(&inode->root->fs_info->flags, BTRFS_FS_NEED_ZONE_FINISH,
7051 TASK_UNINTERRUPTIBLE);
7052 goto again;
7053 }
7054 if (ret)
7055 return ERR_PTR(ret);
7056
7057 file_extent.disk_bytenr = ins.objectid;
7058 file_extent.disk_num_bytes = ins.offset;
7059 file_extent.num_bytes = ins.offset;
7060 file_extent.ram_bytes = ins.offset;
7061 file_extent.offset = 0;
7062 file_extent.compression = BTRFS_COMPRESS_NONE;
7063 em = btrfs_create_dio_extent(inode, dio_data, start, &file_extent,
7064 BTRFS_ORDERED_REGULAR);
7065 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7066 if (IS_ERR(em))
7067 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset,
7068 1);
7069
7070 return em;
7071}
7072
7073static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
7074{
7075 struct btrfs_block_group *block_group;
7076 bool readonly = false;
7077
7078 block_group = btrfs_lookup_block_group(fs_info, bytenr);
7079 if (!block_group || block_group->ro)
7080 readonly = true;

--- 114 unchanged lines hidden (view full) ---

7195
7196 *len = nocow_args.file_extent.num_bytes;
7197 ret = 1;
7198out:
7199 btrfs_free_path(path);
7200 return ret;
7201}
7202
6940static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
6941{
6942 struct btrfs_block_group *block_group;
6943 bool readonly = false;
6944
6945 block_group = btrfs_lookup_block_group(fs_info, bytenr);
6946 if (!block_group || block_group->ro)
6947 readonly = true;

--- 114 unchanged lines hidden (view full) ---

7062
7063 *len = nocow_args.file_extent.num_bytes;
7064 ret = 1;
7065out:
7066 btrfs_free_path(path);
7067 return ret;
7068}
7069
7203static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7204 struct extent_state **cached_state,
7205 unsigned int iomap_flags)
7206{
7207 const bool writing = (iomap_flags & IOMAP_WRITE);
7208 const bool nowait = (iomap_flags & IOMAP_NOWAIT);
7209 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7210 struct btrfs_ordered_extent *ordered;
7211 int ret = 0;
7212
7213 while (1) {
7214 if (nowait) {
7215 if (!try_lock_extent(io_tree, lockstart, lockend,
7216 cached_state))
7217 return -EAGAIN;
7218 } else {
7219 lock_extent(io_tree, lockstart, lockend, cached_state);
7220 }
7221 /*
7222 * We're concerned with the entire range that we're going to be
7223 * doing DIO to, so we need to make sure there's no ordered
7224 * extents in this range.
7225 */
7226 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7227 lockend - lockstart + 1);
7228
7229 /*
7230 * We need to make sure there are no buffered pages in this
7231 * range either, we could have raced between the invalidate in
7232 * generic_file_direct_write and locking the extent. The
7233 * invalidate needs to happen so that reads after a write do not
7234 * get stale data.
7235 */
7236 if (!ordered &&
7237 (!writing || !filemap_range_has_page(inode->i_mapping,
7238 lockstart, lockend)))
7239 break;
7240
7241 unlock_extent(io_tree, lockstart, lockend, cached_state);
7242
7243 if (ordered) {
7244 if (nowait) {
7245 btrfs_put_ordered_extent(ordered);
7246 ret = -EAGAIN;
7247 break;
7248 }
7249 /*
7250 * If we are doing a DIO read and the ordered extent we
7251 * found is for a buffered write, we can not wait for it
7252 * to complete and retry, because if we do so we can
7253 * deadlock with concurrent buffered writes on page
7254 * locks. This happens only if our DIO read covers more
7255 * than one extent map, if at this point has already
7256 * created an ordered extent for a previous extent map
7257 * and locked its range in the inode's io tree, and a
7258 * concurrent write against that previous extent map's
7259 * range and this range started (we unlock the ranges
7260 * in the io tree only when the bios complete and
7261 * buffered writes always lock pages before attempting
7262 * to lock range in the io tree).
7263 */
7264 if (writing ||
7265 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7266 btrfs_start_ordered_extent(ordered);
7267 else
7268 ret = nowait ? -EAGAIN : -ENOTBLK;
7269 btrfs_put_ordered_extent(ordered);
7270 } else {
7271 /*
7272 * We could trigger writeback for this range (and wait
7273 * for it to complete) and then invalidate the pages for
7274 * this range (through invalidate_inode_pages2_range()),
7275 * but that can lead us to a deadlock with a concurrent
7276 * call to readahead (a buffered read or a defrag call
7277 * triggered a readahead) on a page lock due to an
7278 * ordered dio extent we created before but did not have
7279 * yet a corresponding bio submitted (whence it can not
7280 * complete), which makes readahead wait for that
7281 * ordered extent to complete while holding a lock on
7282 * that page.
7283 */
7284 ret = nowait ? -EAGAIN : -ENOTBLK;
7285 }
7286
7287 if (ret)
7288 break;
7289
7290 cond_resched();
7291 }
7292
7293 return ret;
7294}
7295
7296/* The callers of this must take lock_extent() */
7070/* The callers of this must take lock_extent() */
7297static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start,
7298 const struct btrfs_file_extent *file_extent,
7299 int type)
7071struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start,
7072 const struct btrfs_file_extent *file_extent,
7073 int type)
7300{
7301 struct extent_map *em;
7302 int ret;
7303
7304 /*
7305 * Note the missing NOCOW type.
7306 *
7307 * For pure NOCOW writes, we should not create an io extent map, but

--- 50 unchanged lines hidden (view full) ---

7358 free_extent_map(em);
7359 return ERR_PTR(ret);
7360 }
7361
7362 /* em got 2 refs now, callers needs to do free_extent_map once. */
7363 return em;
7364}
7365
7074{
7075 struct extent_map *em;
7076 int ret;
7077
7078 /*
7079 * Note the missing NOCOW type.
7080 *
7081 * For pure NOCOW writes, we should not create an io extent map, but

--- 50 unchanged lines hidden (view full) ---

7132 free_extent_map(em);
7133 return ERR_PTR(ret);
7134 }
7135
7136 /* em got 2 refs now, callers needs to do free_extent_map once. */
7137 return em;
7138}
7139
7366
7367static int btrfs_get_blocks_direct_write(struct extent_map **map,
7368 struct inode *inode,
7369 struct btrfs_dio_data *dio_data,
7370 u64 start, u64 *lenp,
7371 unsigned int iomap_flags)
7372{
7373 const bool nowait = (iomap_flags & IOMAP_NOWAIT);
7374 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
7375 struct btrfs_file_extent file_extent;
7376 struct extent_map *em = *map;
7377 int type;
7378 u64 block_start;
7379 struct btrfs_block_group *bg;
7380 bool can_nocow = false;
7381 bool space_reserved = false;
7382 u64 len = *lenp;
7383 u64 prev_len;
7384 int ret = 0;
7385
7386 /*
7387 * We don't allocate a new extent in the following cases
7388 *
7389 * 1) The inode is marked as NODATACOW. In this case we'll just use the
7390 * existing extent.
7391 * 2) The extent is marked as PREALLOC. We're good to go here and can
7392 * just use the extent.
7393 *
7394 */
7395 if ((em->flags & EXTENT_FLAG_PREALLOC) ||
7396 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7397 em->disk_bytenr != EXTENT_MAP_HOLE)) {
7398 if (em->flags & EXTENT_FLAG_PREALLOC)
7399 type = BTRFS_ORDERED_PREALLOC;
7400 else
7401 type = BTRFS_ORDERED_NOCOW;
7402 len = min(len, em->len - (start - em->start));
7403 block_start = extent_map_block_start(em) + (start - em->start);
7404
7405 if (can_nocow_extent(inode, start, &len,
7406 &file_extent, false, false) == 1) {
7407 bg = btrfs_inc_nocow_writers(fs_info, block_start);
7408 if (bg)
7409 can_nocow = true;
7410 }
7411 }
7412
7413 prev_len = len;
7414 if (can_nocow) {
7415 struct extent_map *em2;
7416
7417 /* We can NOCOW, so only need to reserve metadata space. */
7418 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len,
7419 nowait);
7420 if (ret < 0) {
7421 /* Our caller expects us to free the input extent map. */
7422 free_extent_map(em);
7423 *map = NULL;
7424 btrfs_dec_nocow_writers(bg);
7425 if (nowait && (ret == -ENOSPC || ret == -EDQUOT))
7426 ret = -EAGAIN;
7427 goto out;
7428 }
7429 space_reserved = true;
7430
7431 em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start,
7432 &file_extent, type);
7433 btrfs_dec_nocow_writers(bg);
7434 if (type == BTRFS_ORDERED_PREALLOC) {
7435 free_extent_map(em);
7436 *map = em2;
7437 em = em2;
7438 }
7439
7440 if (IS_ERR(em2)) {
7441 ret = PTR_ERR(em2);
7442 goto out;
7443 }
7444
7445 dio_data->nocow_done = true;
7446 } else {
7447 /* Our caller expects us to free the input extent map. */
7448 free_extent_map(em);
7449 *map = NULL;
7450
7451 if (nowait) {
7452 ret = -EAGAIN;
7453 goto out;
7454 }
7455
7456 /*
7457 * If we could not allocate data space before locking the file
7458 * range and we can't do a NOCOW write, then we have to fail.
7459 */
7460 if (!dio_data->data_space_reserved) {
7461 ret = -ENOSPC;
7462 goto out;
7463 }
7464
7465 /*
7466 * We have to COW and we have already reserved data space before,
7467 * so now we reserve only metadata.
7468 */
7469 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len,
7470 false);
7471 if (ret < 0)
7472 goto out;
7473 space_reserved = true;
7474
7475 em = btrfs_new_extent_direct(BTRFS_I(inode), dio_data, start, len);
7476 if (IS_ERR(em)) {
7477 ret = PTR_ERR(em);
7478 goto out;
7479 }
7480 *map = em;
7481 len = min(len, em->len - (start - em->start));
7482 if (len < prev_len)
7483 btrfs_delalloc_release_metadata(BTRFS_I(inode),
7484 prev_len - len, true);
7485 }
7486
7487 /*
7488 * We have created our ordered extent, so we can now release our reservation
7489 * for an outstanding extent.
7490 */
7491 btrfs_delalloc_release_extents(BTRFS_I(inode), prev_len);
7492
7493 /*
7494 * Need to update the i_size under the extent lock so buffered
7495 * readers will get the updated i_size when we unlock.
7496 */
7497 if (start + len > i_size_read(inode))
7498 i_size_write(inode, start + len);
7499out:
7500 if (ret && space_reserved) {
7501 btrfs_delalloc_release_extents(BTRFS_I(inode), len);
7502 btrfs_delalloc_release_metadata(BTRFS_I(inode), len, true);
7503 }
7504 *lenp = len;
7505 return ret;
7506}
7507
7508static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
7509 loff_t length, unsigned int flags, struct iomap *iomap,
7510 struct iomap *srcmap)
7511{
7512 struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap);
7513 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
7514 struct extent_map *em;
7515 struct extent_state *cached_state = NULL;
7516 struct btrfs_dio_data *dio_data = iter->private;
7517 u64 lockstart, lockend;
7518 const bool write = !!(flags & IOMAP_WRITE);
7519 int ret = 0;
7520 u64 len = length;
7521 const u64 data_alloc_len = length;
7522 bool unlock_extents = false;
7523
7524 /*
7525 * We could potentially fault if we have a buffer > PAGE_SIZE, and if
7526 * we're NOWAIT we may submit a bio for a partial range and return
7527 * EIOCBQUEUED, which would result in an errant short read.
7528 *
7529 * The best way to handle this would be to allow for partial completions
7530 * of iocb's, so we could submit the partial bio, return and fault in
7531 * the rest of the pages, and then submit the io for the rest of the
7532 * range. However we don't have that currently, so simply return
7533 * -EAGAIN at this point so that the normal path is used.
7534 */
7535 if (!write && (flags & IOMAP_NOWAIT) && length > PAGE_SIZE)
7536 return -EAGAIN;
7537
7538 /*
7539 * Cap the size of reads to that usually seen in buffered I/O as we need
7540 * to allocate a contiguous array for the checksums.
7541 */
7542 if (!write)
7543 len = min_t(u64, len, fs_info->sectorsize * BTRFS_MAX_BIO_SECTORS);
7544
7545 lockstart = start;
7546 lockend = start + len - 1;
7547
7548 /*
7549 * iomap_dio_rw() only does filemap_write_and_wait_range(), which isn't
7550 * enough if we've written compressed pages to this area, so we need to
7551 * flush the dirty pages again to make absolutely sure that any
7552 * outstanding dirty pages are on disk - the first flush only starts
7553 * compression on the data, while keeping the pages locked, so by the
7554 * time the second flush returns we know bios for the compressed pages
7555 * were submitted and finished, and the pages no longer under writeback.
7556 *
7557 * If we have a NOWAIT request and we have any pages in the range that
7558 * are locked, likely due to compression still in progress, we don't want
7559 * to block on page locks. We also don't want to block on pages marked as
7560 * dirty or under writeback (same as for the non-compression case).
7561 * iomap_dio_rw() did the same check, but after that and before we got
7562 * here, mmap'ed writes may have happened or buffered reads started
7563 * (readpage() and readahead(), which lock pages), as we haven't locked
7564 * the file range yet.
7565 */
7566 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
7567 &BTRFS_I(inode)->runtime_flags)) {
7568 if (flags & IOMAP_NOWAIT) {
7569 if (filemap_range_needs_writeback(inode->i_mapping,
7570 lockstart, lockend))
7571 return -EAGAIN;
7572 } else {
7573 ret = filemap_fdatawrite_range(inode->i_mapping, start,
7574 start + length - 1);
7575 if (ret)
7576 return ret;
7577 }
7578 }
7579
7580 memset(dio_data, 0, sizeof(*dio_data));
7581
7582 /*
7583 * We always try to allocate data space and must do it before locking
7584 * the file range, to avoid deadlocks with concurrent writes to the same
7585 * range if the range has several extents and the writes don't expand the
7586 * current i_size (the inode lock is taken in shared mode). If we fail to
7587 * allocate data space here we continue and later, after locking the
7588 * file range, we fail with ENOSPC only if we figure out we can not do a
7589 * NOCOW write.
7590 */
7591 if (write && !(flags & IOMAP_NOWAIT)) {
7592 ret = btrfs_check_data_free_space(BTRFS_I(inode),
7593 &dio_data->data_reserved,
7594 start, data_alloc_len, false);
7595 if (!ret)
7596 dio_data->data_space_reserved = true;
7597 else if (ret && !(BTRFS_I(inode)->flags &
7598 (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
7599 goto err;
7600 }
7601
7602 /*
7603 * If this errors out it's because we couldn't invalidate pagecache for
7604 * this range and we need to fallback to buffered IO, or we are doing a
7605 * NOWAIT read/write and we need to block.
7606 */
7607 ret = lock_extent_direct(inode, lockstart, lockend, &cached_state, flags);
7608 if (ret < 0)
7609 goto err;
7610
7611 em = btrfs_get_extent(BTRFS_I(inode), NULL, start, len);
7612 if (IS_ERR(em)) {
7613 ret = PTR_ERR(em);
7614 goto unlock_err;
7615 }
7616
7617 /*
7618 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7619 * io. INLINE is special, and we could probably kludge it in here, but
7620 * it's still buffered so for safety lets just fall back to the generic
7621 * buffered path.
7622 *
7623 * For COMPRESSED we _have_ to read the entire extent in so we can
7624 * decompress it, so there will be buffering required no matter what we
7625 * do, so go ahead and fallback to buffered.
7626 *
7627 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7628 * to buffered IO. Don't blame me, this is the price we pay for using
7629 * the generic code.
7630 */
7631 if (extent_map_is_compressed(em) || em->disk_bytenr == EXTENT_MAP_INLINE) {
7632 free_extent_map(em);
7633 /*
7634 * If we are in a NOWAIT context, return -EAGAIN in order to
7635 * fallback to buffered IO. This is not only because we can
7636 * block with buffered IO (no support for NOWAIT semantics at
7637 * the moment) but also to avoid returning short reads to user
7638 * space - this happens if we were able to read some data from
7639 * previous non-compressed extents and then when we fallback to
7640 * buffered IO, at btrfs_file_read_iter() by calling
7641 * filemap_read(), we fail to fault in pages for the read buffer,
7642 * in which case filemap_read() returns a short read (the number
7643 * of bytes previously read is > 0, so it does not return -EFAULT).
7644 */
7645 ret = (flags & IOMAP_NOWAIT) ? -EAGAIN : -ENOTBLK;
7646 goto unlock_err;
7647 }
7648
7649 len = min(len, em->len - (start - em->start));
7650
7651 /*
7652 * If we have a NOWAIT request and the range contains multiple extents
7653 * (or a mix of extents and holes), then we return -EAGAIN to make the
7654 * caller fallback to a context where it can do a blocking (without
7655 * NOWAIT) request. This way we avoid doing partial IO and returning
7656 * success to the caller, which is not optimal for writes and for reads
7657 * it can result in unexpected behaviour for an application.
7658 *
7659 * When doing a read, because we use IOMAP_DIO_PARTIAL when calling
7660 * iomap_dio_rw(), we can end up returning less data then what the caller
7661 * asked for, resulting in an unexpected, and incorrect, short read.
7662 * That is, the caller asked to read N bytes and we return less than that,
7663 * which is wrong unless we are crossing EOF. This happens if we get a
7664 * page fault error when trying to fault in pages for the buffer that is
7665 * associated to the struct iov_iter passed to iomap_dio_rw(), and we
7666 * have previously submitted bios for other extents in the range, in
7667 * which case iomap_dio_rw() may return us EIOCBQUEUED if not all of
7668 * those bios have completed by the time we get the page fault error,
7669 * which we return back to our caller - we should only return EIOCBQUEUED
7670 * after we have submitted bios for all the extents in the range.
7671 */
7672 if ((flags & IOMAP_NOWAIT) && len < length) {
7673 free_extent_map(em);
7674 ret = -EAGAIN;
7675 goto unlock_err;
7676 }
7677
7678 if (write) {
7679 ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
7680 start, &len, flags);
7681 if (ret < 0)
7682 goto unlock_err;
7683 unlock_extents = true;
7684 /* Recalc len in case the new em is smaller than requested */
7685 len = min(len, em->len - (start - em->start));
7686 if (dio_data->data_space_reserved) {
7687 u64 release_offset;
7688 u64 release_len = 0;
7689
7690 if (dio_data->nocow_done) {
7691 release_offset = start;
7692 release_len = data_alloc_len;
7693 } else if (len < data_alloc_len) {
7694 release_offset = start + len;
7695 release_len = data_alloc_len - len;
7696 }
7697
7698 if (release_len > 0)
7699 btrfs_free_reserved_data_space(BTRFS_I(inode),
7700 dio_data->data_reserved,
7701 release_offset,
7702 release_len);
7703 }
7704 } else {
7705 /*
7706 * We need to unlock only the end area that we aren't using.
7707 * The rest is going to be unlocked by the endio routine.
7708 */
7709 lockstart = start + len;
7710 if (lockstart < lockend)
7711 unlock_extents = true;
7712 }
7713
7714 if (unlock_extents)
7715 unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7716 &cached_state);
7717 else
7718 free_extent_state(cached_state);
7719
7720 /*
7721 * Translate extent map information to iomap.
7722 * We trim the extents (and move the addr) even though iomap code does
7723 * that, since we have locked only the parts we are performing I/O in.
7724 */
7725 if ((em->disk_bytenr == EXTENT_MAP_HOLE) ||
7726 ((em->flags & EXTENT_FLAG_PREALLOC) && !write)) {
7727 iomap->addr = IOMAP_NULL_ADDR;
7728 iomap->type = IOMAP_HOLE;
7729 } else {
7730 iomap->addr = extent_map_block_start(em) + (start - em->start);
7731 iomap->type = IOMAP_MAPPED;
7732 }
7733 iomap->offset = start;
7734 iomap->bdev = fs_info->fs_devices->latest_dev->bdev;
7735 iomap->length = len;
7736 free_extent_map(em);
7737
7738 return 0;
7739
7740unlock_err:
7741 unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7742 &cached_state);
7743err:
7744 if (dio_data->data_space_reserved) {
7745 btrfs_free_reserved_data_space(BTRFS_I(inode),
7746 dio_data->data_reserved,
7747 start, data_alloc_len);
7748 extent_changeset_free(dio_data->data_reserved);
7749 }
7750
7751 return ret;
7752}
7753
7754static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
7755 ssize_t written, unsigned int flags, struct iomap *iomap)
7756{
7757 struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap);
7758 struct btrfs_dio_data *dio_data = iter->private;
7759 size_t submitted = dio_data->submitted;
7760 const bool write = !!(flags & IOMAP_WRITE);
7761 int ret = 0;
7762
7763 if (!write && (iomap->type == IOMAP_HOLE)) {
7764 /* If reading from a hole, unlock and return */
7765 unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1,
7766 NULL);
7767 return 0;
7768 }
7769
7770 if (submitted < length) {
7771 pos += submitted;
7772 length -= submitted;
7773 if (write)
7774 btrfs_finish_ordered_extent(dio_data->ordered, NULL,
7775 pos, length, false);
7776 else
7777 unlock_extent(&BTRFS_I(inode)->io_tree, pos,
7778 pos + length - 1, NULL);
7779 ret = -ENOTBLK;
7780 }
7781 if (write) {
7782 btrfs_put_ordered_extent(dio_data->ordered);
7783 dio_data->ordered = NULL;
7784 }
7785
7786 if (write)
7787 extent_changeset_free(dio_data->data_reserved);
7788 return ret;
7789}
7790
7791static void btrfs_dio_end_io(struct btrfs_bio *bbio)
7792{
7793 struct btrfs_dio_private *dip =
7794 container_of(bbio, struct btrfs_dio_private, bbio);
7795 struct btrfs_inode *inode = bbio->inode;
7796 struct bio *bio = &bbio->bio;
7797
7798 if (bio->bi_status) {
7799 btrfs_warn(inode->root->fs_info,
7800 "direct IO failed ino %llu op 0x%0x offset %#llx len %u err no %d",
7801 btrfs_ino(inode), bio->bi_opf,
7802 dip->file_offset, dip->bytes, bio->bi_status);
7803 }
7804
7805 if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
7806 btrfs_finish_ordered_extent(bbio->ordered, NULL,
7807 dip->file_offset, dip->bytes,
7808 !bio->bi_status);
7809 } else {
7810 unlock_extent(&inode->io_tree, dip->file_offset,
7811 dip->file_offset + dip->bytes - 1, NULL);
7812 }
7813
7814 bbio->bio.bi_private = bbio->private;
7815 iomap_dio_bio_end_io(bio);
7816}
7817
7818static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio,
7819 loff_t file_offset)
7820{
7821 struct btrfs_bio *bbio = btrfs_bio(bio);
7822 struct btrfs_dio_private *dip =
7823 container_of(bbio, struct btrfs_dio_private, bbio);
7824 struct btrfs_dio_data *dio_data = iter->private;
7825
7826 btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info,
7827 btrfs_dio_end_io, bio->bi_private);
7828 bbio->inode = BTRFS_I(iter->inode);
7829 bbio->file_offset = file_offset;
7830
7831 dip->file_offset = file_offset;
7832 dip->bytes = bio->bi_iter.bi_size;
7833
7834 dio_data->submitted += bio->bi_iter.bi_size;
7835
7836 /*
7837 * Check if we are doing a partial write. If we are, we need to split
7838 * the ordered extent to match the submitted bio. Hang on to the
7839 * remaining unfinishable ordered_extent in dio_data so that it can be
7840 * cancelled in iomap_end to avoid a deadlock wherein faulting the
7841 * remaining pages is blocked on the outstanding ordered extent.
7842 */
7843 if (iter->flags & IOMAP_WRITE) {
7844 int ret;
7845
7846 ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);
7847 if (ret) {
7848 btrfs_finish_ordered_extent(dio_data->ordered, NULL,
7849 file_offset, dip->bytes,
7850 !ret);
7851 bio->bi_status = errno_to_blk_status(ret);
7852 iomap_dio_bio_end_io(bio);
7853 return;
7854 }
7855 }
7856
7857 btrfs_submit_bio(bbio, 0);
7858}
7859
7860static const struct iomap_ops btrfs_dio_iomap_ops = {
7861 .iomap_begin = btrfs_dio_iomap_begin,
7862 .iomap_end = btrfs_dio_iomap_end,
7863};
7864
7865static const struct iomap_dio_ops btrfs_dio_ops = {
7866 .submit_io = btrfs_dio_submit_io,
7867 .bio_set = &btrfs_dio_bioset,
7868};
7869
7870ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, size_t done_before)
7871{
7872 struct btrfs_dio_data data = { 0 };
7873
7874 return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
7875 IOMAP_DIO_PARTIAL, &data, done_before);
7876}
7877
7878struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter,
7879 size_t done_before)
7880{
7881 struct btrfs_dio_data data = { 0 };
7882
7883 return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
7884 IOMAP_DIO_PARTIAL, &data, done_before);
7885}
7886
7887/*
7888 * For release_folio() and invalidate_folio() we have a race window where
7889 * folio_end_writeback() is called but the subpage spinlock is not yet released.
7890 * If we continue to release/invalidate the page, we could cause use-after-free
7891 * for subpage spinlock. So this function is to spin and wait for subpage
7892 * spinlock.
7893 */
7894static void wait_subpage_spinlock(struct page *page)

--- 603 unchanged lines hidden (view full) ---

8498
8499void __cold btrfs_destroy_cachep(void)
8500{
8501 /*
8502 * Make sure all delayed rcu free inodes are flushed before we
8503 * destroy cache.
8504 */
8505 rcu_barrier();
7140/*
7141 * For release_folio() and invalidate_folio() we have a race window where
7142 * folio_end_writeback() is called but the subpage spinlock is not yet released.
7143 * If we continue to release/invalidate the page, we could cause use-after-free
7144 * for subpage spinlock. So this function is to spin and wait for subpage
7145 * spinlock.
7146 */
7147static void wait_subpage_spinlock(struct page *page)

--- 603 unchanged lines hidden (view full) ---

7751
7752void __cold btrfs_destroy_cachep(void)
7753{
7754 /*
7755 * Make sure all delayed rcu free inodes are flushed before we
7756 * destroy cache.
7757 */
7758 rcu_barrier();
8506 bioset_exit(&btrfs_dio_bioset);
8507 kmem_cache_destroy(btrfs_inode_cachep);
8508}
8509
8510int __init btrfs_init_cachep(void)
8511{
8512 btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
8513 sizeof(struct btrfs_inode), 0,
8514 SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
8515 init_once);
8516 if (!btrfs_inode_cachep)
7759 kmem_cache_destroy(btrfs_inode_cachep);
7760}
7761
7762int __init btrfs_init_cachep(void)
7763{
7764 btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
7765 sizeof(struct btrfs_inode), 0,
7766 SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
7767 init_once);
7768 if (!btrfs_inode_cachep)
8517 goto fail;
7769 return -ENOMEM;
8518
7770
8519 if (bioset_init(&btrfs_dio_bioset, BIO_POOL_SIZE,
8520 offsetof(struct btrfs_dio_private, bbio.bio),
8521 BIOSET_NEED_BVECS))
8522 goto fail;
8523
8524 return 0;
7771 return 0;
8525fail:
8526 btrfs_destroy_cachep();
8527 return -ENOMEM;
8528}
8529
8530static int btrfs_getattr(struct mnt_idmap *idmap,
8531 const struct path *path, struct kstat *stat,
8532 u32 request_mask, unsigned int flags)
8533{
8534 u64 delalloc_bytes;
8535 u64 inode_bytes;

--- 1726 unchanged lines hidden (view full) ---

10262 extent_reserved = true;
10263
10264 file_extent.disk_bytenr = ins.objectid;
10265 file_extent.disk_num_bytes = ins.offset;
10266 file_extent.num_bytes = num_bytes;
10267 file_extent.ram_bytes = ram_bytes;
10268 file_extent.offset = encoded->unencoded_offset;
10269 file_extent.compression = compression;
7772}
7773
7774static int btrfs_getattr(struct mnt_idmap *idmap,
7775 const struct path *path, struct kstat *stat,
7776 u32 request_mask, unsigned int flags)
7777{
7778 u64 delalloc_bytes;
7779 u64 inode_bytes;

--- 1726 unchanged lines hidden (view full) ---

9506 extent_reserved = true;
9507
9508 file_extent.disk_bytenr = ins.objectid;
9509 file_extent.disk_num_bytes = ins.offset;
9510 file_extent.num_bytes = num_bytes;
9511 file_extent.ram_bytes = ram_bytes;
9512 file_extent.offset = encoded->unencoded_offset;
9513 file_extent.compression = compression;
10270 em = create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
9514 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
10271 if (IS_ERR(em)) {
10272 ret = PTR_ERR(em);
10273 goto out_free_reserved;
10274 }
10275 free_extent_map(em);
10276
10277 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
10278 (1 << BTRFS_ORDERED_ENCODED) |

--- 627 unchanged lines hidden --- 		9515 if (IS_ERR(em)) {
9516 ret = PTR_ERR(em);
9517 goto out_free_reserved;
9518 }
9519 free_extent_map(em);
9520
9521 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
9522 (1 << BTRFS_ORDERED_ENCODED) |

--- 627 unchanged lines hidden ---