1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (c) 2023, Klara Inc.
14 */
15
16 #ifdef CONFIG_COMPAT
17 #include <linux/compat.h>
18 #endif
19 #include <linux/fs.h>
20 #ifdef HAVE_VFS_SPLICE_COPY_FILE_RANGE
21 #include <linux/splice.h>
22 #endif
23 #include <sys/file.h>
24 #include <sys/zfs_znode.h>
25 #include <sys/zfs_vnops.h>
26 #include <sys/zfeature.h>
27
28 /*
29 * Take the source and destination inode locks for a remap (clone or dedupe).
30 *
31 * Since Linux 4.20 the VFS does not lock the inodes for ->remap_file_range();
32 * the filesystem must, and it must impose an order on the two, or two remaps
33 * running in opposite directions deadlock: each would hold one inode and wait
34 * for the other, and an rwsem writer queues behind the existing readers. Order
35 * by inode address, as btrfs does. The destination is taken exclusively and
36 * the source shared, so concurrent remaps out of one hot source still proceed.
37 * Only the acquisition needs the order: releasing never waits, so the unlock
38 * side does not mirror it.
39 */
40 static void
zpl_remap_lock_two(struct inode * src_i,struct inode * dst_i)41 zpl_remap_lock_two(struct inode *src_i, struct inode *dst_i)
42 {
43 if (src_i == dst_i) {
44 spl_inode_lock(dst_i);
45 } else if (src_i < dst_i) {
46 spl_inode_lock_shared(src_i);
47 spl_inode_lock(dst_i);
48 } else {
49 spl_inode_lock(dst_i);
50 spl_inode_lock_shared(src_i);
51 }
52 }
53
54 static void
zpl_remap_unlock_two(struct inode * src_i,struct inode * dst_i)55 zpl_remap_unlock_two(struct inode *src_i, struct inode *dst_i)
56 {
57 spl_inode_unlock(dst_i);
58 if (src_i != dst_i)
59 spl_inode_unlock_shared(src_i);
60 }
61
62 /*
63 * Clone part of a file via block cloning.
64 *
65 * Note that we are not required to update file offsets; the kernel will take
66 * care of that depending on how it was called.
67 */
68 static ssize_t
zpl_clone_file_range_impl(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,size_t len)69 zpl_clone_file_range_impl(struct file *src_file, loff_t src_off,
70 struct file *dst_file, loff_t dst_off, size_t len)
71 {
72 struct inode *src_i = file_inode(src_file);
73 struct inode *dst_i = file_inode(dst_file);
74 uint64_t src_off_o = (uint64_t)src_off;
75 uint64_t dst_off_o = (uint64_t)dst_off;
76 uint64_t len_o = (uint64_t)len;
77 cred_t *cr = CRED();
78 fstrans_cookie_t cookie;
79 int err;
80
81 if (!zfs_bclone_enabled)
82 return (-EOPNOTSUPP);
83
84 if (!spa_feature_is_enabled(
85 dmu_objset_spa(ITOZSB(dst_i)->z_os), SPA_FEATURE_BLOCK_CLONING))
86 return (-EOPNOTSUPP);
87
88 zpl_remap_lock_two(src_i, dst_i);
89
90 crhold(cr);
91 cookie = spl_fstrans_mark();
92
93 err = -zfs_clone_range(ITOZ(src_i), &src_off_o, ITOZ(dst_i),
94 &dst_off_o, &len_o, cr);
95
96 spl_fstrans_unmark(cookie);
97 crfree(cr);
98
99 zpl_remap_unlock_two(src_i, dst_i);
100
101 if (err < 0)
102 return (err);
103
104 return ((ssize_t)len_o);
105 }
106
107 #if defined(HAVE_VFS_REMAP_FILE_RANGE) || \
108 defined(HAVE_VFS_DEDUPE_FILE_RANGE)
109 /*
110 * Logic shared by the FIDEDUPERANGE entry points. Compare len bytes at
111 * src_off in src_file with dst_off in dst_file and, if they are identical,
112 * share the underlying blocks via zfs_dedupe_range(). Returns the number of
113 * bytes deduped, -EBADE if the ranges differ (which the VFS reports as
114 * FILE_DEDUPE_RANGE_DIFFERS), or another negative errno on failure.
115 */
116 static ssize_t
zpl_dedupe_file_range_impl(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,size_t len,boolean_t can_shorten)117 zpl_dedupe_file_range_impl(struct file *src_file, loff_t src_off,
118 struct file *dst_file, loff_t dst_off, size_t len, boolean_t can_shorten)
119 {
120 struct inode *src_i = file_inode(src_file);
121 struct inode *dst_i = file_inode(dst_file);
122 uint64_t src_off_o = (uint64_t)src_off;
123 uint64_t dst_off_o = (uint64_t)dst_off;
124 uint64_t len_o = (uint64_t)len;
125 cred_t *cr = CRED();
126 fstrans_cookie_t cookie;
127 boolean_t same = B_FALSE;
128 int err;
129
130 if (!zfs_bclone_enabled)
131 return (-EOPNOTSUPP);
132
133 if (!spa_feature_is_enabled(
134 dmu_objset_spa(ITOZSB(dst_i)->z_os), SPA_FEATURE_BLOCK_CLONING))
135 return (-EOPNOTSUPP);
136
137 zpl_remap_lock_two(src_i, dst_i);
138
139 crhold(cr);
140 cookie = spl_fstrans_mark();
141
142 err = -zfs_dedupe_range(ITOZ(src_i), src_off_o, ITOZ(dst_i),
143 dst_off_o, &len_o, cr, can_shorten, &same);
144
145 spl_fstrans_unmark(cookie);
146 crfree(cr);
147
148 zpl_remap_unlock_two(src_i, dst_i);
149
150 if (err < 0)
151 return (err);
152
153 if (!same)
154 return (-EBADE);
155
156 return ((ssize_t)len_o);
157 }
158 #endif
159
160 /*
161 * Entry point for copy_file_range(). Copy len bytes from src_off in src_file
162 * to dst_off in dst_file. We are permitted to do this however we like, so we
163 * try to just clone the blocks, and if we can't support it, fall back to the
164 * kernel's generic byte copy function.
165 */
166 ssize_t
zpl_copy_file_range(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,size_t len,unsigned int flags)167 zpl_copy_file_range(struct file *src_file, loff_t src_off,
168 struct file *dst_file, loff_t dst_off, size_t len, unsigned int flags)
169 {
170 ssize_t ret;
171
172 /* Flags is reserved for future extensions and must be zero. */
173 if (flags != 0)
174 return (-EINVAL);
175
176 /* Try to do it via zfs_clone_range() and allow shortening. */
177 ret = zpl_clone_file_range_impl(src_file, src_off,
178 dst_file, dst_off, len);
179
180 #if defined(HAVE_VFS_GENERIC_COPY_FILE_RANGE)
181 /*
182 * Since Linux 5.3 the filesystem driver is responsible for executing
183 * an appropriate fallback, and a generic fallback function is provided.
184 */
185 if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -EXDEV ||
186 ret == -EAGAIN)
187 ret = generic_copy_file_range(src_file, src_off, dst_file,
188 dst_off, len, flags);
189 #elif defined(HAVE_VFS_SPLICE_COPY_FILE_RANGE)
190 /*
191 * Since 6.8 the fallback function is called splice_copy_file_range
192 * and has a slightly different signature.
193 */
194 if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -EXDEV ||
195 ret == -EAGAIN)
196 ret = splice_copy_file_range(src_file, src_off, dst_file,
197 dst_off, len);
198 #else
199 /*
200 * Before Linux 5.3 the filesystem has to return -EOPNOTSUPP to signal
201 * to the kernel that it should fallback to a content copy.
202 */
203 if (ret == -EINVAL || ret == -EXDEV || ret == -EAGAIN)
204 ret = -EOPNOTSUPP;
205 #endif /* HAVE_VFS_GENERIC_COPY_FILE_RANGE || HAVE_VFS_SPLICE_COPY_FILE_RANGE */
206
207 return (ret);
208 }
209
210 #ifdef HAVE_VFS_REMAP_FILE_RANGE
211 /*
212 * Entry point for FICLONE/FICLONERANGE/FIDEDUPERANGE.
213 *
214 * FICLONE and FICLONERANGE are basically the same as copy_file_range(), except
215 * that they must clone - they cannot fall back to copying. FICLONE is exactly
216 * FICLONERANGE, for the entire file. We don't need to try to tell them apart;
217 * the kernel will sort that out for us.
218 *
219 * FIDEDUPERANGE is for turning a non-clone into a clone, that is, compare the
220 * range in both files and if they're the same, arrange for them to be backed
221 * by the same storage.
222 *
223 * REMAP_FILE_CAN_SHORTEN lets us know we can clone less than the given range
224 * if we want. It's designed for filesystems that may need to shorten the
225 * length for alignment, EOF, or any other requirement. ZFS may shorten the
226 * request when there is outstanding dirty data which hasn't been written.
227 */
228 loff_t
zpl_remap_file_range(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,loff_t len,unsigned int flags)229 zpl_remap_file_range(struct file *src_file, loff_t src_off,
230 struct file *dst_file, loff_t dst_off, loff_t len, unsigned int flags)
231 {
232 if (flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_CAN_SHORTEN))
233 return (-EINVAL);
234
235 if (flags & REMAP_FILE_DEDUP)
236 return (zpl_dedupe_file_range_impl(src_file, src_off, dst_file,
237 dst_off, len, !!(flags & REMAP_FILE_CAN_SHORTEN)));
238
239 /* Zero length means to clone everything to the end of the file */
240 if (len == 0)
241 len = i_size_read(file_inode(src_file)) - src_off;
242
243 ssize_t ret = zpl_clone_file_range_impl(src_file, src_off,
244 dst_file, dst_off, len);
245
246 if (!(flags & REMAP_FILE_CAN_SHORTEN) && ret >= 0 && ret != len)
247 ret = -EINVAL;
248
249 return (ret);
250 }
251 #endif /* HAVE_VFS_REMAP_FILE_RANGE */
252
253 #if defined(HAVE_VFS_CLONE_FILE_RANGE)
254 /*
255 * Entry point for FICLONE and FICLONERANGE, before Linux 4.20.
256 */
257 int
zpl_clone_file_range(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,uint64_t len)258 zpl_clone_file_range(struct file *src_file, loff_t src_off,
259 struct file *dst_file, loff_t dst_off, uint64_t len)
260 {
261 /* Zero length means to clone everything to the end of the file */
262 if (len == 0)
263 len = i_size_read(file_inode(src_file)) - src_off;
264
265 /* The entire length must be cloned or this is an error. */
266 ssize_t ret = zpl_clone_file_range_impl(src_file, src_off,
267 dst_file, dst_off, len);
268
269 if (ret >= 0 && ret != len)
270 ret = -EINVAL;
271
272 return (ret);
273 }
274 #endif /* HAVE_VFS_CLONE_FILE_RANGE */
275
276 #ifdef HAVE_VFS_DEDUPE_FILE_RANGE
277 /*
278 * Entry point for FIDEDUPERANGE, before Linux 4.20.
279 */
280 int
zpl_dedupe_file_range(struct file * src_file,loff_t src_off,struct file * dst_file,loff_t dst_off,uint64_t len)281 zpl_dedupe_file_range(struct file *src_file, loff_t src_off,
282 struct file *dst_file, loff_t dst_off, uint64_t len)
283 {
284 /*
285 * The pre-4.20 dedupe interface has no way to signal that a short
286 * dedupe is acceptable, so the whole range must match.
287 */
288 return (zpl_dedupe_file_range_impl(src_file, src_off, dst_file,
289 dst_off, len, B_FALSE));
290 }
291 #endif /* HAVE_VFS_DEDUPE_FILE_RANGE */
292