xref: /linux/fs/cachefiles/io.c (revision 85cdaca6970028bf6f544c355c90035586836ddf)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* kiocb-using read/write
3  *
4  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5  * Written by David Howells (dhowells@redhat.com)
6  */
7 
8 #include <linux/mount.h>
9 #include <linux/slab.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/bio.h>
13 #include <linux/falloc.h>
14 #include <linux/sched/mm.h>
15 #include <trace/events/fscache.h>
16 #include <trace/events/netfs.h>
17 #include "internal.h"
18 
19 struct cachefiles_kiocb {
20 	struct kiocb		iocb;
21 	refcount_t		ki_refcnt;
22 	loff_t			start;
23 	union {
24 		size_t		skipped;
25 		size_t		len;
26 	};
27 	struct cachefiles_object *object;
28 	netfs_io_terminated_t	term_func;
29 	void			*term_func_priv;
30 	bool			was_async;
31 	unsigned int		inval_counter;	/* Copy of cookie->inval_counter */
32 	u64			b_writing;
33 };
34 
35 static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
36 {
37 	if (refcount_dec_and_test(&ki->ki_refcnt)) {
38 		cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
39 		fput(ki->iocb.ki_filp);
40 		kfree(ki);
41 	}
42 }
43 
44 /*
45  * Handle completion of a read from the cache.
46  */
47 static void cachefiles_read_complete(struct kiocb *iocb, long ret)
48 {
49 	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
50 	struct inode *inode = file_inode(ki->iocb.ki_filp);
51 
52 	_enter("%ld", ret);
53 
54 	if (ret < 0)
55 		trace_cachefiles_io_error(ki->object, inode, ret,
56 					  cachefiles_trace_read_error);
57 
58 	if (ki->term_func) {
59 		if (ret >= 0) {
60 			if (ki->object->cookie->inval_counter == ki->inval_counter)
61 				ki->skipped += ret;
62 			else
63 				ret = -ESTALE;
64 		}
65 
66 		ki->term_func(ki->term_func_priv, ret);
67 	}
68 
69 	cachefiles_put_kiocb(ki);
70 }
71 
72 /*
73  * Initiate a read from the cache.
74  */
75 static int cachefiles_read(struct netfs_cache_resources *cres,
76 			   loff_t start_pos,
77 			   struct iov_iter *iter,
78 			   enum netfs_read_from_hole read_hole,
79 			   netfs_io_terminated_t term_func,
80 			   void *term_func_priv)
81 {
82 	struct cachefiles_object *object;
83 	struct cachefiles_kiocb *ki;
84 	struct file *file;
85 	unsigned int old_nofs;
86 	ssize_t ret = -ENOBUFS;
87 	size_t len = iov_iter_count(iter), skipped = 0;
88 
89 	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
90 		goto presubmission_error;
91 
92 	fscache_count_read();
93 	object = cachefiles_cres_object(cres);
94 	file = cachefiles_cres_file(cres);
95 
96 	_enter("%pD,%llu,%llx,%zx/%llx",
97 	       file, file_inode(file)->i_ino, start_pos, len,
98 	       i_size_read(file_inode(file)));
99 
100 	/* If the caller asked us to seek for data before doing the read, then
101 	 * we should do that now.  If we find a gap, we fill it with zeros.
102 	 */
103 	if (read_hole != NETFS_READ_HOLE_IGNORE) {
104 		loff_t off = start_pos, off2;
105 
106 		off2 = cachefiles_inject_read_error();
107 		if (off2 == 0)
108 			off2 = vfs_llseek(file, off, SEEK_DATA);
109 		if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
110 			skipped = 0;
111 			ret = off2;
112 			goto presubmission_error;
113 		}
114 
115 		if (off2 == -ENXIO || off2 >= start_pos + len) {
116 			/* The region is beyond the EOF or there's no more data
117 			 * in the region, so clear the rest of the buffer and
118 			 * return success.
119 			 */
120 			ret = -ENODATA;
121 			if (read_hole == NETFS_READ_HOLE_FAIL)
122 				goto presubmission_error;
123 
124 			iov_iter_zero(len, iter);
125 			skipped = len;
126 			ret = 0;
127 			goto presubmission_error;
128 		}
129 
130 		skipped = off2 - off;
131 		iov_iter_zero(skipped, iter);
132 	}
133 
134 	ret = -ENOMEM;
135 	ki = kzalloc_obj(struct cachefiles_kiocb);
136 	if (!ki)
137 		goto presubmission_error;
138 
139 	refcount_set(&ki->ki_refcnt, 2);
140 	ki->iocb.ki_filp	= file;
141 	ki->iocb.ki_pos		= start_pos + skipped;
142 	ki->iocb.ki_flags	= IOCB_DIRECT;
143 	ki->iocb.ki_ioprio	= get_current_ioprio();
144 	ki->skipped		= skipped;
145 	ki->object		= object;
146 	ki->inval_counter	= cres->inval_counter;
147 	ki->term_func		= term_func;
148 	ki->term_func_priv	= term_func_priv;
149 	ki->was_async		= true;
150 
151 	if (ki->term_func)
152 		ki->iocb.ki_complete = cachefiles_read_complete;
153 
154 	get_file(ki->iocb.ki_filp);
155 	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
156 
157 	trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
158 	old_nofs = memalloc_nofs_save();
159 	ret = cachefiles_inject_read_error();
160 	if (ret == 0)
161 		ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
162 	memalloc_nofs_restore(old_nofs);
163 	switch (ret) {
164 	case -EIOCBQUEUED:
165 		goto in_progress;
166 
167 	case -ERESTARTSYS:
168 	case -ERESTARTNOINTR:
169 	case -ERESTARTNOHAND:
170 	case -ERESTART_RESTARTBLOCK:
171 		/* There's no easy way to restart the syscall since other AIO's
172 		 * may be already running. Just fail this IO with EINTR.
173 		 */
174 		ret = -EINTR;
175 		fallthrough;
176 	default:
177 		ki->was_async = false;
178 		cachefiles_read_complete(&ki->iocb, ret);
179 		if (ret > 0)
180 			ret = 0;
181 		break;
182 	}
183 
184 in_progress:
185 	cachefiles_put_kiocb(ki);
186 	_leave(" = %zd", ret);
187 	return ret;
188 
189 presubmission_error:
190 	if (term_func)
191 		term_func(term_func_priv, ret < 0 ? ret : skipped);
192 	return ret;
193 }
194 
195 /*
196  * Query the occupancy of the cache in a region, returning where the next chunk
197  * of data starts and how long it is.
198  */
199 static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
200 				      loff_t start, size_t len, size_t granularity,
201 				      loff_t *_data_start, size_t *_data_len)
202 {
203 	struct cachefiles_object *object;
204 	struct file *file;
205 	loff_t off, off2;
206 
207 	*_data_start = -1;
208 	*_data_len = 0;
209 
210 	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
211 		return -ENOBUFS;
212 
213 	object = cachefiles_cres_object(cres);
214 	file = cachefiles_cres_file(cres);
215 	granularity = max_t(size_t, object->volume->cache->bsize, granularity);
216 
217 	_enter("%pD,%llu,%llx,%zx/%llx",
218 	       file, file_inode(file)->i_ino, start, len,
219 	       i_size_read(file_inode(file)));
220 
221 	off = cachefiles_inject_read_error();
222 	if (off == 0)
223 		off = vfs_llseek(file, start, SEEK_DATA);
224 	if (off == -ENXIO)
225 		return -ENODATA; /* Beyond EOF */
226 	if (off < 0 && off >= (loff_t)-MAX_ERRNO)
227 		return -ENOBUFS; /* Error. */
228 	if (round_up(off, granularity) >= start + len)
229 		return -ENODATA; /* No data in range */
230 
231 	off2 = cachefiles_inject_read_error();
232 	if (off2 == 0)
233 		off2 = vfs_llseek(file, off, SEEK_HOLE);
234 	if (off2 == -ENXIO)
235 		return -ENODATA; /* Beyond EOF */
236 	if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
237 		return -ENOBUFS; /* Error. */
238 
239 	/* Round away partial blocks */
240 	off = round_up(off, granularity);
241 	off2 = round_down(off2, granularity);
242 	if (off2 <= off)
243 		return -ENODATA;
244 
245 	*_data_start = off;
246 	if (off2 > start + len)
247 		*_data_len = len;
248 	else
249 		*_data_len = off2 - off;
250 	return 0;
251 }
252 
253 /*
254  * Handle completion of a write to the cache.
255  */
256 static void cachefiles_write_complete(struct kiocb *iocb, long ret)
257 {
258 	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
259 	struct cachefiles_object *object = ki->object;
260 	struct inode *inode = file_inode(ki->iocb.ki_filp);
261 
262 	_enter("%ld", ret);
263 
264 	if (ki->was_async)
265 		kiocb_end_write(iocb);
266 
267 	if (ret < 0)
268 		trace_cachefiles_io_error(object, inode, ret,
269 					  cachefiles_trace_write_error);
270 
271 	atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
272 	set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
273 	if (ki->term_func)
274 		ki->term_func(ki->term_func_priv, ret);
275 	cachefiles_put_kiocb(ki);
276 }
277 
278 /*
279  * Initiate a write to the cache.
280  */
281 int __cachefiles_write(struct cachefiles_object *object,
282 		       struct file *file,
283 		       loff_t start_pos,
284 		       struct iov_iter *iter,
285 		       netfs_io_terminated_t term_func,
286 		       void *term_func_priv)
287 {
288 	struct cachefiles_cache *cache;
289 	struct cachefiles_kiocb *ki;
290 	unsigned int old_nofs;
291 	ssize_t ret;
292 	size_t len = iov_iter_count(iter);
293 
294 	fscache_count_write();
295 	cache = object->volume->cache;
296 
297 	_enter("%pD,%llu,%llx,%zx/%llx",
298 	       file, file_inode(file)->i_ino, start_pos, len,
299 	       i_size_read(file_inode(file)));
300 
301 	ki = kzalloc_obj(struct cachefiles_kiocb);
302 	if (!ki) {
303 		if (term_func)
304 			term_func(term_func_priv, -ENOMEM);
305 		return -ENOMEM;
306 	}
307 
308 	refcount_set(&ki->ki_refcnt, 2);
309 	ki->iocb.ki_filp	= file;
310 	ki->iocb.ki_pos		= start_pos;
311 	ki->iocb.ki_flags	= IOCB_DIRECT | IOCB_WRITE;
312 	ki->iocb.ki_ioprio	= get_current_ioprio();
313 	ki->object		= object;
314 	ki->start		= start_pos;
315 	ki->len			= len;
316 	ki->term_func		= term_func;
317 	ki->term_func_priv	= term_func_priv;
318 	ki->was_async		= true;
319 	ki->b_writing		= (len + (1 << cache->bshift) - 1) >> cache->bshift;
320 
321 	if (ki->term_func)
322 		ki->iocb.ki_complete = cachefiles_write_complete;
323 	atomic_long_add(ki->b_writing, &cache->b_writing);
324 
325 	get_file(ki->iocb.ki_filp);
326 	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
327 
328 	trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len);
329 	old_nofs = memalloc_nofs_save();
330 	ret = cachefiles_inject_write_error();
331 	if (ret == 0)
332 		ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
333 	memalloc_nofs_restore(old_nofs);
334 	switch (ret) {
335 	case -EIOCBQUEUED:
336 		goto in_progress;
337 
338 	case -ERESTARTSYS:
339 	case -ERESTARTNOINTR:
340 	case -ERESTARTNOHAND:
341 	case -ERESTART_RESTARTBLOCK:
342 		/* There's no easy way to restart the syscall since other AIO's
343 		 * may be already running. Just fail this IO with EINTR.
344 		 */
345 		ret = -EINTR;
346 		fallthrough;
347 	default:
348 		ki->was_async = false;
349 		cachefiles_write_complete(&ki->iocb, ret);
350 		break;
351 	}
352 
353 in_progress:
354 	cachefiles_put_kiocb(ki);
355 	_leave(" = %zd", ret);
356 	return ret;
357 }
358 
359 static int cachefiles_write(struct netfs_cache_resources *cres,
360 			    loff_t start_pos,
361 			    struct iov_iter *iter,
362 			    netfs_io_terminated_t term_func,
363 			    void *term_func_priv)
364 {
365 	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
366 		if (term_func)
367 			term_func(term_func_priv, -ENOBUFS);
368 		trace_netfs_sreq(term_func_priv, netfs_sreq_trace_cache_nowrite);
369 		return -ENOBUFS;
370 	}
371 
372 	return __cachefiles_write(cachefiles_cres_object(cres),
373 				  cachefiles_cres_file(cres),
374 				  start_pos, iter,
375 				  term_func, term_func_priv);
376 }
377 
378 static inline enum netfs_io_source
379 cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
380 			   loff_t start, size_t *_len, loff_t i_size,
381 			   unsigned long *_flags, ino_t netfs_ino)
382 {
383 	enum cachefiles_prepare_read_trace why;
384 	struct cachefiles_object *object = NULL;
385 	struct cachefiles_cache *cache;
386 	struct fscache_cookie *cookie = fscache_cres_cookie(cres);
387 	const struct cred *saved_cred;
388 	struct file *file = cachefiles_cres_file(cres);
389 	enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
390 	size_t len = *_len;
391 	loff_t off, to;
392 	ino_t ino = file ? file_inode(file)->i_ino : 0;
393 
394 	_enter("%zx @%llx/%llx", len, start, i_size);
395 
396 	if (start >= i_size) {
397 		ret = NETFS_FILL_WITH_ZEROES;
398 		why = cachefiles_trace_read_after_eof;
399 		goto out_no_object;
400 	}
401 
402 	if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
403 		__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
404 		why = cachefiles_trace_read_no_data;
405 		goto out_no_object;
406 	}
407 
408 	/* The object and the file may be being created in the background. */
409 	if (!file) {
410 		why = cachefiles_trace_read_no_file;
411 		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
412 			goto out_no_object;
413 		file = cachefiles_cres_file(cres);
414 		if (!file)
415 			goto out_no_object;
416 		ino = file_inode(file)->i_ino;
417 	}
418 
419 	object = cachefiles_cres_object(cres);
420 	cache = object->volume->cache;
421 	cachefiles_begin_secure(cache, &saved_cred);
422 	off = cachefiles_inject_read_error();
423 	if (off == 0)
424 		off = vfs_llseek(file, start, SEEK_DATA);
425 	if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
426 		if (off == (loff_t)-ENXIO) {
427 			why = cachefiles_trace_read_seek_nxio;
428 			goto download_and_store;
429 		}
430 		trace_cachefiles_io_error(object, file_inode(file), off,
431 					  cachefiles_trace_seek_error);
432 		why = cachefiles_trace_read_seek_error;
433 		goto out;
434 	}
435 
436 	if (off >= start + len) {
437 		why = cachefiles_trace_read_found_hole;
438 		goto download_and_store;
439 	}
440 
441 	if (off > start) {
442 		off = round_up(off, cache->bsize);
443 		len = off - start;
444 		*_len = len;
445 		why = cachefiles_trace_read_found_part;
446 		goto download_and_store;
447 	}
448 
449 	to = cachefiles_inject_read_error();
450 	if (to == 0)
451 		to = vfs_llseek(file, start, SEEK_HOLE);
452 	if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
453 		trace_cachefiles_io_error(object, file_inode(file), to,
454 					  cachefiles_trace_seek_error);
455 		why = cachefiles_trace_read_seek_error;
456 		goto out;
457 	}
458 
459 	if (to < start + len) {
460 		if (start + len >= i_size)
461 			to = round_up(to, cache->bsize);
462 		else
463 			to = round_down(to, cache->bsize);
464 		len = to - start;
465 		*_len = len;
466 	}
467 
468 	why = cachefiles_trace_read_have_data;
469 	ret = NETFS_READ_FROM_CACHE;
470 	goto out;
471 
472 download_and_store:
473 	__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
474 out:
475 	cachefiles_end_secure(cache, saved_cred);
476 out_no_object:
477 	trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
478 	return ret;
479 }
480 
481 /*
482  * Prepare a read operation, shortening it to a cached/uncached
483  * boundary as appropriate.
484  */
485 static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
486 						    unsigned long long i_size)
487 {
488 	return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
489 					  subreq->start, &subreq->len, i_size,
490 					  &subreq->flags, subreq->rreq->inode->i_ino);
491 }
492 
493 /*
494  * Prepare for a write to occur.
495  */
496 int __cachefiles_prepare_write(struct cachefiles_object *object,
497 			       struct file *file,
498 			       loff_t *_start, size_t *_len, size_t upper_len,
499 			       bool no_space_allocated_yet)
500 {
501 	struct cachefiles_cache *cache = object->volume->cache;
502 	loff_t start = *_start, pos;
503 	size_t len = *_len;
504 	int ret;
505 
506 	/* Round to DIO size */
507 	start = round_down(*_start, PAGE_SIZE);
508 	if (start != *_start || *_len > upper_len) {
509 		/* Probably asked to cache a streaming write written into the
510 		 * pagecache when the cookie was temporarily out of service to
511 		 * culling.
512 		 */
513 		fscache_count_dio_misfit();
514 		return -ENOBUFS;
515 	}
516 
517 	*_len = round_up(len, PAGE_SIZE);
518 
519 	/* We need to work out whether there's sufficient disk space to perform
520 	 * the write - but we can skip that check if we have space already
521 	 * allocated.
522 	 */
523 	if (no_space_allocated_yet)
524 		goto check_space;
525 
526 	pos = cachefiles_inject_read_error();
527 	if (pos == 0)
528 		pos = vfs_llseek(file, start, SEEK_DATA);
529 	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
530 		if (pos == -ENXIO)
531 			goto check_space; /* Unallocated tail */
532 		trace_cachefiles_io_error(object, file_inode(file), pos,
533 					  cachefiles_trace_seek_error);
534 		return pos;
535 	}
536 	if ((u64)pos >= (u64)start + *_len)
537 		goto check_space; /* Unallocated region */
538 
539 	/* We have a block that's at least partially filled - if we're low on
540 	 * space, we need to see if it's fully allocated.  If it's not, we may
541 	 * want to cull it.
542 	 */
543 	if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
544 				 cachefiles_has_space_check) == 0)
545 		return 0; /* Enough space to simply overwrite the whole block */
546 
547 	pos = cachefiles_inject_read_error();
548 	if (pos == 0)
549 		pos = vfs_llseek(file, start, SEEK_HOLE);
550 	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
551 		trace_cachefiles_io_error(object, file_inode(file), pos,
552 					  cachefiles_trace_seek_error);
553 		return pos;
554 	}
555 	if ((u64)pos >= (u64)start + *_len)
556 		return 0; /* Fully allocated */
557 
558 	/* Partially allocated, but insufficient space: cull. */
559 	fscache_count_no_write_space();
560 	ret = cachefiles_inject_remove_error();
561 	if (ret == 0)
562 		ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
563 				    start, *_len);
564 	if (ret < 0) {
565 		trace_cachefiles_io_error(object, file_inode(file), ret,
566 					  cachefiles_trace_fallocate_error);
567 		cachefiles_io_error_obj(object,
568 					"CacheFiles: fallocate failed (%d)\n", ret);
569 		ret = -EIO;
570 	}
571 
572 	return ret;
573 
574 check_space:
575 	return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
576 				    cachefiles_has_space_for_write);
577 }
578 
579 static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
580 				    loff_t *_start, size_t *_len, size_t upper_len,
581 				    loff_t i_size, bool no_space_allocated_yet)
582 {
583 	struct cachefiles_object *object = cachefiles_cres_object(cres);
584 	struct cachefiles_cache *cache = object->volume->cache;
585 	const struct cred *saved_cred;
586 	int ret;
587 
588 	if (!cachefiles_cres_file(cres)) {
589 		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
590 			return -ENOBUFS;
591 		if (!cachefiles_cres_file(cres))
592 			return -ENOBUFS;
593 	}
594 
595 	cachefiles_begin_secure(cache, &saved_cred);
596 	ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
597 					 _start, _len, upper_len,
598 					 no_space_allocated_yet);
599 	cachefiles_end_secure(cache, saved_cred);
600 	return ret;
601 }
602 
603 static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq)
604 {
605 	struct netfs_io_request *wreq = subreq->rreq;
606 	struct netfs_cache_resources *cres = &wreq->cache_resources;
607 	struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
608 
609 	_enter("W=%x[%x] %llx", wreq->debug_id, subreq->debug_index, subreq->start);
610 
611 	stream->sreq_max_len = MAX_RW_COUNT;
612 	stream->sreq_max_segs = BIO_MAX_VECS;
613 
614 	if (!cachefiles_cres_file(cres)) {
615 		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
616 			return netfs_prepare_write_failed(subreq);
617 		if (!cachefiles_cres_file(cres))
618 			return netfs_prepare_write_failed(subreq);
619 	}
620 }
621 
622 static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
623 {
624 	struct netfs_io_request *wreq = subreq->rreq;
625 	struct netfs_cache_resources *cres = &wreq->cache_resources;
626 	struct cachefiles_object *object = cachefiles_cres_object(cres);
627 	struct cachefiles_cache *cache = object->volume->cache;
628 	struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
629 	const struct cred *saved_cred;
630 	size_t off, pre, post, len = subreq->len;
631 	loff_t start = subreq->start;
632 	int ret;
633 
634 	_enter("W=%x[%x] %llx-%llx",
635 	       wreq->debug_id, subreq->debug_index, start, start + len - 1);
636 
637 	/* We need to start on the cache granularity boundary */
638 	off = start & (CACHEFILES_DIO_BLOCK_SIZE - 1);
639 	if (off) {
640 		pre = CACHEFILES_DIO_BLOCK_SIZE - off;
641 		if (pre >= len) {
642 			fscache_count_dio_misfit();
643 			netfs_write_subrequest_terminated(subreq, len);
644 			return;
645 		}
646 		subreq->transferred += pre;
647 		start += pre;
648 		len -= pre;
649 		iov_iter_advance(&subreq->io_iter, pre);
650 	}
651 
652 	/* We also need to end on the cache granularity boundary */
653 	if (start + len == wreq->i_size) {
654 		size_t part = len % CACHEFILES_DIO_BLOCK_SIZE;
655 		size_t need = CACHEFILES_DIO_BLOCK_SIZE - part;
656 
657 		if (part && stream->submit_extendable_to >= need) {
658 			len += need;
659 			subreq->len += need;
660 			subreq->io_iter.count += need;
661 		}
662 	}
663 
664 	post = len & (CACHEFILES_DIO_BLOCK_SIZE - 1);
665 	if (post) {
666 		len -= post;
667 		if (len == 0) {
668 			fscache_count_dio_misfit();
669 			netfs_write_subrequest_terminated(subreq, post);
670 			return;
671 		}
672 		iov_iter_truncate(&subreq->io_iter, len);
673 	}
674 
675 	trace_netfs_sreq(subreq, netfs_sreq_trace_cache_prepare);
676 	cachefiles_begin_secure(cache, &saved_cred);
677 	ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
678 					 &start, &len, len, true);
679 	cachefiles_end_secure(cache, saved_cred);
680 	if (ret < 0) {
681 		netfs_write_subrequest_terminated(subreq, ret);
682 		return;
683 	}
684 
685 	trace_netfs_sreq(subreq, netfs_sreq_trace_cache_write);
686 	cachefiles_write(&subreq->rreq->cache_resources,
687 			 subreq->start, &subreq->io_iter,
688 			 netfs_write_subrequest_terminated, subreq);
689 }
690 
691 /*
692  * Clean up an operation.
693  */
694 static void cachefiles_end_operation(struct netfs_cache_resources *cres)
695 {
696 	struct file *file = cachefiles_cres_file(cres);
697 
698 	if (file)
699 		fput(file);
700 	fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
701 }
702 
703 static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
704 	.end_operation		= cachefiles_end_operation,
705 	.read			= cachefiles_read,
706 	.write			= cachefiles_write,
707 	.issue_write		= cachefiles_issue_write,
708 	.prepare_read		= cachefiles_prepare_read,
709 	.prepare_write		= cachefiles_prepare_write,
710 	.prepare_write_subreq	= cachefiles_prepare_write_subreq,
711 	.query_occupancy	= cachefiles_query_occupancy,
712 };
713 
714 /*
715  * Open the cache file when beginning a cache operation.
716  */
717 bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
718 				enum fscache_want_state want_state)
719 {
720 	struct cachefiles_object *object = cachefiles_cres_object(cres);
721 
722 	if (!cachefiles_cres_file(cres)) {
723 		cres->ops = &cachefiles_netfs_cache_ops;
724 		if (object->file) {
725 			spin_lock(&object->lock);
726 			if (!cres->cache_priv2 && object->file)
727 				cres->cache_priv2 = get_file(object->file);
728 			spin_unlock(&object->lock);
729 		}
730 	}
731 
732 	if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
733 		pr_err("failed to get cres->file\n");
734 		return false;
735 	}
736 
737 	return true;
738 }
739