1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* Internal definitions for network filesystem support
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/slab.h>
9 #include <linux/seq_file.h>
10 #include <linux/folio_queue.h>
11 #include <linux/netfs.h>
12 #include <linux/fscache.h>
13 #include <linux/fscache-cache.h>
14 #include <trace/events/netfs.h>
15 #include <trace/events/fscache.h>
16
17 #ifdef pr_fmt
18 #undef pr_fmt
19 #endif
20
21 #define pr_fmt(fmt) "netfs: " fmt
22
23 /*
24 * buffered_read.c
25 */
26 void netfs_queue_read(struct netfs_io_request *rreq,
27 struct netfs_io_subrequest *subreq);
28 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
29 int netfs_prefetch_for_write(struct file *file, struct folio *folio,
30 size_t offset, size_t len);
31
32 /*
33 * buffered_write.c
34 */
35 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
36 loff_t pos, size_t copied);
37
38 /*
39 * main.c
40 */
41 extern unsigned int netfs_debug;
42 extern struct list_head netfs_io_requests;
43 extern spinlock_t netfs_proc_lock;
44 extern mempool_t netfs_request_pool;
45 extern mempool_t netfs_subrequest_pool;
46
47 #ifdef CONFIG_PROC_FS
netfs_proc_add_rreq(struct netfs_io_request * rreq)48 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq)
49 {
50 spin_lock(&netfs_proc_lock);
51 list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests);
52 spin_unlock(&netfs_proc_lock);
53 }
netfs_proc_del_rreq(struct netfs_io_request * rreq)54 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq)
55 {
56 if (!list_empty(&rreq->proc_link)) {
57 spin_lock(&netfs_proc_lock);
58 list_del_rcu(&rreq->proc_link);
59 spin_unlock(&netfs_proc_lock);
60 }
61 }
62 #else
netfs_proc_add_rreq(struct netfs_io_request * rreq)63 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {}
netfs_proc_del_rreq(struct netfs_io_request * rreq)64 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {}
65 #endif
66
67 /*
68 * misc.c
69 */
70 struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq,
71 enum netfs_folioq_trace trace);
72 void netfs_reset_iter(struct netfs_io_subrequest *subreq);
73 void netfs_wake_collector(struct netfs_io_request *rreq);
74 void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq);
75 void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
76 struct netfs_io_stream *stream);
77 ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
78 ssize_t netfs_wait_for_write(struct netfs_io_request *rreq);
79 void netfs_wait_for_paused_read(struct netfs_io_request *rreq);
80 void netfs_wait_for_paused_write(struct netfs_io_request *rreq);
81
82 /*
83 * objects.c
84 */
85 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
86 struct file *file,
87 loff_t start, size_t len,
88 enum netfs_io_origin origin);
89 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
90 void netfs_clear_subrequests(struct netfs_io_request *rreq);
91 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
92 void netfs_put_failed_request(struct netfs_io_request *rreq);
93 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq);
94
netfs_see_request(struct netfs_io_request * rreq,enum netfs_rreq_ref_trace what)95 static inline void netfs_see_request(struct netfs_io_request *rreq,
96 enum netfs_rreq_ref_trace what)
97 {
98 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
99 }
100
netfs_see_subrequest(struct netfs_io_subrequest * subreq,enum netfs_sreq_ref_trace what)101 static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
102 enum netfs_sreq_ref_trace what)
103 {
104 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index,
105 refcount_read(&subreq->ref), what);
106 }
107
108 /*
109 * read_collect.c
110 */
111 bool netfs_read_collection(struct netfs_io_request *rreq);
112 void netfs_read_collection_worker(struct work_struct *work);
113 void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
114 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
115
116 /*
117 * read_pgpriv2.c
118 */
119 void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
120 void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
121 bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
122
123 /*
124 * read_retry.c
125 */
126 void netfs_retry_reads(struct netfs_io_request *rreq);
127 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
128
129 /*
130 * stats.c
131 */
132 #ifdef CONFIG_NETFS_STATS
133 extern atomic_t netfs_n_rh_dio_read;
134 extern atomic_t netfs_n_rh_readahead;
135 extern atomic_t netfs_n_rh_read_folio;
136 extern atomic_t netfs_n_rh_read_single;
137 extern atomic_t netfs_n_rh_rreq;
138 extern atomic_t netfs_n_rh_sreq;
139 extern atomic_t netfs_n_rh_download;
140 extern atomic_t netfs_n_rh_download_done;
141 extern atomic_t netfs_n_rh_download_failed;
142 extern atomic_t netfs_n_rh_download_instead;
143 extern atomic_t netfs_n_rh_read;
144 extern atomic_t netfs_n_rh_read_done;
145 extern atomic_t netfs_n_rh_read_failed;
146 extern atomic_t netfs_n_rh_zero;
147 extern atomic_t netfs_n_rh_short_read;
148 extern atomic_t netfs_n_rh_write;
149 extern atomic_t netfs_n_rh_write_begin;
150 extern atomic_t netfs_n_rh_write_done;
151 extern atomic_t netfs_n_rh_write_failed;
152 extern atomic_t netfs_n_rh_write_zskip;
153 extern atomic_t netfs_n_rh_retry_read_req;
154 extern atomic_t netfs_n_rh_retry_read_subreq;
155 extern atomic_t netfs_n_wh_buffered_write;
156 extern atomic_t netfs_n_wh_writethrough;
157 extern atomic_t netfs_n_wh_dio_write;
158 extern atomic_t netfs_n_wh_writepages;
159 extern atomic_t netfs_n_wh_copy_to_cache;
160 extern atomic_t netfs_n_wh_wstream_conflict;
161 extern atomic_t netfs_n_wh_upload;
162 extern atomic_t netfs_n_wh_upload_done;
163 extern atomic_t netfs_n_wh_upload_failed;
164 extern atomic_t netfs_n_wh_write;
165 extern atomic_t netfs_n_wh_write_done;
166 extern atomic_t netfs_n_wh_write_failed;
167 extern atomic_t netfs_n_wh_retry_write_req;
168 extern atomic_t netfs_n_wh_retry_write_subreq;
169 extern atomic_t netfs_n_wb_lock_skip;
170 extern atomic_t netfs_n_wb_lock_wait;
171 extern atomic_t netfs_n_folioq;
172
173 int netfs_stats_show(struct seq_file *m, void *v);
174
netfs_stat(atomic_t * stat)175 static inline void netfs_stat(atomic_t *stat)
176 {
177 atomic_inc(stat);
178 }
179
netfs_stat_d(atomic_t * stat)180 static inline void netfs_stat_d(atomic_t *stat)
181 {
182 atomic_dec(stat);
183 }
184
185 #else
186 #define netfs_stat(x) do {} while(0)
187 #define netfs_stat_d(x) do {} while(0)
188 #endif
189
190 /*
191 * write_collect.c
192 */
193 int netfs_folio_written_back(struct folio *folio);
194 bool netfs_write_collection(struct netfs_io_request *wreq);
195 void netfs_write_collection_worker(struct work_struct *work);
196
197 /*
198 * write_issue.c
199 */
200 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
201 struct file *file,
202 loff_t start,
203 enum netfs_io_origin origin);
204 void netfs_prepare_write(struct netfs_io_request *wreq,
205 struct netfs_io_stream *stream,
206 loff_t start);
207 void netfs_reissue_write(struct netfs_io_stream *stream,
208 struct netfs_io_subrequest *subreq,
209 struct iov_iter *source);
210 void netfs_issue_write(struct netfs_io_request *wreq,
211 struct netfs_io_stream *stream);
212 size_t netfs_advance_write(struct netfs_io_request *wreq,
213 struct netfs_io_stream *stream,
214 loff_t start, size_t len, bool to_eof);
215 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
216 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
217 struct folio *folio, size_t copied, bool to_page_end,
218 struct folio **writethrough_cache);
219 ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
220 struct folio *writethrough_cache);
221
222 /*
223 * write_retry.c
224 */
225 void netfs_retry_writes(struct netfs_io_request *wreq);
226
227 /*
228 * Miscellaneous functions.
229 */
netfs_is_cache_enabled(struct netfs_inode * ctx)230 static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx)
231 {
232 #if IS_ENABLED(CONFIG_FSCACHE)
233 struct fscache_cookie *cookie = ctx->cache;
234
235 return fscache_cookie_valid(cookie) && cookie->cache_priv &&
236 fscache_cookie_enabled(cookie);
237 #else
238 return false;
239 #endif
240 }
241
242 /*
243 * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap).
244 */
netfs_get_group(struct netfs_group * netfs_group)245 static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group)
246 {
247 if (netfs_group && netfs_group != NETFS_FOLIO_COPY_TO_CACHE)
248 refcount_inc(&netfs_group->ref);
249 return netfs_group;
250 }
251
252 /*
253 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
254 */
netfs_put_group(struct netfs_group * netfs_group)255 static inline void netfs_put_group(struct netfs_group *netfs_group)
256 {
257 if (netfs_group &&
258 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
259 refcount_dec_and_test(&netfs_group->ref))
260 netfs_group->free(netfs_group);
261 }
262
263 /*
264 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
265 */
netfs_put_group_many(struct netfs_group * netfs_group,int nr)266 static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr)
267 {
268 if (netfs_group &&
269 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
270 refcount_sub_and_test(nr, &netfs_group->ref))
271 netfs_group->free(netfs_group);
272 }
273
274 /*
275 * Clear and wake up a NETFS_RREQ_* flag bit on a request.
276 */
netfs_wake_rreq_flag(struct netfs_io_request * rreq,unsigned int rreq_flag,enum netfs_rreq_trace trace)277 static inline void netfs_wake_rreq_flag(struct netfs_io_request *rreq,
278 unsigned int rreq_flag,
279 enum netfs_rreq_trace trace)
280 {
281 if (test_bit(rreq_flag, &rreq->flags)) {
282 clear_bit_unlock(rreq_flag, &rreq->flags);
283 smp_mb__after_atomic(); /* Set flag before task state */
284 trace_netfs_rreq(rreq, trace);
285 wake_up(&rreq->waitq);
286 }
287 }
288
289 /*
290 * Test the NETFS_RREQ_IN_PROGRESS flag, inserting an appropriate barrier.
291 */
netfs_check_rreq_in_progress(const struct netfs_io_request * rreq)292 static inline bool netfs_check_rreq_in_progress(const struct netfs_io_request *rreq)
293 {
294 /* Order read of flags before read of anything else, such as error. */
295 return test_bit_acquire(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
296 }
297
298 /*
299 * Test the NETFS_SREQ_IN_PROGRESS flag, inserting an appropriate barrier.
300 */
netfs_check_subreq_in_progress(const struct netfs_io_subrequest * subreq)301 static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subrequest *subreq)
302 {
303 /* Order read of flags before read of anything else, such as error. */
304 return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
305 }
306
307 /*
308 * fscache-cache.c
309 */
310 #ifdef CONFIG_PROC_FS
311 extern const struct seq_operations fscache_caches_seq_ops;
312 #endif
313 bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
314 void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
315 struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache);
316 void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where);
317
fscache_cache_state(const struct fscache_cache * cache)318 static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache)
319 {
320 return smp_load_acquire(&cache->state);
321 }
322
fscache_cache_is_live(const struct fscache_cache * cache)323 static inline bool fscache_cache_is_live(const struct fscache_cache *cache)
324 {
325 return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE;
326 }
327
fscache_set_cache_state(struct fscache_cache * cache,enum fscache_cache_state new_state)328 static inline void fscache_set_cache_state(struct fscache_cache *cache,
329 enum fscache_cache_state new_state)
330 {
331 smp_store_release(&cache->state, new_state);
332
333 }
334
fscache_set_cache_state_maybe(struct fscache_cache * cache,enum fscache_cache_state old_state,enum fscache_cache_state new_state)335 static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache,
336 enum fscache_cache_state old_state,
337 enum fscache_cache_state new_state)
338 {
339 return try_cmpxchg_release(&cache->state, &old_state, new_state);
340 }
341
342 /*
343 * fscache-cookie.c
344 */
345 extern struct kmem_cache *fscache_cookie_jar;
346 #ifdef CONFIG_PROC_FS
347 extern const struct seq_operations fscache_cookies_seq_ops;
348 #endif
349 extern struct timer_list fscache_cookie_lru_timer;
350
351 extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix);
352 extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
353 enum fscache_access_trace why);
354
fscache_see_cookie(struct fscache_cookie * cookie,enum fscache_cookie_trace where)355 static inline void fscache_see_cookie(struct fscache_cookie *cookie,
356 enum fscache_cookie_trace where)
357 {
358 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
359 where);
360 }
361
362 /*
363 * fscache-main.c
364 */
365 extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len);
366 #ifdef CONFIG_FSCACHE
367 int __init fscache_init(void);
368 void __exit fscache_exit(void);
369 #else
fscache_init(void)370 static inline int fscache_init(void) { return 0; }
fscache_exit(void)371 static inline void fscache_exit(void) {}
372 #endif
373
374 /*
375 * fscache-proc.c
376 */
377 #ifdef CONFIG_PROC_FS
378 extern int __init fscache_proc_init(void);
379 extern void fscache_proc_cleanup(void);
380 #else
381 #define fscache_proc_init() (0)
382 #define fscache_proc_cleanup() do {} while (0)
383 #endif
384
385 /*
386 * fscache-stats.c
387 */
388 #ifdef CONFIG_FSCACHE_STATS
389 extern atomic_t fscache_n_volumes;
390 extern atomic_t fscache_n_volumes_collision;
391 extern atomic_t fscache_n_volumes_nomem;
392 extern atomic_t fscache_n_cookies;
393 extern atomic_t fscache_n_cookies_lru;
394 extern atomic_t fscache_n_cookies_lru_expired;
395 extern atomic_t fscache_n_cookies_lru_removed;
396 extern atomic_t fscache_n_cookies_lru_dropped;
397
398 extern atomic_t fscache_n_acquires;
399 extern atomic_t fscache_n_acquires_ok;
400 extern atomic_t fscache_n_acquires_oom;
401
402 extern atomic_t fscache_n_invalidates;
403
404 extern atomic_t fscache_n_relinquishes;
405 extern atomic_t fscache_n_relinquishes_retire;
406 extern atomic_t fscache_n_relinquishes_dropped;
407
408 extern atomic_t fscache_n_resizes;
409 extern atomic_t fscache_n_resizes_null;
410
fscache_stat(atomic_t * stat)411 static inline void fscache_stat(atomic_t *stat)
412 {
413 atomic_inc(stat);
414 }
415
fscache_stat_d(atomic_t * stat)416 static inline void fscache_stat_d(atomic_t *stat)
417 {
418 atomic_dec(stat);
419 }
420
421 #define __fscache_stat(stat) (stat)
422
423 int fscache_stats_show(struct seq_file *m);
424 #else
425
426 #define __fscache_stat(stat) (NULL)
427 #define fscache_stat(stat) do {} while (0)
428 #define fscache_stat_d(stat) do {} while (0)
429
fscache_stats_show(struct seq_file * m)430 static inline int fscache_stats_show(struct seq_file *m) { return 0; }
431 #endif
432
433 /*
434 * fscache-volume.c
435 */
436 #ifdef CONFIG_PROC_FS
437 extern const struct seq_operations fscache_volumes_seq_ops;
438 #endif
439
440 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
441 enum fscache_volume_trace where);
442 bool fscache_begin_volume_access(struct fscache_volume *volume,
443 struct fscache_cookie *cookie,
444 enum fscache_access_trace why);
445 void fscache_create_volume(struct fscache_volume *volume, bool wait);
446
447 /*****************************************************************************/
448 /*
449 * debug tracing
450 */
451 #define dbgprintk(FMT, ...) \
452 printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__)
453
454 #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__)
455 #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
456 #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__)
457
458 #ifdef __KDEBUG
459 #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__)
460 #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__)
461 #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__)
462
463 #elif defined(CONFIG_NETFS_DEBUG)
464 #define _enter(FMT, ...) \
465 do { \
466 if (netfs_debug) \
467 kenter(FMT, ##__VA_ARGS__); \
468 } while (0)
469
470 #define _leave(FMT, ...) \
471 do { \
472 if (netfs_debug) \
473 kleave(FMT, ##__VA_ARGS__); \
474 } while (0)
475
476 #define _debug(FMT, ...) \
477 do { \
478 if (netfs_debug) \
479 kdebug(FMT, ##__VA_ARGS__); \
480 } while (0)
481
482 #else
483 #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__)
484 #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
485 #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
486 #endif
487
488 /*
489 * assertions
490 */
491 #if 1 /* defined(__KDEBUGALL) */
492
493 #define ASSERT(X) \
494 do { \
495 if (unlikely(!(X))) { \
496 pr_err("\n"); \
497 pr_err("Assertion failed\n"); \
498 BUG(); \
499 } \
500 } while (0)
501
502 #define ASSERTCMP(X, OP, Y) \
503 do { \
504 if (unlikely(!((X) OP (Y)))) { \
505 pr_err("\n"); \
506 pr_err("Assertion failed\n"); \
507 pr_err("%lx " #OP " %lx is false\n", \
508 (unsigned long)(X), (unsigned long)(Y)); \
509 BUG(); \
510 } \
511 } while (0)
512
513 #define ASSERTIF(C, X) \
514 do { \
515 if (unlikely((C) && !(X))) { \
516 pr_err("\n"); \
517 pr_err("Assertion failed\n"); \
518 BUG(); \
519 } \
520 } while (0)
521
522 #define ASSERTIFCMP(C, X, OP, Y) \
523 do { \
524 if (unlikely((C) && !((X) OP (Y)))) { \
525 pr_err("\n"); \
526 pr_err("Assertion failed\n"); \
527 pr_err("%lx " #OP " %lx is false\n", \
528 (unsigned long)(X), (unsigned long)(Y)); \
529 BUG(); \
530 } \
531 } while (0)
532
533 #else
534
535 #define ASSERT(X) do {} while (0)
536 #define ASSERTCMP(X, OP, Y) do {} while (0)
537 #define ASSERTIF(C, X) do {} while (0)
538 #define ASSERTIFCMP(C, X, OP, Y) do {} while (0)
539
540 #endif /* assert or not */
541