1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* Internal definitions for network filesystem support
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/slab.h>
9 #include <linux/seq_file.h>
10 #include <linux/folio_queue.h>
11 #include <linux/netfs.h>
12 #include <linux/fscache.h>
13 #include <linux/fscache-cache.h>
14 #include <trace/events/netfs.h>
15 #include <trace/events/fscache.h>
16
17 #ifdef pr_fmt
18 #undef pr_fmt
19 #endif
20
21 #define pr_fmt(fmt) "netfs: " fmt
22
23 /*
24 * buffered_read.c
25 */
26 void netfs_queue_read(struct netfs_io_request *rreq,
27 struct netfs_io_subrequest *subreq);
28 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
29 int netfs_prefetch_for_write(struct file *file, struct folio *folio,
30 size_t offset, size_t len);
31
32 /*
33 * buffered_write.c
34 */
35 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
36 loff_t pos, size_t copied);
37
38 /*
39 * main.c
40 */
41 extern unsigned int netfs_debug;
42 extern struct list_head netfs_io_requests;
43 extern spinlock_t netfs_proc_lock;
44 extern mempool_t netfs_request_pool;
45 extern mempool_t netfs_subrequest_pool;
46 extern mempool_t netfs_folioq_pool;
47
48 #ifdef CONFIG_PROC_FS
netfs_proc_add_rreq(struct netfs_io_request * rreq)49 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq)
50 {
51 spin_lock(&netfs_proc_lock);
52 list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests);
53 spin_unlock(&netfs_proc_lock);
54 }
netfs_proc_del_rreq(struct netfs_io_request * rreq)55 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq)
56 {
57 if (!list_empty(&rreq->proc_link)) {
58 spin_lock(&netfs_proc_lock);
59 list_del_rcu(&rreq->proc_link);
60 spin_unlock(&netfs_proc_lock);
61 }
62 }
63 #else
netfs_proc_add_rreq(struct netfs_io_request * rreq)64 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {}
netfs_proc_del_rreq(struct netfs_io_request * rreq)65 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {}
66 #endif
67
68 /*
69 * misc.c
70 */
71 struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq,
72 enum netfs_folioq_trace trace);
73 void netfs_reset_iter(struct netfs_io_subrequest *subreq);
74 void netfs_wake_collector(struct netfs_io_request *rreq);
75 void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq);
76 void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
77 struct netfs_io_stream *stream);
78 ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
79 ssize_t netfs_wait_for_write(struct netfs_io_request *rreq);
80 void netfs_wait_for_paused_read(struct netfs_io_request *rreq);
81 void netfs_wait_for_paused_write(struct netfs_io_request *rreq);
82
83 /*
84 * objects.c
85 */
86 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
87 struct file *file,
88 loff_t start, size_t len,
89 enum netfs_io_origin origin);
90 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
91 void netfs_clear_subrequests(struct netfs_io_request *rreq);
92 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
93 void netfs_put_failed_request(struct netfs_io_request *rreq);
94 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq);
95
netfs_see_request(struct netfs_io_request * rreq,enum netfs_rreq_ref_trace what)96 static inline void netfs_see_request(struct netfs_io_request *rreq,
97 enum netfs_rreq_ref_trace what)
98 {
99 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
100 }
101
netfs_see_subrequest(struct netfs_io_subrequest * subreq,enum netfs_sreq_ref_trace what)102 static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
103 enum netfs_sreq_ref_trace what)
104 {
105 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index,
106 refcount_read(&subreq->ref), what);
107 }
108
109 /*
110 * read_collect.c
111 */
112 bool netfs_read_collection(struct netfs_io_request *rreq);
113 void netfs_read_collection_worker(struct work_struct *work);
114 void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
115 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
116
117 /*
118 * read_pgpriv2.c
119 */
120 void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
121 void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
122 bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
123
124 /*
125 * read_retry.c
126 */
127 void netfs_retry_reads(struct netfs_io_request *rreq);
128 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
129
130 /*
131 * stats.c
132 */
133 #ifdef CONFIG_NETFS_STATS
134 extern atomic_t netfs_n_rh_dio_read;
135 extern atomic_t netfs_n_rh_readahead;
136 extern atomic_t netfs_n_rh_read_folio;
137 extern atomic_t netfs_n_rh_read_single;
138 extern atomic_t netfs_n_rh_rreq;
139 extern atomic_t netfs_n_rh_sreq;
140 extern atomic_t netfs_n_rh_download;
141 extern atomic_t netfs_n_rh_download_done;
142 extern atomic_t netfs_n_rh_download_failed;
143 extern atomic_t netfs_n_rh_download_instead;
144 extern atomic_t netfs_n_rh_read;
145 extern atomic_t netfs_n_rh_read_done;
146 extern atomic_t netfs_n_rh_read_failed;
147 extern atomic_t netfs_n_rh_zero;
148 extern atomic_t netfs_n_rh_short_read;
149 extern atomic_t netfs_n_rh_write;
150 extern atomic_t netfs_n_rh_write_begin;
151 extern atomic_t netfs_n_rh_write_done;
152 extern atomic_t netfs_n_rh_write_failed;
153 extern atomic_t netfs_n_rh_write_zskip;
154 extern atomic_t netfs_n_rh_retry_read_req;
155 extern atomic_t netfs_n_rh_retry_read_subreq;
156 extern atomic_t netfs_n_wh_buffered_write;
157 extern atomic_t netfs_n_wh_writethrough;
158 extern atomic_t netfs_n_wh_dio_write;
159 extern atomic_t netfs_n_wh_writepages;
160 extern atomic_t netfs_n_wh_copy_to_cache;
161 extern atomic_t netfs_n_wh_wstream_conflict;
162 extern atomic_t netfs_n_wh_upload;
163 extern atomic_t netfs_n_wh_upload_done;
164 extern atomic_t netfs_n_wh_upload_failed;
165 extern atomic_t netfs_n_wh_write;
166 extern atomic_t netfs_n_wh_write_done;
167 extern atomic_t netfs_n_wh_write_failed;
168 extern atomic_t netfs_n_wh_retry_write_req;
169 extern atomic_t netfs_n_wh_retry_write_subreq;
170 extern atomic_t netfs_n_wb_lock_skip;
171 extern atomic_t netfs_n_wb_lock_wait;
172 extern atomic_t netfs_n_folioq;
173
174 int netfs_stats_show(struct seq_file *m, void *v);
175
netfs_stat(atomic_t * stat)176 static inline void netfs_stat(atomic_t *stat)
177 {
178 atomic_inc(stat);
179 }
180
netfs_stat_d(atomic_t * stat)181 static inline void netfs_stat_d(atomic_t *stat)
182 {
183 atomic_dec(stat);
184 }
185
186 #else
187 #define netfs_stat(x) do {} while(0)
188 #define netfs_stat_d(x) do {} while(0)
189 #endif
190
191 /*
192 * write_collect.c
193 */
194 int netfs_folio_written_back(struct folio *folio);
195 bool netfs_write_collection(struct netfs_io_request *wreq);
196 void netfs_write_collection_worker(struct work_struct *work);
197
198 /*
199 * write_issue.c
200 */
201 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
202 struct file *file,
203 loff_t start,
204 enum netfs_io_origin origin);
205 void netfs_prepare_write(struct netfs_io_request *wreq,
206 struct netfs_io_stream *stream,
207 loff_t start);
208 void netfs_reissue_write(struct netfs_io_stream *stream,
209 struct netfs_io_subrequest *subreq,
210 struct iov_iter *source);
211 void netfs_issue_write(struct netfs_io_request *wreq,
212 struct netfs_io_stream *stream);
213 size_t netfs_advance_write(struct netfs_io_request *wreq,
214 struct netfs_io_stream *stream,
215 loff_t start, size_t len, bool to_eof);
216 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
217 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
218 struct folio *folio, size_t copied, bool to_page_end,
219 struct folio **writethrough_cache);
220 ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
221 struct folio *writethrough_cache);
222
223 /*
224 * write_retry.c
225 */
226 void netfs_retry_writes(struct netfs_io_request *wreq);
227
228 /*
229 * Miscellaneous functions.
230 */
netfs_is_cache_enabled(struct netfs_inode * ctx)231 static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx)
232 {
233 #if IS_ENABLED(CONFIG_FSCACHE)
234 struct fscache_cookie *cookie = ctx->cache;
235
236 return fscache_cookie_valid(cookie) && cookie->cache_priv &&
237 fscache_cookie_enabled(cookie);
238 #else
239 return false;
240 #endif
241 }
242
netfs_is_cache_maybe_enabled(struct netfs_inode * ctx)243 static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx)
244 {
245 #if IS_ENABLED(CONFIG_FSCACHE)
246 struct fscache_cookie *cookie = ctx->cache;
247
248 return fscache_cookie_valid(cookie) &&
249 test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
250 #else
251 return false;
252 #endif
253 }
254
255 /*
256 * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap).
257 */
netfs_get_group(struct netfs_group * netfs_group)258 static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group)
259 {
260 if (netfs_group && netfs_group != NETFS_FOLIO_COPY_TO_CACHE)
261 refcount_inc(&netfs_group->ref);
262 return netfs_group;
263 }
264
265 /*
266 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
267 */
netfs_put_group(struct netfs_group * netfs_group)268 static inline void netfs_put_group(struct netfs_group *netfs_group)
269 {
270 if (netfs_group &&
271 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
272 refcount_dec_and_test(&netfs_group->ref))
273 netfs_group->free(netfs_group);
274 }
275
276 /*
277 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
278 */
netfs_put_group_many(struct netfs_group * netfs_group,int nr)279 static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr)
280 {
281 if (netfs_group &&
282 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
283 refcount_sub_and_test(nr, &netfs_group->ref))
284 netfs_group->free(netfs_group);
285 }
286
287 /*
288 * Clear and wake up a NETFS_RREQ_* flag bit on a request.
289 */
netfs_wake_rreq_flag(struct netfs_io_request * rreq,unsigned int rreq_flag,enum netfs_rreq_trace trace)290 static inline void netfs_wake_rreq_flag(struct netfs_io_request *rreq,
291 unsigned int rreq_flag,
292 enum netfs_rreq_trace trace)
293 {
294 if (test_bit(rreq_flag, &rreq->flags)) {
295 clear_bit_unlock(rreq_flag, &rreq->flags);
296 smp_mb__after_atomic(); /* Set flag before task state */
297 trace_netfs_rreq(rreq, trace);
298 wake_up(&rreq->waitq);
299 }
300 }
301
302 /*
303 * Test the NETFS_RREQ_IN_PROGRESS flag, inserting an appropriate barrier.
304 */
netfs_check_rreq_in_progress(const struct netfs_io_request * rreq)305 static inline bool netfs_check_rreq_in_progress(const struct netfs_io_request *rreq)
306 {
307 /* Order read of flags before read of anything else, such as error. */
308 return test_bit_acquire(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
309 }
310
311 /*
312 * Test the NETFS_SREQ_IN_PROGRESS flag, inserting an appropriate barrier.
313 */
netfs_check_subreq_in_progress(const struct netfs_io_subrequest * subreq)314 static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subrequest *subreq)
315 {
316 /* Order read of flags before read of anything else, such as error. */
317 return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
318 }
319
320 /*
321 * fscache-cache.c
322 */
323 #ifdef CONFIG_PROC_FS
324 extern const struct seq_operations fscache_caches_seq_ops;
325 #endif
326 bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
327 void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
328 struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache);
329 void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where);
330
fscache_cache_state(const struct fscache_cache * cache)331 static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache)
332 {
333 return smp_load_acquire(&cache->state);
334 }
335
fscache_cache_is_live(const struct fscache_cache * cache)336 static inline bool fscache_cache_is_live(const struct fscache_cache *cache)
337 {
338 return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE;
339 }
340
fscache_set_cache_state(struct fscache_cache * cache,enum fscache_cache_state new_state)341 static inline void fscache_set_cache_state(struct fscache_cache *cache,
342 enum fscache_cache_state new_state)
343 {
344 smp_store_release(&cache->state, new_state);
345
346 }
347
fscache_set_cache_state_maybe(struct fscache_cache * cache,enum fscache_cache_state old_state,enum fscache_cache_state new_state)348 static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache,
349 enum fscache_cache_state old_state,
350 enum fscache_cache_state new_state)
351 {
352 return try_cmpxchg_release(&cache->state, &old_state, new_state);
353 }
354
355 /*
356 * fscache-cookie.c
357 */
358 extern struct kmem_cache *fscache_cookie_jar;
359 #ifdef CONFIG_PROC_FS
360 extern const struct seq_operations fscache_cookies_seq_ops;
361 #endif
362 extern struct timer_list fscache_cookie_lru_timer;
363
364 extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix);
365 extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
366 enum fscache_access_trace why);
367
fscache_see_cookie(struct fscache_cookie * cookie,enum fscache_cookie_trace where)368 static inline void fscache_see_cookie(struct fscache_cookie *cookie,
369 enum fscache_cookie_trace where)
370 {
371 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
372 where);
373 }
374
375 /*
376 * fscache-main.c
377 */
378 extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len);
379 #ifdef CONFIG_FSCACHE
380 int __init fscache_init(void);
381 void __exit fscache_exit(void);
382 #else
fscache_init(void)383 static inline int fscache_init(void) { return 0; }
fscache_exit(void)384 static inline void fscache_exit(void) {}
385 #endif
386
387 /*
388 * fscache-proc.c
389 */
390 #ifdef CONFIG_PROC_FS
391 extern int __init fscache_proc_init(void);
392 extern void fscache_proc_cleanup(void);
393 #else
394 #define fscache_proc_init() (0)
395 #define fscache_proc_cleanup() do {} while (0)
396 #endif
397
398 /*
399 * fscache-stats.c
400 */
401 #ifdef CONFIG_FSCACHE_STATS
402 extern atomic_t fscache_n_volumes;
403 extern atomic_t fscache_n_volumes_collision;
404 extern atomic_t fscache_n_volumes_nomem;
405 extern atomic_t fscache_n_cookies;
406 extern atomic_t fscache_n_cookies_lru;
407 extern atomic_t fscache_n_cookies_lru_expired;
408 extern atomic_t fscache_n_cookies_lru_removed;
409 extern atomic_t fscache_n_cookies_lru_dropped;
410
411 extern atomic_t fscache_n_acquires;
412 extern atomic_t fscache_n_acquires_ok;
413 extern atomic_t fscache_n_acquires_oom;
414
415 extern atomic_t fscache_n_invalidates;
416
417 extern atomic_t fscache_n_relinquishes;
418 extern atomic_t fscache_n_relinquishes_retire;
419 extern atomic_t fscache_n_relinquishes_dropped;
420
421 extern atomic_t fscache_n_resizes;
422 extern atomic_t fscache_n_resizes_null;
423
fscache_stat(atomic_t * stat)424 static inline void fscache_stat(atomic_t *stat)
425 {
426 atomic_inc(stat);
427 }
428
fscache_stat_d(atomic_t * stat)429 static inline void fscache_stat_d(atomic_t *stat)
430 {
431 atomic_dec(stat);
432 }
433
434 #define __fscache_stat(stat) (stat)
435
436 int fscache_stats_show(struct seq_file *m);
437 #else
438
439 #define __fscache_stat(stat) (NULL)
440 #define fscache_stat(stat) do {} while (0)
441 #define fscache_stat_d(stat) do {} while (0)
442
fscache_stats_show(struct seq_file * m)443 static inline int fscache_stats_show(struct seq_file *m) { return 0; }
444 #endif
445
446 /*
447 * fscache-volume.c
448 */
449 #ifdef CONFIG_PROC_FS
450 extern const struct seq_operations fscache_volumes_seq_ops;
451 #endif
452
453 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
454 enum fscache_volume_trace where);
455 bool fscache_begin_volume_access(struct fscache_volume *volume,
456 struct fscache_cookie *cookie,
457 enum fscache_access_trace why);
458 void fscache_create_volume(struct fscache_volume *volume, bool wait);
459
460 /*****************************************************************************/
461 /*
462 * debug tracing
463 */
464 #define dbgprintk(FMT, ...) \
465 printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__)
466
467 #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__)
468 #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
469 #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__)
470
471 #ifdef __KDEBUG
472 #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__)
473 #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__)
474 #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__)
475
476 #elif defined(CONFIG_NETFS_DEBUG)
477 #define _enter(FMT, ...) \
478 do { \
479 if (netfs_debug) \
480 kenter(FMT, ##__VA_ARGS__); \
481 } while (0)
482
483 #define _leave(FMT, ...) \
484 do { \
485 if (netfs_debug) \
486 kleave(FMT, ##__VA_ARGS__); \
487 } while (0)
488
489 #define _debug(FMT, ...) \
490 do { \
491 if (netfs_debug) \
492 kdebug(FMT, ##__VA_ARGS__); \
493 } while (0)
494
495 #else
496 #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__)
497 #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
498 #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
499 #endif
500
501 /*
502 * assertions
503 */
504 #if 1 /* defined(__KDEBUGALL) */
505
506 #define ASSERT(X) \
507 do { \
508 if (unlikely(!(X))) { \
509 pr_err("\n"); \
510 pr_err("Assertion failed\n"); \
511 BUG(); \
512 } \
513 } while (0)
514
515 #define ASSERTCMP(X, OP, Y) \
516 do { \
517 if (unlikely(!((X) OP (Y)))) { \
518 pr_err("\n"); \
519 pr_err("Assertion failed\n"); \
520 pr_err("%lx " #OP " %lx is false\n", \
521 (unsigned long)(X), (unsigned long)(Y)); \
522 BUG(); \
523 } \
524 } while (0)
525
526 #define ASSERTIF(C, X) \
527 do { \
528 if (unlikely((C) && !(X))) { \
529 pr_err("\n"); \
530 pr_err("Assertion failed\n"); \
531 BUG(); \
532 } \
533 } while (0)
534
535 #define ASSERTIFCMP(C, X, OP, Y) \
536 do { \
537 if (unlikely((C) && !((X) OP (Y)))) { \
538 pr_err("\n"); \
539 pr_err("Assertion failed\n"); \
540 pr_err("%lx " #OP " %lx is false\n", \
541 (unsigned long)(X), (unsigned long)(Y)); \
542 BUG(); \
543 } \
544 } while (0)
545
546 #else
547
548 #define ASSERT(X) do {} while (0)
549 #define ASSERTCMP(X, OP, Y) do {} while (0)
550 #define ASSERTIF(C, X) do {} while (0)
551 #define ASSERTIFCMP(C, X, OP, Y) do {} while (0)
552
553 #endif /* assert or not */
554