1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* Internal definitions for network filesystem support
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/slab.h>
9 #include <linux/seq_file.h>
10 #include <linux/folio_queue.h>
11 #include <linux/netfs.h>
12 #include <linux/fscache.h>
13 #include <linux/fscache-cache.h>
14 #include <trace/events/netfs.h>
15 #include <trace/events/fscache.h>
16
17 #ifdef pr_fmt
18 #undef pr_fmt
19 #endif
20
21 #define pr_fmt(fmt) "netfs: " fmt
22
23 /*
24 * buffered_read.c
25 */
26 void netfs_queue_read(struct netfs_io_request *rreq,
27 struct netfs_io_subrequest *subreq);
28 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
29 int netfs_prefetch_for_write(struct file *file, struct folio *folio,
30 size_t offset, size_t len);
31
32 /*
33 * buffered_write.c
34 */
35 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
36 loff_t pos, size_t copied);
37
38 /*
39 * main.c
40 */
41 extern unsigned int netfs_debug;
42 extern struct list_head netfs_io_requests;
43 extern spinlock_t netfs_proc_lock;
44 extern mempool_t netfs_request_pool;
45 extern mempool_t netfs_subrequest_pool;
46 extern mempool_t netfs_folioq_pool;
47
48 #ifdef CONFIG_PROC_FS
netfs_proc_add_rreq(struct netfs_io_request * rreq)49 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq)
50 {
51 spin_lock(&netfs_proc_lock);
52 list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests);
53 spin_unlock(&netfs_proc_lock);
54 }
netfs_proc_del_rreq(struct netfs_io_request * rreq)55 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq)
56 {
57 if (!list_empty(&rreq->proc_link)) {
58 spin_lock(&netfs_proc_lock);
59 list_del_rcu(&rreq->proc_link);
60 spin_unlock(&netfs_proc_lock);
61 }
62 }
63 #else
netfs_proc_add_rreq(struct netfs_io_request * rreq)64 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {}
netfs_proc_del_rreq(struct netfs_io_request * rreq)65 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {}
66 #endif
67
68 /*
69 * misc.c
70 */
71 struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq,
72 enum netfs_folioq_trace trace);
73 void netfs_reset_iter(struct netfs_io_subrequest *subreq);
74 void netfs_wake_collector(struct netfs_io_request *rreq);
75 void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq);
76 void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
77 struct netfs_io_stream *stream);
78 ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
79 ssize_t netfs_wait_for_write(struct netfs_io_request *rreq);
80 void netfs_wait_for_paused_read(struct netfs_io_request *rreq);
81 void netfs_wait_for_paused_write(struct netfs_io_request *rreq);
82 void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq);
83
84 /*
85 * objects.c
86 */
87 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
88 struct file *file,
89 loff_t start, size_t len,
90 enum netfs_io_origin origin);
91 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
92 void netfs_clear_subrequests(struct netfs_io_request *rreq);
93 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
94 void netfs_put_failed_request(struct netfs_io_request *rreq);
95 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq);
96
netfs_see_request(struct netfs_io_request * rreq,enum netfs_rreq_ref_trace what)97 static inline void netfs_see_request(struct netfs_io_request *rreq,
98 enum netfs_rreq_ref_trace what)
99 {
100 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
101 }
102
netfs_see_subrequest(struct netfs_io_subrequest * subreq,enum netfs_sreq_ref_trace what)103 static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
104 enum netfs_sreq_ref_trace what)
105 {
106 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index,
107 refcount_read(&subreq->ref), what);
108 }
109
110 /*
111 * read_collect.c
112 */
113 void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
114 void netfs_read_set_unlock_at(struct netfs_io_request *rreq);
115 bool netfs_read_collection(struct netfs_io_request *rreq);
116 void netfs_read_collection_worker(struct work_struct *work);
117 void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
118 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
119
120 /*
121 * read_pgpriv2.c
122 */
123 void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
124 void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
125 bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
126
127 /*
128 * read_retry.c
129 */
130 void netfs_retry_reads(struct netfs_io_request *rreq);
131 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
132
133 /*
134 * stats.c
135 */
136 #ifdef CONFIG_NETFS_STATS
137 extern atomic_t netfs_n_rh_dio_read;
138 extern atomic_t netfs_n_rh_readahead;
139 extern atomic_t netfs_n_rh_read_folio;
140 extern atomic_t netfs_n_rh_read_single;
141 extern atomic_t netfs_n_rh_rreq;
142 extern atomic_t netfs_n_rh_sreq;
143 extern atomic_t netfs_n_rh_download;
144 extern atomic_t netfs_n_rh_download_done;
145 extern atomic_t netfs_n_rh_download_failed;
146 extern atomic_t netfs_n_rh_download_instead;
147 extern atomic_t netfs_n_rh_read;
148 extern atomic_t netfs_n_rh_read_done;
149 extern atomic_t netfs_n_rh_read_failed;
150 extern atomic_t netfs_n_rh_zero;
151 extern atomic_t netfs_n_rh_short_read;
152 extern atomic_t netfs_n_rh_write;
153 extern atomic_t netfs_n_rh_write_begin;
154 extern atomic_t netfs_n_rh_write_done;
155 extern atomic_t netfs_n_rh_write_failed;
156 extern atomic_t netfs_n_rh_write_zskip;
157 extern atomic_t netfs_n_rh_retry_read_req;
158 extern atomic_t netfs_n_rh_retry_read_subreq;
159 extern atomic_t netfs_n_wh_buffered_write;
160 extern atomic_t netfs_n_wh_writethrough;
161 extern atomic_t netfs_n_wh_dio_write;
162 extern atomic_t netfs_n_wh_writepages;
163 extern atomic_t netfs_n_wh_copy_to_cache;
164 extern atomic_t netfs_n_wh_wstream_conflict;
165 extern atomic_t netfs_n_wh_upload;
166 extern atomic_t netfs_n_wh_upload_done;
167 extern atomic_t netfs_n_wh_upload_failed;
168 extern atomic_t netfs_n_wh_write;
169 extern atomic_t netfs_n_wh_write_done;
170 extern atomic_t netfs_n_wh_write_failed;
171 extern atomic_t netfs_n_wh_retry_write_req;
172 extern atomic_t netfs_n_wh_retry_write_subreq;
173 extern atomic_t netfs_n_wb_lock_skip;
174 extern atomic_t netfs_n_wb_lock_wait;
175 extern atomic_t netfs_n_folioq;
176
177 int netfs_stats_show(struct seq_file *m, void *v);
178
netfs_stat(atomic_t * stat)179 static inline void netfs_stat(atomic_t *stat)
180 {
181 atomic_inc(stat);
182 }
183
netfs_stat_d(atomic_t * stat)184 static inline void netfs_stat_d(atomic_t *stat)
185 {
186 atomic_dec(stat);
187 }
188
189 #else
190 #define netfs_stat(x) do {} while(0)
191 #define netfs_stat_d(x) do {} while(0)
192 #endif
193
194 /*
195 * write_collect.c
196 */
197 int netfs_folio_written_back(struct folio *folio);
198 bool netfs_write_collection(struct netfs_io_request *wreq);
199 void netfs_write_collection_worker(struct work_struct *work);
200
201 /*
202 * write_issue.c
203 */
204 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
205 struct file *file,
206 loff_t start,
207 enum netfs_io_origin origin);
208 void netfs_prepare_write(struct netfs_io_request *wreq,
209 struct netfs_io_stream *stream,
210 loff_t start);
211 void netfs_reissue_write(struct netfs_io_stream *stream,
212 struct netfs_io_subrequest *subreq,
213 struct iov_iter *source);
214 void netfs_issue_write(struct netfs_io_request *wreq,
215 struct netfs_io_stream *stream);
216 size_t netfs_advance_write(struct netfs_io_request *wreq,
217 struct netfs_io_stream *stream,
218 loff_t start, size_t len, bool to_eof);
219 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
220 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
221 struct folio *folio, size_t copied, bool to_page_end,
222 struct folio **writethrough_cache);
223 ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
224 struct folio *writethrough_cache);
225
226 /*
227 * write_retry.c
228 */
229 void netfs_retry_writes(struct netfs_io_request *wreq);
230
231 /*
232 * Miscellaneous functions.
233 */
netfs_is_cache_enabled(struct netfs_inode * ctx)234 static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx)
235 {
236 #if IS_ENABLED(CONFIG_FSCACHE)
237 struct fscache_cookie *cookie = ctx->cache;
238
239 return fscache_cookie_valid(cookie) && cookie->cache_priv &&
240 fscache_cookie_enabled(cookie);
241 #else
242 return false;
243 #endif
244 }
245
netfs_is_cache_maybe_enabled(struct netfs_inode * ctx)246 static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx)
247 {
248 #if IS_ENABLED(CONFIG_FSCACHE)
249 struct fscache_cookie *cookie = ctx->cache;
250
251 return fscache_cookie_valid(cookie) &&
252 test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
253 #else
254 return false;
255 #endif
256 }
257
258 /*
259 * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap).
260 */
netfs_get_group(struct netfs_group * netfs_group)261 static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group)
262 {
263 if (netfs_group && netfs_group != NETFS_FOLIO_COPY_TO_CACHE)
264 refcount_inc(&netfs_group->ref);
265 return netfs_group;
266 }
267
268 /*
269 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
270 */
netfs_put_group(struct netfs_group * netfs_group)271 static inline void netfs_put_group(struct netfs_group *netfs_group)
272 {
273 if (netfs_group &&
274 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
275 refcount_dec_and_test(&netfs_group->ref))
276 netfs_group->free(netfs_group);
277 }
278
279 /*
280 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
281 */
netfs_put_group_many(struct netfs_group * netfs_group,int nr)282 static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr)
283 {
284 if (netfs_group &&
285 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
286 refcount_sub_and_test(nr, &netfs_group->ref))
287 netfs_group->free(netfs_group);
288 }
289
290 /*
291 * Clear and wake up a NETFS_RREQ_* flag bit on a request.
292 */
netfs_wake_rreq_flag(struct netfs_io_request * rreq,unsigned int rreq_flag,enum netfs_rreq_trace trace)293 static inline void netfs_wake_rreq_flag(struct netfs_io_request *rreq,
294 unsigned int rreq_flag,
295 enum netfs_rreq_trace trace)
296 {
297 if (test_bit(rreq_flag, &rreq->flags)) {
298 clear_bit_unlock(rreq_flag, &rreq->flags);
299 smp_mb__after_atomic(); /* Set flag before task state */
300 trace_netfs_rreq(rreq, trace);
301 wake_up(&rreq->waitq);
302 }
303 }
304
305 /*
306 * Test the NETFS_RREQ_IN_PROGRESS flag, inserting an appropriate barrier.
307 */
netfs_check_rreq_in_progress(const struct netfs_io_request * rreq)308 static inline bool netfs_check_rreq_in_progress(const struct netfs_io_request *rreq)
309 {
310 /* Order read of flags before read of anything else, such as error. */
311 return test_bit_acquire(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
312 }
313
314 /*
315 * Test the NETFS_SREQ_IN_PROGRESS flag, inserting an appropriate barrier.
316 */
netfs_check_subreq_in_progress(const struct netfs_io_subrequest * subreq)317 static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subrequest *subreq)
318 {
319 /* Order read of flags before read of anything else, such as error. */
320 return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
321 }
322
323 /*
324 * fscache-cache.c
325 */
326 #ifdef CONFIG_PROC_FS
327 extern const struct seq_operations fscache_caches_seq_ops;
328 #endif
329 bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
330 void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
331 struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache);
332 void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where);
333
fscache_cache_state(const struct fscache_cache * cache)334 static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache)
335 {
336 return smp_load_acquire(&cache->state);
337 }
338
fscache_cache_is_live(const struct fscache_cache * cache)339 static inline bool fscache_cache_is_live(const struct fscache_cache *cache)
340 {
341 return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE;
342 }
343
fscache_set_cache_state(struct fscache_cache * cache,enum fscache_cache_state new_state)344 static inline void fscache_set_cache_state(struct fscache_cache *cache,
345 enum fscache_cache_state new_state)
346 {
347 smp_store_release(&cache->state, new_state);
348
349 }
350
fscache_set_cache_state_maybe(struct fscache_cache * cache,enum fscache_cache_state old_state,enum fscache_cache_state new_state)351 static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache,
352 enum fscache_cache_state old_state,
353 enum fscache_cache_state new_state)
354 {
355 return try_cmpxchg_release(&cache->state, &old_state, new_state);
356 }
357
358 /*
359 * fscache-cookie.c
360 */
361 extern struct kmem_cache *fscache_cookie_jar;
362 #ifdef CONFIG_PROC_FS
363 extern const struct seq_operations fscache_cookies_seq_ops;
364 #endif
365 extern struct timer_list fscache_cookie_lru_timer;
366
367 extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix);
368 extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
369 enum fscache_access_trace why);
370
fscache_see_cookie(struct fscache_cookie * cookie,enum fscache_cookie_trace where)371 static inline void fscache_see_cookie(struct fscache_cookie *cookie,
372 enum fscache_cookie_trace where)
373 {
374 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
375 where);
376 }
377
378 /*
379 * fscache-main.c
380 */
381 extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len);
382 #ifdef CONFIG_FSCACHE
383 int __init fscache_init(void);
384 void __exit fscache_exit(void);
385 #else
fscache_init(void)386 static inline int fscache_init(void) { return 0; }
fscache_exit(void)387 static inline void fscache_exit(void) {}
388 #endif
389
390 /*
391 * fscache-proc.c
392 */
393 #ifdef CONFIG_PROC_FS
394 extern int __init fscache_proc_init(void);
395 extern void fscache_proc_cleanup(void);
396 #else
397 #define fscache_proc_init() (0)
398 #define fscache_proc_cleanup() do {} while (0)
399 #endif
400
401 /*
402 * fscache-stats.c
403 */
404 #ifdef CONFIG_FSCACHE_STATS
405 extern atomic_t fscache_n_volumes;
406 extern atomic_t fscache_n_volumes_collision;
407 extern atomic_t fscache_n_volumes_nomem;
408 extern atomic_t fscache_n_cookies;
409 extern atomic_t fscache_n_cookies_lru;
410 extern atomic_t fscache_n_cookies_lru_expired;
411 extern atomic_t fscache_n_cookies_lru_removed;
412 extern atomic_t fscache_n_cookies_lru_dropped;
413
414 extern atomic_t fscache_n_acquires;
415 extern atomic_t fscache_n_acquires_ok;
416 extern atomic_t fscache_n_acquires_oom;
417
418 extern atomic_t fscache_n_invalidates;
419
420 extern atomic_t fscache_n_relinquishes;
421 extern atomic_t fscache_n_relinquishes_retire;
422 extern atomic_t fscache_n_relinquishes_dropped;
423
424 extern atomic_t fscache_n_resizes;
425 extern atomic_t fscache_n_resizes_null;
426
fscache_stat(atomic_t * stat)427 static inline void fscache_stat(atomic_t *stat)
428 {
429 atomic_inc(stat);
430 }
431
fscache_stat_d(atomic_t * stat)432 static inline void fscache_stat_d(atomic_t *stat)
433 {
434 atomic_dec(stat);
435 }
436
437 #define __fscache_stat(stat) (stat)
438
439 int fscache_stats_show(struct seq_file *m);
440 #else
441
442 #define __fscache_stat(stat) (NULL)
443 #define fscache_stat(stat) do {} while (0)
444 #define fscache_stat_d(stat) do {} while (0)
445
fscache_stats_show(struct seq_file * m)446 static inline int fscache_stats_show(struct seq_file *m) { return 0; }
447 #endif
448
449 /*
450 * fscache-volume.c
451 */
452 #ifdef CONFIG_PROC_FS
453 extern const struct seq_operations fscache_volumes_seq_ops;
454 #endif
455
456 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
457 enum fscache_volume_trace where);
458 bool fscache_begin_volume_access(struct fscache_volume *volume,
459 struct fscache_cookie *cookie,
460 enum fscache_access_trace why);
461 void fscache_create_volume(struct fscache_volume *volume, bool wait);
462
463 /*****************************************************************************/
464 /*
465 * debug tracing
466 */
467 #define dbgprintk(FMT, ...) \
468 printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__)
469
470 #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__)
471 #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
472 #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__)
473
474 #ifdef __KDEBUG
475 #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__)
476 #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__)
477 #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__)
478
479 #elif defined(CONFIG_NETFS_DEBUG)
480 #define _enter(FMT, ...) \
481 do { \
482 if (netfs_debug) \
483 kenter(FMT, ##__VA_ARGS__); \
484 } while (0)
485
486 #define _leave(FMT, ...) \
487 do { \
488 if (netfs_debug) \
489 kleave(FMT, ##__VA_ARGS__); \
490 } while (0)
491
492 #define _debug(FMT, ...) \
493 do { \
494 if (netfs_debug) \
495 kdebug(FMT, ##__VA_ARGS__); \
496 } while (0)
497
498 #else
499 #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__)
500 #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
501 #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
502 #endif
503
504 /*
505 * assertions
506 */
507 #if 1 /* defined(__KDEBUGALL) */
508
509 #define ASSERT(X) \
510 do { \
511 if (unlikely(!(X))) { \
512 pr_err("\n"); \
513 pr_err("Assertion failed\n"); \
514 BUG(); \
515 } \
516 } while (0)
517
518 #define ASSERTCMP(X, OP, Y) \
519 do { \
520 if (unlikely(!((X) OP (Y)))) { \
521 pr_err("\n"); \
522 pr_err("Assertion failed\n"); \
523 pr_err("%lx " #OP " %lx is false\n", \
524 (unsigned long)(X), (unsigned long)(Y)); \
525 BUG(); \
526 } \
527 } while (0)
528
529 #define ASSERTIF(C, X) \
530 do { \
531 if (unlikely((C) && !(X))) { \
532 pr_err("\n"); \
533 pr_err("Assertion failed\n"); \
534 BUG(); \
535 } \
536 } while (0)
537
538 #define ASSERTIFCMP(C, X, OP, Y) \
539 do { \
540 if (unlikely((C) && !((X) OP (Y)))) { \
541 pr_err("\n"); \
542 pr_err("Assertion failed\n"); \
543 pr_err("%lx " #OP " %lx is false\n", \
544 (unsigned long)(X), (unsigned long)(Y)); \
545 BUG(); \
546 } \
547 } while (0)
548
549 #else
550
551 #define ASSERT(X) do {} while (0)
552 #define ASSERTCMP(X, OP, Y) do {} while (0)
553 #define ASSERTIF(C, X) do {} while (0)
554 #define ASSERTIFCMP(C, X, OP, Y) do {} while (0)
555
556 #endif /* assert or not */
557