1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * The NFSD open file cache. 4 * 5 * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com> 6 * 7 * An nfsd_file object is a per-file collection of open state that binds 8 * together: 9 * - a struct file * 10 * - a user credential 11 * - a network namespace 12 * - a read-ahead context 13 * - monitoring for writeback errors 14 * 15 * nfsd_file objects are reference-counted. Consumers acquire a new 16 * object via the nfsd_file_acquire API. They manage their interest in 17 * the acquired object, and hence the object's reference count, via 18 * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file 19 * object: 20 * 21 * * non-garbage-collected: When a consumer wants to precisely control 22 * the lifetime of a file's open state, it acquires a non-garbage- 23 * collected nfsd_file. The final nfsd_file_put releases the open 24 * state immediately. 25 * 26 * * garbage-collected: When a consumer does not control the lifetime 27 * of open state, it acquires a garbage-collected nfsd_file. The 28 * final nfsd_file_put allows the open state to linger for a period 29 * during which it may be re-used. 30 */ 31 32 #include <linux/hash.h> 33 #include <linux/slab.h> 34 #include <linux/file.h> 35 #include <linux/pagemap.h> 36 #include <linux/sched.h> 37 #include <linux/list_lru.h> 38 #include <linux/fsnotify_backend.h> 39 #include <linux/fsnotify.h> 40 #include <linux/seq_file.h> 41 #include <linux/rhashtable.h> 42 43 #include "vfs.h" 44 #include "nfsd.h" 45 #include "nfsfh.h" 46 #include "netns.h" 47 #include "filecache.h" 48 #include "trace.h" 49 50 #define NFSD_LAUNDRETTE_DELAY (2 * HZ) 51 52 #define NFSD_FILE_CACHE_UP (0) 53 54 /* We only care about NFSD_MAY_READ/WRITE for this cache */ 55 #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) 56 57 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); 58 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); 59 static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); 60 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); 61 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); 62 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); 63 64 struct nfsd_fcache_disposal { 65 spinlock_t lock; 66 struct list_head freeme; 67 }; 68 69 static struct kmem_cache *nfsd_file_slab; 70 static struct kmem_cache *nfsd_file_mark_slab; 71 static struct list_lru nfsd_file_lru; 72 static unsigned long nfsd_file_flags; 73 static struct fsnotify_group *nfsd_file_fsnotify_group; 74 static struct delayed_work nfsd_filecache_laundrette; 75 static struct rhltable nfsd_file_rhltable 76 ____cacheline_aligned_in_smp; 77 78 static bool 79 nfsd_match_cred(const struct cred *c1, const struct cred *c2) 80 { 81 int i; 82 83 if (!uid_eq(c1->fsuid, c2->fsuid)) 84 return false; 85 if (!gid_eq(c1->fsgid, c2->fsgid)) 86 return false; 87 if (c1->group_info == NULL || c2->group_info == NULL) 88 return c1->group_info == c2->group_info; 89 if (c1->group_info->ngroups != c2->group_info->ngroups) 90 return false; 91 for (i = 0; i < c1->group_info->ngroups; i++) { 92 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i])) 93 return false; 94 } 95 return true; 96 } 97 98 static const struct rhashtable_params nfsd_file_rhash_params = { 99 .key_len = sizeof_field(struct nfsd_file, nf_inode), 100 .key_offset = offsetof(struct nfsd_file, nf_inode), 101 .head_offset = offsetof(struct nfsd_file, nf_rlist), 102 103 /* 104 * Start with a single page hash table to reduce resizing churn 105 * on light workloads. 106 */ 107 .min_size = 256, 108 .automatic_shrinking = true, 109 }; 110 111 static void 112 nfsd_file_schedule_laundrette(void) 113 { 114 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags)) 115 queue_delayed_work(system_unbound_wq, &nfsd_filecache_laundrette, 116 NFSD_LAUNDRETTE_DELAY); 117 } 118 119 static void 120 nfsd_file_slab_free(struct rcu_head *rcu) 121 { 122 struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu); 123 124 put_cred(nf->nf_cred); 125 kmem_cache_free(nfsd_file_slab, nf); 126 } 127 128 static void 129 nfsd_file_mark_free(struct fsnotify_mark *mark) 130 { 131 struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark, 132 nfm_mark); 133 134 kmem_cache_free(nfsd_file_mark_slab, nfm); 135 } 136 137 static struct nfsd_file_mark * 138 nfsd_file_mark_get(struct nfsd_file_mark *nfm) 139 { 140 if (!refcount_inc_not_zero(&nfm->nfm_ref)) 141 return NULL; 142 return nfm; 143 } 144 145 static void 146 nfsd_file_mark_put(struct nfsd_file_mark *nfm) 147 { 148 if (refcount_dec_and_test(&nfm->nfm_ref)) { 149 fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group); 150 fsnotify_put_mark(&nfm->nfm_mark); 151 } 152 } 153 154 static struct nfsd_file_mark * 155 nfsd_file_mark_find_or_create(struct inode *inode) 156 { 157 int err; 158 struct fsnotify_mark *mark; 159 struct nfsd_file_mark *nfm = NULL, *new; 160 161 do { 162 fsnotify_group_lock(nfsd_file_fsnotify_group); 163 mark = fsnotify_find_inode_mark(inode, 164 nfsd_file_fsnotify_group); 165 if (mark) { 166 nfm = nfsd_file_mark_get(container_of(mark, 167 struct nfsd_file_mark, 168 nfm_mark)); 169 fsnotify_group_unlock(nfsd_file_fsnotify_group); 170 if (nfm) { 171 fsnotify_put_mark(mark); 172 break; 173 } 174 /* Avoid soft lockup race with nfsd_file_mark_put() */ 175 fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group); 176 fsnotify_put_mark(mark); 177 } else { 178 fsnotify_group_unlock(nfsd_file_fsnotify_group); 179 } 180 181 /* allocate a new nfm */ 182 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL); 183 if (!new) 184 return NULL; 185 fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group); 186 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF; 187 refcount_set(&new->nfm_ref, 1); 188 189 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); 190 191 /* 192 * If the add was successful, then return the object. 193 * Otherwise, we need to put the reference we hold on the 194 * nfm_mark. The fsnotify code will take a reference and put 195 * it on failure, so we can't just free it directly. It's also 196 * not safe to call fsnotify_destroy_mark on it as the 197 * mark->group will be NULL. Thus, we can't let the nfm_ref 198 * counter drive the destruction at this point. 199 */ 200 if (likely(!err)) 201 nfm = new; 202 else 203 fsnotify_put_mark(&new->nfm_mark); 204 } while (unlikely(err == -EEXIST)); 205 206 return nfm; 207 } 208 209 static struct nfsd_file * 210 nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, 211 bool want_gc) 212 { 213 struct nfsd_file *nf; 214 215 nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL); 216 if (unlikely(!nf)) 217 return NULL; 218 219 this_cpu_inc(nfsd_file_allocations); 220 INIT_LIST_HEAD(&nf->nf_lru); 221 INIT_LIST_HEAD(&nf->nf_gc); 222 nf->nf_birthtime = ktime_get(); 223 nf->nf_file = NULL; 224 nf->nf_cred = get_current_cred(); 225 nf->nf_net = net; 226 nf->nf_flags = want_gc ? 227 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) : 228 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING); 229 nf->nf_inode = inode; 230 refcount_set(&nf->nf_ref, 1); 231 nf->nf_may = need; 232 nf->nf_mark = NULL; 233 return nf; 234 } 235 236 /** 237 * nfsd_file_check_write_error - check for writeback errors on a file 238 * @nf: nfsd_file to check for writeback errors 239 * 240 * Check whether a nfsd_file has an unseen error. Reset the write 241 * verifier if so. 242 */ 243 static void 244 nfsd_file_check_write_error(struct nfsd_file *nf) 245 { 246 struct file *file = nf->nf_file; 247 248 if ((file->f_mode & FMODE_WRITE) && 249 filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err))) 250 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id)); 251 } 252 253 static void 254 nfsd_file_hash_remove(struct nfsd_file *nf) 255 { 256 trace_nfsd_file_unhash(nf); 257 rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist, 258 nfsd_file_rhash_params); 259 } 260 261 static bool 262 nfsd_file_unhash(struct nfsd_file *nf) 263 { 264 if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { 265 nfsd_file_hash_remove(nf); 266 return true; 267 } 268 return false; 269 } 270 271 static void 272 nfsd_file_free(struct nfsd_file *nf) 273 { 274 s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime)); 275 276 trace_nfsd_file_free(nf); 277 278 this_cpu_inc(nfsd_file_releases); 279 this_cpu_add(nfsd_file_total_age, age); 280 281 nfsd_file_unhash(nf); 282 if (nf->nf_mark) 283 nfsd_file_mark_put(nf->nf_mark); 284 if (nf->nf_file) { 285 nfsd_file_check_write_error(nf); 286 nfsd_filp_close(nf->nf_file); 287 } 288 289 /* 290 * If this item is still linked via nf_lru, that's a bug. 291 * WARN and leak it to preserve system stability. 292 */ 293 if (WARN_ON_ONCE(!list_empty(&nf->nf_lru))) 294 return; 295 296 call_rcu(&nf->nf_rcu, nfsd_file_slab_free); 297 } 298 299 static bool 300 nfsd_file_check_writeback(struct nfsd_file *nf) 301 { 302 struct file *file = nf->nf_file; 303 struct address_space *mapping; 304 305 /* File not open for write? */ 306 if (!(file->f_mode & FMODE_WRITE)) 307 return false; 308 309 /* 310 * Some filesystems (e.g. NFS) flush all dirty data on close. 311 * On others, there is no need to wait for writeback. 312 */ 313 if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE)) 314 return false; 315 316 mapping = file->f_mapping; 317 return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) || 318 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK); 319 } 320 321 322 static bool nfsd_file_lru_add(struct nfsd_file *nf) 323 { 324 set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); 325 if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) { 326 trace_nfsd_file_lru_add(nf); 327 return true; 328 } 329 return false; 330 } 331 332 static bool nfsd_file_lru_remove(struct nfsd_file *nf) 333 { 334 if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) { 335 trace_nfsd_file_lru_del(nf); 336 return true; 337 } 338 return false; 339 } 340 341 struct nfsd_file * 342 nfsd_file_get(struct nfsd_file *nf) 343 { 344 if (nf && refcount_inc_not_zero(&nf->nf_ref)) 345 return nf; 346 return NULL; 347 } 348 349 /** 350 * nfsd_file_put - put the reference to a nfsd_file 351 * @nf: nfsd_file of which to put the reference 352 * 353 * Put a reference to a nfsd_file. In the non-GC case, we just put the 354 * reference immediately. In the GC case, if the reference would be 355 * the last one, the put it on the LRU instead to be cleaned up later. 356 */ 357 void 358 nfsd_file_put(struct nfsd_file *nf) 359 { 360 might_sleep(); 361 trace_nfsd_file_put(nf); 362 363 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) && 364 test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { 365 /* 366 * If this is the last reference (nf_ref == 1), then try to 367 * transfer it to the LRU. 368 */ 369 if (refcount_dec_not_one(&nf->nf_ref)) 370 return; 371 372 /* Try to add it to the LRU. If that fails, decrement. */ 373 if (nfsd_file_lru_add(nf)) { 374 /* If it's still hashed, we're done */ 375 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { 376 nfsd_file_schedule_laundrette(); 377 return; 378 } 379 380 /* 381 * We're racing with unhashing, so try to remove it from 382 * the LRU. If removal fails, then someone else already 383 * has our reference. 384 */ 385 if (!nfsd_file_lru_remove(nf)) 386 return; 387 } 388 } 389 if (refcount_dec_and_test(&nf->nf_ref)) 390 nfsd_file_free(nf); 391 } 392 393 /** 394 * nfsd_file_put_local - put nfsd_file reference and arm nfsd_serv_put in caller 395 * @nf: nfsd_file of which to put the reference 396 * 397 * First save the associated net to return to caller, then put 398 * the reference of the nfsd_file. 399 */ 400 struct net * 401 nfsd_file_put_local(struct nfsd_file *nf) 402 { 403 struct net *net = nf->nf_net; 404 405 nfsd_file_put(nf); 406 return net; 407 } 408 409 /** 410 * nfsd_file_file - get the backing file of an nfsd_file 411 * @nf: nfsd_file of which to access the backing file. 412 * 413 * Return backing file for @nf. 414 */ 415 struct file * 416 nfsd_file_file(struct nfsd_file *nf) 417 { 418 return nf->nf_file; 419 } 420 421 static void 422 nfsd_file_dispose_list(struct list_head *dispose) 423 { 424 struct nfsd_file *nf; 425 426 while (!list_empty(dispose)) { 427 nf = list_first_entry(dispose, struct nfsd_file, nf_gc); 428 list_del_init(&nf->nf_gc); 429 nfsd_file_free(nf); 430 } 431 } 432 433 /** 434 * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list 435 * @dispose: list of nfsd_files to be disposed 436 * 437 * Transfers each file to the "freeme" list for its nfsd_net, to eventually 438 * be disposed of by the per-net garbage collector. 439 */ 440 static void 441 nfsd_file_dispose_list_delayed(struct list_head *dispose) 442 { 443 while(!list_empty(dispose)) { 444 struct nfsd_file *nf = list_first_entry(dispose, 445 struct nfsd_file, nf_gc); 446 struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); 447 struct nfsd_fcache_disposal *l = nn->fcache_disposal; 448 449 spin_lock(&l->lock); 450 list_move_tail(&nf->nf_gc, &l->freeme); 451 spin_unlock(&l->lock); 452 svc_wake_up(nn->nfsd_serv); 453 } 454 } 455 456 /** 457 * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed. 458 * @nn: nfsd_net in which to find files to be disposed. 459 * 460 * When files held open for nfsv3 are removed from the filecache, whether 461 * due to memory pressure or garbage collection, they are queued to 462 * a per-net-ns queue. This function completes the disposal, either 463 * directly or by waking another nfsd thread to help with the work. 464 */ 465 void nfsd_file_net_dispose(struct nfsd_net *nn) 466 { 467 struct nfsd_fcache_disposal *l = nn->fcache_disposal; 468 469 if (!list_empty(&l->freeme)) { 470 LIST_HEAD(dispose); 471 int i; 472 473 spin_lock(&l->lock); 474 for (i = 0; i < 8 && !list_empty(&l->freeme); i++) 475 list_move(l->freeme.next, &dispose); 476 spin_unlock(&l->lock); 477 if (!list_empty(&l->freeme)) 478 /* Wake up another thread to share the work 479 * *before* doing any actual disposing. 480 */ 481 svc_wake_up(nn->nfsd_serv); 482 nfsd_file_dispose_list(&dispose); 483 } 484 } 485 486 /** 487 * nfsd_file_lru_cb - Examine an entry on the LRU list 488 * @item: LRU entry to examine 489 * @lru: controlling LRU 490 * @arg: dispose list 491 * 492 * Return values: 493 * %LRU_REMOVED: @item was removed from the LRU 494 * %LRU_ROTATE: @item is to be moved to the LRU tail 495 * %LRU_SKIP: @item cannot be evicted 496 */ 497 static enum lru_status 498 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru, 499 void *arg) 500 { 501 struct list_head *head = arg; 502 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru); 503 504 /* We should only be dealing with GC entries here */ 505 WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags)); 506 507 /* 508 * Don't throw out files that are still undergoing I/O or 509 * that have uncleared errors pending. 510 */ 511 if (nfsd_file_check_writeback(nf)) { 512 trace_nfsd_file_gc_writeback(nf); 513 return LRU_SKIP; 514 } 515 516 /* If it was recently added to the list, skip it */ 517 if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) { 518 trace_nfsd_file_gc_referenced(nf); 519 return LRU_ROTATE; 520 } 521 522 /* 523 * Put the reference held on behalf of the LRU. If it wasn't the last 524 * one, then just remove it from the LRU and ignore it. 525 */ 526 if (!refcount_dec_and_test(&nf->nf_ref)) { 527 trace_nfsd_file_gc_in_use(nf); 528 list_lru_isolate(lru, &nf->nf_lru); 529 return LRU_REMOVED; 530 } 531 532 /* Refcount went to zero. Unhash it and queue it to the dispose list */ 533 nfsd_file_unhash(nf); 534 list_lru_isolate(lru, &nf->nf_lru); 535 list_add(&nf->nf_gc, head); 536 this_cpu_inc(nfsd_file_evictions); 537 trace_nfsd_file_gc_disposed(nf); 538 return LRU_REMOVED; 539 } 540 541 static void 542 nfsd_file_gc(void) 543 { 544 LIST_HEAD(dispose); 545 unsigned long ret; 546 547 ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb, 548 &dispose, list_lru_count(&nfsd_file_lru)); 549 trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru)); 550 nfsd_file_dispose_list_delayed(&dispose); 551 } 552 553 static void 554 nfsd_file_gc_worker(struct work_struct *work) 555 { 556 nfsd_file_gc(); 557 if (list_lru_count(&nfsd_file_lru)) 558 nfsd_file_schedule_laundrette(); 559 } 560 561 static unsigned long 562 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc) 563 { 564 return list_lru_count(&nfsd_file_lru); 565 } 566 567 static unsigned long 568 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc) 569 { 570 LIST_HEAD(dispose); 571 unsigned long ret; 572 573 ret = list_lru_shrink_walk(&nfsd_file_lru, sc, 574 nfsd_file_lru_cb, &dispose); 575 trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru)); 576 nfsd_file_dispose_list_delayed(&dispose); 577 return ret; 578 } 579 580 static struct shrinker *nfsd_file_shrinker; 581 582 /** 583 * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file 584 * @nf: nfsd_file to attempt to queue 585 * @dispose: private list to queue successfully-put objects 586 * 587 * Unhash an nfsd_file, try to get a reference to it, and then put that 588 * reference. If it's the last reference, queue it to the dispose list. 589 */ 590 static void 591 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose) 592 __must_hold(RCU) 593 { 594 int decrement = 1; 595 596 /* If we raced with someone else unhashing, ignore it */ 597 if (!nfsd_file_unhash(nf)) 598 return; 599 600 /* If we can't get a reference, ignore it */ 601 if (!nfsd_file_get(nf)) 602 return; 603 604 /* Extra decrement if we remove from the LRU */ 605 if (nfsd_file_lru_remove(nf)) 606 ++decrement; 607 608 /* If refcount goes to 0, then put on the dispose list */ 609 if (refcount_sub_and_test(decrement, &nf->nf_ref)) { 610 list_add(&nf->nf_gc, dispose); 611 trace_nfsd_file_closing(nf); 612 } 613 } 614 615 /** 616 * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode 617 * @inode: inode on which to close out nfsd_files 618 * @dispose: list on which to gather nfsd_files to close out 619 * 620 * An nfsd_file represents a struct file being held open on behalf of nfsd. 621 * An open file however can block other activity (such as leases), or cause 622 * undesirable behavior (e.g. spurious silly-renames when reexporting NFS). 623 * 624 * This function is intended to find open nfsd_files when this sort of 625 * conflicting access occurs and then attempt to close those files out. 626 * 627 * Populates the dispose list with entries that have already had their 628 * refcounts go to zero. The actual free of an nfsd_file can be expensive, 629 * so we leave it up to the caller whether it wants to wait or not. 630 */ 631 static void 632 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose) 633 { 634 struct rhlist_head *tmp, *list; 635 struct nfsd_file *nf; 636 637 rcu_read_lock(); 638 list = rhltable_lookup(&nfsd_file_rhltable, &inode, 639 nfsd_file_rhash_params); 640 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) { 641 if (!test_bit(NFSD_FILE_GC, &nf->nf_flags)) 642 continue; 643 nfsd_file_cond_queue(nf, dispose); 644 } 645 rcu_read_unlock(); 646 } 647 648 /** 649 * nfsd_file_close_inode - attempt a delayed close of a nfsd_file 650 * @inode: inode of the file to attempt to remove 651 * 652 * Close out any open nfsd_files that can be reaped for @inode. The 653 * actual freeing is deferred to the dispose_list_delayed infrastructure. 654 * 655 * This is used by the fsnotify callbacks and setlease notifier. 656 */ 657 static void 658 nfsd_file_close_inode(struct inode *inode) 659 { 660 LIST_HEAD(dispose); 661 662 nfsd_file_queue_for_close(inode, &dispose); 663 nfsd_file_dispose_list_delayed(&dispose); 664 } 665 666 /** 667 * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file 668 * @inode: inode of the file to attempt to remove 669 * 670 * Close out any open nfsd_files that can be reaped for @inode. The 671 * nfsd_files are closed out synchronously. 672 * 673 * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames 674 * when reexporting NFS. 675 */ 676 void 677 nfsd_file_close_inode_sync(struct inode *inode) 678 { 679 struct nfsd_file *nf; 680 LIST_HEAD(dispose); 681 682 trace_nfsd_file_close(inode); 683 684 nfsd_file_queue_for_close(inode, &dispose); 685 while (!list_empty(&dispose)) { 686 nf = list_first_entry(&dispose, struct nfsd_file, nf_gc); 687 list_del_init(&nf->nf_gc); 688 nfsd_file_free(nf); 689 } 690 } 691 692 static int 693 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg, 694 void *data) 695 { 696 struct file_lease *fl = data; 697 698 /* Only close files for F_SETLEASE leases */ 699 if (fl->c.flc_flags & FL_LEASE) 700 nfsd_file_close_inode(file_inode(fl->c.flc_file)); 701 return 0; 702 } 703 704 static struct notifier_block nfsd_file_lease_notifier = { 705 .notifier_call = nfsd_file_lease_notifier_call, 706 }; 707 708 static int 709 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask, 710 struct inode *inode, struct inode *dir, 711 const struct qstr *name, u32 cookie) 712 { 713 if (WARN_ON_ONCE(!inode)) 714 return 0; 715 716 trace_nfsd_file_fsnotify_handle_event(inode, mask); 717 718 /* Should be no marks on non-regular files */ 719 if (!S_ISREG(inode->i_mode)) { 720 WARN_ON_ONCE(1); 721 return 0; 722 } 723 724 /* don't close files if this was not the last link */ 725 if (mask & FS_ATTRIB) { 726 if (inode->i_nlink) 727 return 0; 728 } 729 730 nfsd_file_close_inode(inode); 731 return 0; 732 } 733 734 735 static const struct fsnotify_ops nfsd_file_fsnotify_ops = { 736 .handle_inode_event = nfsd_file_fsnotify_handle_event, 737 .free_mark = nfsd_file_mark_free, 738 }; 739 740 int 741 nfsd_file_cache_init(void) 742 { 743 int ret; 744 745 lockdep_assert_held(&nfsd_mutex); 746 if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) 747 return 0; 748 749 ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params); 750 if (ret) 751 goto out; 752 753 ret = -ENOMEM; 754 nfsd_file_slab = KMEM_CACHE(nfsd_file, 0); 755 if (!nfsd_file_slab) { 756 pr_err("nfsd: unable to create nfsd_file_slab\n"); 757 goto out_err; 758 } 759 760 nfsd_file_mark_slab = KMEM_CACHE(nfsd_file_mark, 0); 761 if (!nfsd_file_mark_slab) { 762 pr_err("nfsd: unable to create nfsd_file_mark_slab\n"); 763 goto out_err; 764 } 765 766 ret = list_lru_init(&nfsd_file_lru); 767 if (ret) { 768 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret); 769 goto out_err; 770 } 771 772 nfsd_file_shrinker = shrinker_alloc(0, "nfsd-filecache"); 773 if (!nfsd_file_shrinker) { 774 ret = -ENOMEM; 775 pr_err("nfsd: failed to allocate nfsd_file_shrinker\n"); 776 goto out_lru; 777 } 778 779 nfsd_file_shrinker->count_objects = nfsd_file_lru_count; 780 nfsd_file_shrinker->scan_objects = nfsd_file_lru_scan; 781 nfsd_file_shrinker->seeks = 1; 782 783 shrinker_register(nfsd_file_shrinker); 784 785 ret = lease_register_notifier(&nfsd_file_lease_notifier); 786 if (ret) { 787 pr_err("nfsd: unable to register lease notifier: %d\n", ret); 788 goto out_shrinker; 789 } 790 791 nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, 792 0); 793 if (IS_ERR(nfsd_file_fsnotify_group)) { 794 pr_err("nfsd: unable to create fsnotify group: %ld\n", 795 PTR_ERR(nfsd_file_fsnotify_group)); 796 ret = PTR_ERR(nfsd_file_fsnotify_group); 797 nfsd_file_fsnotify_group = NULL; 798 goto out_notifier; 799 } 800 801 INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker); 802 out: 803 if (ret) 804 clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags); 805 return ret; 806 out_notifier: 807 lease_unregister_notifier(&nfsd_file_lease_notifier); 808 out_shrinker: 809 shrinker_free(nfsd_file_shrinker); 810 out_lru: 811 list_lru_destroy(&nfsd_file_lru); 812 out_err: 813 kmem_cache_destroy(nfsd_file_slab); 814 nfsd_file_slab = NULL; 815 kmem_cache_destroy(nfsd_file_mark_slab); 816 nfsd_file_mark_slab = NULL; 817 rhltable_destroy(&nfsd_file_rhltable); 818 goto out; 819 } 820 821 /** 822 * __nfsd_file_cache_purge: clean out the cache for shutdown 823 * @net: net-namespace to shut down the cache (may be NULL) 824 * 825 * Walk the nfsd_file cache and close out any that match @net. If @net is NULL, 826 * then close out everything. Called when an nfsd instance is being shut down, 827 * and when the exports table is flushed. 828 */ 829 static void 830 __nfsd_file_cache_purge(struct net *net) 831 { 832 struct rhashtable_iter iter; 833 struct nfsd_file *nf; 834 LIST_HEAD(dispose); 835 836 rhltable_walk_enter(&nfsd_file_rhltable, &iter); 837 do { 838 rhashtable_walk_start(&iter); 839 840 nf = rhashtable_walk_next(&iter); 841 while (!IS_ERR_OR_NULL(nf)) { 842 if (!net || nf->nf_net == net) 843 nfsd_file_cond_queue(nf, &dispose); 844 nf = rhashtable_walk_next(&iter); 845 } 846 847 rhashtable_walk_stop(&iter); 848 } while (nf == ERR_PTR(-EAGAIN)); 849 rhashtable_walk_exit(&iter); 850 851 nfsd_file_dispose_list(&dispose); 852 } 853 854 static struct nfsd_fcache_disposal * 855 nfsd_alloc_fcache_disposal(void) 856 { 857 struct nfsd_fcache_disposal *l; 858 859 l = kmalloc(sizeof(*l), GFP_KERNEL); 860 if (!l) 861 return NULL; 862 spin_lock_init(&l->lock); 863 INIT_LIST_HEAD(&l->freeme); 864 return l; 865 } 866 867 static void 868 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l) 869 { 870 nfsd_file_dispose_list(&l->freeme); 871 kfree(l); 872 } 873 874 static void 875 nfsd_free_fcache_disposal_net(struct net *net) 876 { 877 struct nfsd_net *nn = net_generic(net, nfsd_net_id); 878 struct nfsd_fcache_disposal *l = nn->fcache_disposal; 879 880 nfsd_free_fcache_disposal(l); 881 } 882 883 int 884 nfsd_file_cache_start_net(struct net *net) 885 { 886 struct nfsd_net *nn = net_generic(net, nfsd_net_id); 887 888 nn->fcache_disposal = nfsd_alloc_fcache_disposal(); 889 return nn->fcache_disposal ? 0 : -ENOMEM; 890 } 891 892 /** 893 * nfsd_file_cache_purge - Remove all cache items associated with @net 894 * @net: target net namespace 895 * 896 */ 897 void 898 nfsd_file_cache_purge(struct net *net) 899 { 900 lockdep_assert_held(&nfsd_mutex); 901 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) 902 __nfsd_file_cache_purge(net); 903 } 904 905 void 906 nfsd_file_cache_shutdown_net(struct net *net) 907 { 908 nfsd_file_cache_purge(net); 909 nfsd_free_fcache_disposal_net(net); 910 } 911 912 void 913 nfsd_file_cache_shutdown(void) 914 { 915 int i; 916 917 lockdep_assert_held(&nfsd_mutex); 918 if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0) 919 return; 920 921 lease_unregister_notifier(&nfsd_file_lease_notifier); 922 shrinker_free(nfsd_file_shrinker); 923 /* 924 * make sure all callers of nfsd_file_lru_cb are done before 925 * calling nfsd_file_cache_purge 926 */ 927 cancel_delayed_work_sync(&nfsd_filecache_laundrette); 928 __nfsd_file_cache_purge(NULL); 929 list_lru_destroy(&nfsd_file_lru); 930 rcu_barrier(); 931 fsnotify_put_group(nfsd_file_fsnotify_group); 932 nfsd_file_fsnotify_group = NULL; 933 kmem_cache_destroy(nfsd_file_slab); 934 nfsd_file_slab = NULL; 935 fsnotify_wait_marks_destroyed(); 936 kmem_cache_destroy(nfsd_file_mark_slab); 937 nfsd_file_mark_slab = NULL; 938 rhltable_destroy(&nfsd_file_rhltable); 939 940 for_each_possible_cpu(i) { 941 per_cpu(nfsd_file_cache_hits, i) = 0; 942 per_cpu(nfsd_file_acquisitions, i) = 0; 943 per_cpu(nfsd_file_allocations, i) = 0; 944 per_cpu(nfsd_file_releases, i) = 0; 945 per_cpu(nfsd_file_total_age, i) = 0; 946 per_cpu(nfsd_file_evictions, i) = 0; 947 } 948 } 949 950 static struct nfsd_file * 951 nfsd_file_lookup_locked(const struct net *net, const struct cred *cred, 952 struct inode *inode, unsigned char need, 953 bool want_gc) 954 { 955 struct rhlist_head *tmp, *list; 956 struct nfsd_file *nf; 957 958 list = rhltable_lookup(&nfsd_file_rhltable, &inode, 959 nfsd_file_rhash_params); 960 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) { 961 if (nf->nf_may != need) 962 continue; 963 if (nf->nf_net != net) 964 continue; 965 if (!nfsd_match_cred(nf->nf_cred, cred)) 966 continue; 967 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc) 968 continue; 969 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0) 970 continue; 971 972 if (!nfsd_file_get(nf)) 973 continue; 974 return nf; 975 } 976 return NULL; 977 } 978 979 /** 980 * nfsd_file_is_cached - are there any cached open files for this inode? 981 * @inode: inode to check 982 * 983 * The lookup matches inodes in all net namespaces and is atomic wrt 984 * nfsd_file_acquire(). 985 * 986 * Return values: 987 * %true: filecache contains at least one file matching this inode 988 * %false: filecache contains no files matching this inode 989 */ 990 bool 991 nfsd_file_is_cached(struct inode *inode) 992 { 993 struct rhlist_head *tmp, *list; 994 struct nfsd_file *nf; 995 bool ret = false; 996 997 rcu_read_lock(); 998 list = rhltable_lookup(&nfsd_file_rhltable, &inode, 999 nfsd_file_rhash_params); 1000 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) 1001 if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) { 1002 ret = true; 1003 break; 1004 } 1005 rcu_read_unlock(); 1006 1007 trace_nfsd_file_is_cached(inode, (int)ret); 1008 return ret; 1009 } 1010 1011 static __be32 1012 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net, 1013 struct svc_cred *cred, 1014 struct auth_domain *client, 1015 struct svc_fh *fhp, 1016 unsigned int may_flags, struct file *file, 1017 struct nfsd_file **pnf, bool want_gc) 1018 { 1019 unsigned char need = may_flags & NFSD_FILE_MAY_MASK; 1020 struct nfsd_file *new, *nf; 1021 bool stale_retry = true; 1022 bool open_retry = true; 1023 struct inode *inode; 1024 __be32 status; 1025 int ret; 1026 1027 retry: 1028 if (rqstp) { 1029 status = fh_verify(rqstp, fhp, S_IFREG, 1030 may_flags|NFSD_MAY_OWNER_OVERRIDE); 1031 } else { 1032 status = fh_verify_local(net, cred, client, fhp, S_IFREG, 1033 may_flags|NFSD_MAY_OWNER_OVERRIDE); 1034 } 1035 if (status != nfs_ok) 1036 return status; 1037 inode = d_inode(fhp->fh_dentry); 1038 1039 rcu_read_lock(); 1040 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc); 1041 rcu_read_unlock(); 1042 1043 if (nf) { 1044 /* 1045 * If the nf is on the LRU then it holds an extra reference 1046 * that must be put if it's removed. It had better not be 1047 * the last one however, since we should hold another. 1048 */ 1049 if (nfsd_file_lru_remove(nf)) 1050 refcount_dec(&nf->nf_ref); 1051 goto wait_for_construction; 1052 } 1053 1054 new = nfsd_file_alloc(net, inode, need, want_gc); 1055 if (!new) { 1056 status = nfserr_jukebox; 1057 goto out; 1058 } 1059 1060 rcu_read_lock(); 1061 spin_lock(&inode->i_lock); 1062 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc); 1063 if (unlikely(nf)) { 1064 spin_unlock(&inode->i_lock); 1065 rcu_read_unlock(); 1066 nfsd_file_free(new); 1067 goto wait_for_construction; 1068 } 1069 nf = new; 1070 ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist, 1071 nfsd_file_rhash_params); 1072 spin_unlock(&inode->i_lock); 1073 rcu_read_unlock(); 1074 if (likely(ret == 0)) 1075 goto open_file; 1076 1077 trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret); 1078 status = nfserr_jukebox; 1079 goto construction_err; 1080 1081 wait_for_construction: 1082 wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE); 1083 1084 /* Did construction of this file fail? */ 1085 if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { 1086 trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf); 1087 if (!open_retry) { 1088 status = nfserr_jukebox; 1089 goto construction_err; 1090 } 1091 nfsd_file_put(nf); 1092 open_retry = false; 1093 fh_put(fhp); 1094 goto retry; 1095 } 1096 this_cpu_inc(nfsd_file_cache_hits); 1097 1098 status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags)); 1099 if (status != nfs_ok) { 1100 nfsd_file_put(nf); 1101 nf = NULL; 1102 } 1103 1104 out: 1105 if (status == nfs_ok) { 1106 this_cpu_inc(nfsd_file_acquisitions); 1107 nfsd_file_check_write_error(nf); 1108 *pnf = nf; 1109 } 1110 trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status); 1111 return status; 1112 1113 open_file: 1114 trace_nfsd_file_alloc(nf); 1115 nf->nf_mark = nfsd_file_mark_find_or_create(inode); 1116 if (nf->nf_mark) { 1117 if (file) { 1118 get_file(file); 1119 nf->nf_file = file; 1120 status = nfs_ok; 1121 trace_nfsd_file_opened(nf, status); 1122 } else { 1123 ret = nfsd_open_verified(fhp, may_flags, &nf->nf_file); 1124 if (ret == -EOPENSTALE && stale_retry) { 1125 stale_retry = false; 1126 nfsd_file_unhash(nf); 1127 clear_and_wake_up_bit(NFSD_FILE_PENDING, 1128 &nf->nf_flags); 1129 if (refcount_dec_and_test(&nf->nf_ref)) 1130 nfsd_file_free(nf); 1131 nf = NULL; 1132 fh_put(fhp); 1133 goto retry; 1134 } 1135 status = nfserrno(ret); 1136 trace_nfsd_file_open(nf, status); 1137 } 1138 } else 1139 status = nfserr_jukebox; 1140 /* 1141 * If construction failed, or we raced with a call to unlink() 1142 * then unhash. 1143 */ 1144 if (status != nfs_ok || inode->i_nlink == 0) 1145 nfsd_file_unhash(nf); 1146 clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags); 1147 if (status == nfs_ok) 1148 goto out; 1149 1150 construction_err: 1151 if (refcount_dec_and_test(&nf->nf_ref)) 1152 nfsd_file_free(nf); 1153 nf = NULL; 1154 goto out; 1155 } 1156 1157 /** 1158 * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file 1159 * @rqstp: the RPC transaction being executed 1160 * @fhp: the NFS filehandle of the file to be opened 1161 * @may_flags: NFSD_MAY_ settings for the file 1162 * @pnf: OUT: new or found "struct nfsd_file" object 1163 * 1164 * The nfsd_file object returned by this API is reference-counted 1165 * and garbage-collected. The object is retained for a few 1166 * seconds after the final nfsd_file_put() in case the caller 1167 * wants to re-use it. 1168 * 1169 * Return values: 1170 * %nfs_ok - @pnf points to an nfsd_file with its reference 1171 * count boosted. 1172 * 1173 * On error, an nfsstat value in network byte order is returned. 1174 */ 1175 __be32 1176 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp, 1177 unsigned int may_flags, struct nfsd_file **pnf) 1178 { 1179 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, 1180 fhp, may_flags, NULL, pnf, true); 1181 } 1182 1183 /** 1184 * nfsd_file_acquire - Get a struct nfsd_file with an open file 1185 * @rqstp: the RPC transaction being executed 1186 * @fhp: the NFS filehandle of the file to be opened 1187 * @may_flags: NFSD_MAY_ settings for the file 1188 * @pnf: OUT: new or found "struct nfsd_file" object 1189 * 1190 * The nfsd_file_object returned by this API is reference-counted 1191 * but not garbage-collected. The object is unhashed after the 1192 * final nfsd_file_put(). 1193 * 1194 * Return values: 1195 * %nfs_ok - @pnf points to an nfsd_file with its reference 1196 * count boosted. 1197 * 1198 * On error, an nfsstat value in network byte order is returned. 1199 */ 1200 __be32 1201 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, 1202 unsigned int may_flags, struct nfsd_file **pnf) 1203 { 1204 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, 1205 fhp, may_flags, NULL, pnf, false); 1206 } 1207 1208 /** 1209 * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio 1210 * @net: The network namespace in which to perform a lookup 1211 * @cred: the user credential with which to validate access 1212 * @client: the auth_domain for LOCALIO lookup 1213 * @fhp: the NFS filehandle of the file to be opened 1214 * @may_flags: NFSD_MAY_ settings for the file 1215 * @pnf: OUT: new or found "struct nfsd_file" object 1216 * 1217 * This file lookup interface provide access to a file given the 1218 * filehandle and credential. No connection-based authorisation 1219 * is performed and in that way it is quite different to other 1220 * file access mediated by nfsd. It allows a kernel module such as the NFS 1221 * client to reach across network and filesystem namespaces to access 1222 * a file. The security implications of this should be carefully 1223 * considered before use. 1224 * 1225 * The nfsd_file object returned by this API is reference-counted 1226 * and garbage-collected. The object is retained for a few 1227 * seconds after the final nfsd_file_put() in case the caller 1228 * wants to re-use it. 1229 * 1230 * Return values: 1231 * %nfs_ok - @pnf points to an nfsd_file with its reference 1232 * count boosted. 1233 * 1234 * On error, an nfsstat value in network byte order is returned. 1235 */ 1236 __be32 1237 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, 1238 struct auth_domain *client, struct svc_fh *fhp, 1239 unsigned int may_flags, struct nfsd_file **pnf) 1240 { 1241 /* 1242 * Save creds before calling nfsd_file_do_acquire() (which calls 1243 * nfsd_setuser). Important because caller (LOCALIO) is from 1244 * client context. 1245 */ 1246 const struct cred *save_cred = get_current_cred(); 1247 __be32 beres; 1248 1249 beres = nfsd_file_do_acquire(NULL, net, cred, client, 1250 fhp, may_flags, NULL, pnf, true); 1251 revert_creds(save_cred); 1252 return beres; 1253 } 1254 1255 /** 1256 * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file 1257 * @rqstp: the RPC transaction being executed 1258 * @fhp: the NFS filehandle of the file just created 1259 * @may_flags: NFSD_MAY_ settings for the file 1260 * @file: cached, already-open file (may be NULL) 1261 * @pnf: OUT: new or found "struct nfsd_file" object 1262 * 1263 * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist, 1264 * and @file is non-NULL, use it to instantiate a new nfsd_file instead of 1265 * opening a new one. 1266 * 1267 * Return values: 1268 * %nfs_ok - @pnf points to an nfsd_file with its reference 1269 * count boosted. 1270 * 1271 * On error, an nfsstat value in network byte order is returned. 1272 */ 1273 __be32 1274 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, 1275 unsigned int may_flags, struct file *file, 1276 struct nfsd_file **pnf) 1277 { 1278 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, 1279 fhp, may_flags, file, pnf, false); 1280 } 1281 1282 /* 1283 * Note that fields may be added, removed or reordered in the future. Programs 1284 * scraping this file for info should test the labels to ensure they're 1285 * getting the correct field. 1286 */ 1287 int nfsd_file_cache_stats_show(struct seq_file *m, void *v) 1288 { 1289 unsigned long allocations = 0, releases = 0, evictions = 0; 1290 unsigned long hits = 0, acquisitions = 0; 1291 unsigned int i, count = 0, buckets = 0; 1292 unsigned long lru = 0, total_age = 0; 1293 1294 /* Serialize with server shutdown */ 1295 mutex_lock(&nfsd_mutex); 1296 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) { 1297 struct bucket_table *tbl; 1298 struct rhashtable *ht; 1299 1300 lru = list_lru_count(&nfsd_file_lru); 1301 1302 rcu_read_lock(); 1303 ht = &nfsd_file_rhltable.ht; 1304 count = atomic_read(&ht->nelems); 1305 tbl = rht_dereference_rcu(ht->tbl, ht); 1306 buckets = tbl->size; 1307 rcu_read_unlock(); 1308 } 1309 mutex_unlock(&nfsd_mutex); 1310 1311 for_each_possible_cpu(i) { 1312 hits += per_cpu(nfsd_file_cache_hits, i); 1313 acquisitions += per_cpu(nfsd_file_acquisitions, i); 1314 allocations += per_cpu(nfsd_file_allocations, i); 1315 releases += per_cpu(nfsd_file_releases, i); 1316 total_age += per_cpu(nfsd_file_total_age, i); 1317 evictions += per_cpu(nfsd_file_evictions, i); 1318 } 1319 1320 seq_printf(m, "total inodes: %u\n", count); 1321 seq_printf(m, "hash buckets: %u\n", buckets); 1322 seq_printf(m, "lru entries: %lu\n", lru); 1323 seq_printf(m, "cache hits: %lu\n", hits); 1324 seq_printf(m, "acquisitions: %lu\n", acquisitions); 1325 seq_printf(m, "allocations: %lu\n", allocations); 1326 seq_printf(m, "releases: %lu\n", releases); 1327 seq_printf(m, "evictions: %lu\n", evictions); 1328 if (releases) 1329 seq_printf(m, "mean age (ms): %ld\n", total_age / releases); 1330 else 1331 seq_printf(m, "mean age (ms): -\n"); 1332 return 0; 1333 } 1334