1 /* $NetBSD: tmpfs_subr.c,v 1.35 2007/07/09 21:10:50 ad Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause-NetBSD 5 * 6 * Copyright (c) 2005 The NetBSD Foundation, Inc. 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by Julio M. Merino Vidal, developed as part of Google's Summer of Code 11 * 2005 program. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /* 36 * Efficient memory file system supporting functions. 37 */ 38 #include <sys/cdefs.h> 39 __FBSDID("$FreeBSD$"); 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/dirent.h> 44 #include <sys/fnv_hash.h> 45 #include <sys/lock.h> 46 #include <sys/limits.h> 47 #include <sys/mount.h> 48 #include <sys/namei.h> 49 #include <sys/priv.h> 50 #include <sys/proc.h> 51 #include <sys/random.h> 52 #include <sys/refcount.h> 53 #include <sys/rwlock.h> 54 #include <sys/smr.h> 55 #include <sys/stat.h> 56 #include <sys/sysctl.h> 57 #include <sys/user.h> 58 #include <sys/vnode.h> 59 #include <sys/vmmeter.h> 60 61 #include <vm/vm.h> 62 #include <vm/vm_param.h> 63 #include <vm/vm_object.h> 64 #include <vm/vm_page.h> 65 #include <vm/vm_pageout.h> 66 #include <vm/vm_pager.h> 67 #include <vm/vm_extern.h> 68 #include <vm/swap_pager.h> 69 70 #include <fs/tmpfs/tmpfs.h> 71 #include <fs/tmpfs/tmpfs_fifoops.h> 72 #include <fs/tmpfs/tmpfs_vnops.h> 73 74 SYSCTL_NODE(_vfs, OID_AUTO, tmpfs, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 75 "tmpfs file system"); 76 77 static long tmpfs_pages_reserved = TMPFS_PAGES_MINRESERVED; 78 79 MALLOC_DEFINE(M_TMPFSDIR, "tmpfs dir", "tmpfs dirent structure"); 80 static uma_zone_t tmpfs_node_pool; 81 VFS_SMR_DECLARE; 82 83 int tmpfs_pager_type = -1; 84 85 static vm_object_t 86 tmpfs_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 87 vm_ooffset_t offset, struct ucred *cred) 88 { 89 vm_object_t object; 90 91 MPASS(handle == NULL); 92 MPASS(offset == 0); 93 object = vm_object_allocate_dyn(tmpfs_pager_type, size, 94 OBJ_COLORED | OBJ_SWAP); 95 if (!swap_pager_init_object(object, NULL, NULL, size, 0)) { 96 vm_object_deallocate(object); 97 object = NULL; 98 } 99 return (object); 100 } 101 102 /* 103 * Make sure tmpfs vnodes with writable mappings can be found on the lazy list. 104 * 105 * This allows for periodic mtime updates while only scanning vnodes which are 106 * plausibly dirty, see tmpfs_update_mtime_lazy. 107 */ 108 static void 109 tmpfs_pager_writecount_recalc(vm_object_t object, vm_offset_t old, 110 vm_offset_t new) 111 { 112 struct vnode *vp; 113 114 VM_OBJECT_ASSERT_WLOCKED(object); 115 116 vp = VM_TO_TMPFS_VP(object); 117 118 /* 119 * Forced unmount? 120 */ 121 if (vp == NULL) { 122 KASSERT((object->flags & OBJ_TMPFS_VREF) == 0, 123 ("object %p with OBJ_TMPFS_VREF but without vnode", 124 object)); 125 VM_OBJECT_WUNLOCK(object); 126 return; 127 } 128 129 if (old == 0) { 130 VNASSERT((object->flags & OBJ_TMPFS_VREF) == 0, vp, 131 ("object without writable mappings has a reference")); 132 VNPASS(vp->v_usecount > 0, vp); 133 } else { 134 VNASSERT((object->flags & OBJ_TMPFS_VREF) != 0, vp, 135 ("object with writable mappings does not " 136 "have a reference")); 137 } 138 139 if (old == new) { 140 VM_OBJECT_WUNLOCK(object); 141 return; 142 } 143 144 if (new == 0) { 145 vm_object_clear_flag(object, OBJ_TMPFS_VREF); 146 VM_OBJECT_WUNLOCK(object); 147 vrele(vp); 148 } else { 149 if ((object->flags & OBJ_TMPFS_VREF) == 0) { 150 vref(vp); 151 vlazy(vp); 152 vm_object_set_flag(object, OBJ_TMPFS_VREF); 153 } 154 VM_OBJECT_WUNLOCK(object); 155 } 156 } 157 158 static void 159 tmpfs_pager_update_writecount(vm_object_t object, vm_offset_t start, 160 vm_offset_t end) 161 { 162 vm_offset_t new, old; 163 164 VM_OBJECT_WLOCK(object); 165 KASSERT((object->flags & OBJ_ANON) == 0, 166 ("%s: object %p with OBJ_ANON", __func__, object)); 167 old = object->un_pager.swp.writemappings; 168 object->un_pager.swp.writemappings += (vm_ooffset_t)end - start; 169 new = object->un_pager.swp.writemappings; 170 tmpfs_pager_writecount_recalc(object, old, new); 171 VM_OBJECT_ASSERT_UNLOCKED(object); 172 } 173 174 static void 175 tmpfs_pager_release_writecount(vm_object_t object, vm_offset_t start, 176 vm_offset_t end) 177 { 178 vm_offset_t new, old; 179 180 VM_OBJECT_WLOCK(object); 181 KASSERT((object->flags & OBJ_ANON) == 0, 182 ("%s: object %p with OBJ_ANON", __func__, object)); 183 old = object->un_pager.swp.writemappings; 184 object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; 185 new = object->un_pager.swp.writemappings; 186 tmpfs_pager_writecount_recalc(object, old, new); 187 VM_OBJECT_ASSERT_UNLOCKED(object); 188 } 189 190 static void 191 tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp) 192 { 193 struct vnode *vp; 194 195 /* 196 * Tmpfs VREG node, which was reclaimed, has tmpfs_pager_type 197 * type. In this case there is no v_writecount to adjust. 198 */ 199 if (vp_heldp != NULL) 200 VM_OBJECT_RLOCK(object); 201 else 202 VM_OBJECT_ASSERT_LOCKED(object); 203 if ((object->flags & OBJ_TMPFS) != 0) { 204 vp = VM_TO_TMPFS_VP(object); 205 if (vp != NULL) { 206 *vpp = vp; 207 if (vp_heldp != NULL) { 208 vhold(vp); 209 *vp_heldp = true; 210 } 211 } 212 } 213 if (vp_heldp != NULL) 214 VM_OBJECT_RUNLOCK(object); 215 } 216 217 static void 218 tmpfs_pager_freespace(vm_object_t obj, vm_pindex_t start, vm_size_t size) 219 { 220 struct tmpfs_node *node; 221 struct tmpfs_mount *tm; 222 vm_size_t c; 223 224 swap_pager_freespace(obj, start, size, &c); 225 if ((obj->flags & OBJ_TMPFS) == 0 || c == 0) 226 return; 227 228 node = obj->un_pager.swp.swp_priv; 229 MPASS(node->tn_type == VREG); 230 tm = node->tn_reg.tn_tmp; 231 232 KASSERT(tm->tm_pages_used >= c, 233 ("tmpfs tm %p pages %jd free %jd", tm, 234 (uintmax_t)tm->tm_pages_used, (uintmax_t)c)); 235 atomic_add_long(&tm->tm_pages_used, -c); 236 KASSERT(node->tn_reg.tn_pages >= c, 237 ("tmpfs node %p pages %jd free %jd", node, 238 (uintmax_t)node->tn_reg.tn_pages, (uintmax_t)c)); 239 node->tn_reg.tn_pages -= c; 240 } 241 242 static void 243 tmpfs_page_inserted(vm_object_t obj, vm_page_t m) 244 { 245 struct tmpfs_node *node; 246 struct tmpfs_mount *tm; 247 248 if ((obj->flags & OBJ_TMPFS) == 0) 249 return; 250 251 node = obj->un_pager.swp.swp_priv; 252 MPASS(node->tn_type == VREG); 253 tm = node->tn_reg.tn_tmp; 254 255 if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) { 256 atomic_add_long(&tm->tm_pages_used, 1); 257 node->tn_reg.tn_pages += 1; 258 } 259 } 260 261 static void 262 tmpfs_page_removed(vm_object_t obj, vm_page_t m) 263 { 264 struct tmpfs_node *node; 265 struct tmpfs_mount *tm; 266 267 if ((obj->flags & OBJ_TMPFS) == 0) 268 return; 269 270 node = obj->un_pager.swp.swp_priv; 271 MPASS(node->tn_type == VREG); 272 tm = node->tn_reg.tn_tmp; 273 274 if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) { 275 KASSERT(tm->tm_pages_used >= 1, 276 ("tmpfs tm %p pages %jd free 1", tm, 277 (uintmax_t)tm->tm_pages_used)); 278 atomic_add_long(&tm->tm_pages_used, -1); 279 KASSERT(node->tn_reg.tn_pages >= 1, 280 ("tmpfs node %p pages %jd free 1", node, 281 (uintmax_t)node->tn_reg.tn_pages)); 282 node->tn_reg.tn_pages -= 1; 283 } 284 } 285 286 static boolean_t 287 tmpfs_can_alloc_page(vm_object_t obj, vm_pindex_t pindex) 288 { 289 struct tmpfs_mount *tm; 290 291 tm = VM_TO_TMPFS_MP(obj); 292 if (tm == NULL || vm_pager_has_page(obj, pindex, NULL, NULL) || 293 tm->tm_pages_max == 0) 294 return (true); 295 return (tm->tm_pages_max > atomic_load_long(&tm->tm_pages_used)); 296 } 297 298 struct pagerops tmpfs_pager_ops = { 299 .pgo_kvme_type = KVME_TYPE_VNODE, 300 .pgo_alloc = tmpfs_pager_alloc, 301 .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_, 302 .pgo_update_writecount = tmpfs_pager_update_writecount, 303 .pgo_release_writecount = tmpfs_pager_release_writecount, 304 .pgo_mightbedirty = vm_object_mightbedirty_, 305 .pgo_getvp = tmpfs_pager_getvp, 306 .pgo_freespace = tmpfs_pager_freespace, 307 .pgo_page_inserted = tmpfs_page_inserted, 308 .pgo_page_removed = tmpfs_page_removed, 309 .pgo_can_alloc_page = tmpfs_can_alloc_page, 310 }; 311 312 static int 313 tmpfs_node_ctor(void *mem, int size, void *arg, int flags) 314 { 315 struct tmpfs_node *node; 316 317 node = mem; 318 node->tn_gen++; 319 node->tn_size = 0; 320 node->tn_status = 0; 321 node->tn_accessed = false; 322 node->tn_flags = 0; 323 node->tn_links = 0; 324 node->tn_vnode = NULL; 325 node->tn_vpstate = 0; 326 return (0); 327 } 328 329 static void 330 tmpfs_node_dtor(void *mem, int size, void *arg) 331 { 332 struct tmpfs_node *node; 333 334 node = mem; 335 node->tn_type = VNON; 336 } 337 338 static int 339 tmpfs_node_init(void *mem, int size, int flags) 340 { 341 struct tmpfs_node *node; 342 343 node = mem; 344 node->tn_id = 0; 345 mtx_init(&node->tn_interlock, "tmpfsni", NULL, MTX_DEF); 346 node->tn_gen = arc4random(); 347 return (0); 348 } 349 350 static void 351 tmpfs_node_fini(void *mem, int size) 352 { 353 struct tmpfs_node *node; 354 355 node = mem; 356 mtx_destroy(&node->tn_interlock); 357 } 358 359 int 360 tmpfs_subr_init(void) 361 { 362 tmpfs_pager_type = vm_pager_alloc_dyn_type(&tmpfs_pager_ops, 363 OBJT_SWAP); 364 if (tmpfs_pager_type == -1) 365 return (EINVAL); 366 tmpfs_node_pool = uma_zcreate("TMPFS node", 367 sizeof(struct tmpfs_node), tmpfs_node_ctor, tmpfs_node_dtor, 368 tmpfs_node_init, tmpfs_node_fini, UMA_ALIGN_PTR, 0); 369 VFS_SMR_ZONE_SET(tmpfs_node_pool); 370 return (0); 371 } 372 373 void 374 tmpfs_subr_uninit(void) 375 { 376 if (tmpfs_pager_type != -1) 377 vm_pager_free_dyn_type(tmpfs_pager_type); 378 tmpfs_pager_type = -1; 379 uma_zdestroy(tmpfs_node_pool); 380 } 381 382 static int 383 sysctl_mem_reserved(SYSCTL_HANDLER_ARGS) 384 { 385 int error; 386 long pages, bytes; 387 388 pages = *(long *)arg1; 389 bytes = pages * PAGE_SIZE; 390 391 error = sysctl_handle_long(oidp, &bytes, 0, req); 392 if (error || !req->newptr) 393 return (error); 394 395 pages = bytes / PAGE_SIZE; 396 if (pages < TMPFS_PAGES_MINRESERVED) 397 return (EINVAL); 398 399 *(long *)arg1 = pages; 400 return (0); 401 } 402 403 SYSCTL_PROC(_vfs_tmpfs, OID_AUTO, memory_reserved, 404 CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &tmpfs_pages_reserved, 0, 405 sysctl_mem_reserved, "L", 406 "Amount of available memory and swap below which tmpfs growth stops"); 407 408 static __inline int tmpfs_dirtree_cmp(struct tmpfs_dirent *a, 409 struct tmpfs_dirent *b); 410 RB_PROTOTYPE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp); 411 412 size_t 413 tmpfs_mem_avail(void) 414 { 415 size_t avail; 416 long reserved; 417 418 avail = swap_pager_avail + vm_free_count(); 419 reserved = atomic_load_long(&tmpfs_pages_reserved); 420 if (__predict_false(avail < reserved)) 421 return (0); 422 return (avail - reserved); 423 } 424 425 size_t 426 tmpfs_pages_used(struct tmpfs_mount *tmp) 427 { 428 const size_t node_size = sizeof(struct tmpfs_node) + 429 sizeof(struct tmpfs_dirent); 430 size_t meta_pages; 431 432 meta_pages = howmany((uintmax_t)tmp->tm_nodes_inuse * node_size, 433 PAGE_SIZE); 434 return (meta_pages + tmp->tm_pages_used); 435 } 436 437 static bool 438 tmpfs_pages_check_avail(struct tmpfs_mount *tmp, size_t req_pages) 439 { 440 if (tmpfs_mem_avail() < req_pages) 441 return (false); 442 443 if (tmp->tm_pages_max != ULONG_MAX && 444 tmp->tm_pages_max < req_pages + tmpfs_pages_used(tmp)) 445 return (false); 446 447 return (true); 448 } 449 450 static int 451 tmpfs_partial_page_invalidate(vm_object_t object, vm_pindex_t idx, int base, 452 int end, boolean_t ignerr) 453 { 454 vm_page_t m; 455 int rv, error; 456 457 VM_OBJECT_ASSERT_WLOCKED(object); 458 KASSERT(base >= 0, ("%s: base %d", __func__, base)); 459 KASSERT(end - base <= PAGE_SIZE, ("%s: base %d end %d", __func__, base, 460 end)); 461 error = 0; 462 463 retry: 464 m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT); 465 if (m != NULL) { 466 MPASS(vm_page_all_valid(m)); 467 } else if (vm_pager_has_page(object, idx, NULL, NULL)) { 468 m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL | 469 VM_ALLOC_WAITFAIL); 470 if (m == NULL) 471 goto retry; 472 vm_object_pip_add(object, 1); 473 VM_OBJECT_WUNLOCK(object); 474 rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); 475 VM_OBJECT_WLOCK(object); 476 vm_object_pip_wakeup(object); 477 if (rv == VM_PAGER_OK) { 478 /* 479 * Since the page was not resident, and therefore not 480 * recently accessed, immediately enqueue it for 481 * asynchronous laundering. The current operation is 482 * not regarded as an access. 483 */ 484 vm_page_launder(m); 485 } else { 486 vm_page_free(m); 487 m = NULL; 488 if (!ignerr) 489 error = EIO; 490 } 491 } 492 if (m != NULL) { 493 pmap_zero_page_area(m, base, end - base); 494 vm_page_set_dirty(m); 495 vm_page_xunbusy(m); 496 } 497 498 return (error); 499 } 500 501 void 502 tmpfs_ref_node(struct tmpfs_node *node) 503 { 504 #ifdef INVARIANTS 505 u_int old; 506 507 old = 508 #endif 509 refcount_acquire(&node->tn_refcount); 510 #ifdef INVARIANTS 511 KASSERT(old > 0, ("node %p zero refcount", node)); 512 #endif 513 } 514 515 /* 516 * Allocates a new node of type 'type' inside the 'tmp' mount point, with 517 * its owner set to 'uid', its group to 'gid' and its mode set to 'mode', 518 * using the credentials of the process 'p'. 519 * 520 * If the node type is set to 'VDIR', then the parent parameter must point 521 * to the parent directory of the node being created. It may only be NULL 522 * while allocating the root node. 523 * 524 * If the node type is set to 'VBLK' or 'VCHR', then the rdev parameter 525 * specifies the device the node represents. 526 * 527 * If the node type is set to 'VLNK', then the parameter target specifies 528 * the file name of the target file for the symbolic link that is being 529 * created. 530 * 531 * Note that new nodes are retrieved from the available list if it has 532 * items or, if it is empty, from the node pool as long as there is enough 533 * space to create them. 534 * 535 * Returns zero on success or an appropriate error code on failure. 536 */ 537 int 538 tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, enum vtype type, 539 uid_t uid, gid_t gid, mode_t mode, struct tmpfs_node *parent, 540 const char *target, dev_t rdev, struct tmpfs_node **node) 541 { 542 struct tmpfs_node *nnode; 543 char *symlink; 544 char symlink_smr; 545 546 /* If the root directory of the 'tmp' file system is not yet 547 * allocated, this must be the request to do it. */ 548 MPASS(IMPLIES(tmp->tm_root == NULL, parent == NULL && type == VDIR)); 549 550 MPASS(IFF(type == VLNK, target != NULL)); 551 MPASS(IFF(type == VBLK || type == VCHR, rdev != VNOVAL)); 552 553 if (tmp->tm_nodes_inuse >= tmp->tm_nodes_max) 554 return (ENOSPC); 555 if (!tmpfs_pages_check_avail(tmp, 1)) 556 return (ENOSPC); 557 558 if ((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) { 559 /* 560 * When a new tmpfs node is created for fully 561 * constructed mount point, there must be a parent 562 * node, which vnode is locked exclusively. As 563 * consequence, if the unmount is executing in 564 * parallel, vflush() cannot reclaim the parent vnode. 565 * Due to this, the check for MNTK_UNMOUNT flag is not 566 * racy: if we did not see MNTK_UNMOUNT flag, then tmp 567 * cannot be destroyed until node construction is 568 * finished and the parent vnode unlocked. 569 * 570 * Tmpfs does not need to instantiate new nodes during 571 * unmount. 572 */ 573 return (EBUSY); 574 } 575 if ((mp->mnt_kern_flag & MNT_RDONLY) != 0) 576 return (EROFS); 577 578 nnode = uma_zalloc_smr(tmpfs_node_pool, M_WAITOK); 579 580 /* Generic initialization. */ 581 nnode->tn_type = type; 582 vfs_timestamp(&nnode->tn_atime); 583 nnode->tn_birthtime = nnode->tn_ctime = nnode->tn_mtime = 584 nnode->tn_atime; 585 nnode->tn_uid = uid; 586 nnode->tn_gid = gid; 587 nnode->tn_mode = mode; 588 nnode->tn_id = alloc_unr64(&tmp->tm_ino_unr); 589 nnode->tn_refcount = 1; 590 591 /* Type-specific initialization. */ 592 switch (nnode->tn_type) { 593 case VBLK: 594 case VCHR: 595 nnode->tn_rdev = rdev; 596 break; 597 598 case VDIR: 599 RB_INIT(&nnode->tn_dir.tn_dirhead); 600 LIST_INIT(&nnode->tn_dir.tn_dupindex); 601 MPASS(parent != nnode); 602 MPASS(IMPLIES(parent == NULL, tmp->tm_root == NULL)); 603 nnode->tn_dir.tn_parent = (parent == NULL) ? nnode : parent; 604 nnode->tn_dir.tn_readdir_lastn = 0; 605 nnode->tn_dir.tn_readdir_lastp = NULL; 606 nnode->tn_links++; 607 TMPFS_NODE_LOCK(nnode->tn_dir.tn_parent); 608 nnode->tn_dir.tn_parent->tn_links++; 609 TMPFS_NODE_UNLOCK(nnode->tn_dir.tn_parent); 610 break; 611 612 case VFIFO: 613 /* FALLTHROUGH */ 614 case VSOCK: 615 break; 616 617 case VLNK: 618 MPASS(strlen(target) < MAXPATHLEN); 619 nnode->tn_size = strlen(target); 620 621 symlink = NULL; 622 if (!tmp->tm_nonc) { 623 symlink = cache_symlink_alloc(nnode->tn_size + 1, 624 M_WAITOK); 625 symlink_smr = true; 626 } 627 if (symlink == NULL) { 628 symlink = malloc(nnode->tn_size + 1, M_TMPFSNAME, 629 M_WAITOK); 630 symlink_smr = false; 631 } 632 memcpy(symlink, target, nnode->tn_size + 1); 633 634 /* 635 * Allow safe symlink resolving for lockless lookup. 636 * tmpfs_fplookup_symlink references this comment. 637 * 638 * 1. nnode is not yet visible to the world 639 * 2. both tn_link_target and tn_link_smr get populated 640 * 3. release fence publishes their content 641 * 4. tn_link_target content is immutable until node 642 * destruction, where the pointer gets set to NULL 643 * 5. tn_link_smr is never changed once set 644 * 645 * As a result it is sufficient to issue load consume 646 * on the node pointer to also get the above content 647 * in a stable manner. Worst case tn_link_smr flag 648 * may be set to true despite being stale, while the 649 * target buffer is already cleared out. 650 */ 651 atomic_store_ptr(&nnode->tn_link_target, symlink); 652 atomic_store_char((char *)&nnode->tn_link_smr, symlink_smr); 653 atomic_thread_fence_rel(); 654 break; 655 656 case VREG: 657 nnode->tn_reg.tn_aobj = 658 vm_pager_allocate(tmpfs_pager_type, NULL, 0, 659 VM_PROT_DEFAULT, 0, 660 NULL /* XXXKIB - tmpfs needs swap reservation */); 661 nnode->tn_reg.tn_aobj->un_pager.swp.swp_priv = nnode; 662 vm_object_set_flag(nnode->tn_reg.tn_aobj, OBJ_TMPFS); 663 nnode->tn_reg.tn_tmp = tmp; 664 nnode->tn_reg.tn_pages = 0; 665 break; 666 667 default: 668 panic("tmpfs_alloc_node: type %p %d", nnode, 669 (int)nnode->tn_type); 670 } 671 672 TMPFS_LOCK(tmp); 673 LIST_INSERT_HEAD(&tmp->tm_nodes_used, nnode, tn_entries); 674 nnode->tn_attached = true; 675 tmp->tm_nodes_inuse++; 676 tmp->tm_refcount++; 677 TMPFS_UNLOCK(tmp); 678 679 *node = nnode; 680 return (0); 681 } 682 683 /* 684 * Destroys the node pointed to by node from the file system 'tmp'. 685 * If the node references a directory, no entries are allowed. 686 */ 687 void 688 tmpfs_free_node(struct tmpfs_mount *tmp, struct tmpfs_node *node) 689 { 690 if (refcount_release_if_not_last(&node->tn_refcount)) 691 return; 692 693 TMPFS_LOCK(tmp); 694 TMPFS_NODE_LOCK(node); 695 if (!tmpfs_free_node_locked(tmp, node, false)) { 696 TMPFS_NODE_UNLOCK(node); 697 TMPFS_UNLOCK(tmp); 698 } 699 } 700 701 bool 702 tmpfs_free_node_locked(struct tmpfs_mount *tmp, struct tmpfs_node *node, 703 bool detach) 704 { 705 vm_object_t uobj; 706 char *symlink; 707 bool last; 708 709 TMPFS_MP_ASSERT_LOCKED(tmp); 710 TMPFS_NODE_ASSERT_LOCKED(node); 711 712 last = refcount_release(&node->tn_refcount); 713 if (node->tn_attached && (detach || last)) { 714 MPASS(tmp->tm_nodes_inuse > 0); 715 tmp->tm_nodes_inuse--; 716 LIST_REMOVE(node, tn_entries); 717 node->tn_attached = false; 718 } 719 if (!last) 720 return (false); 721 722 TMPFS_NODE_UNLOCK(node); 723 724 #ifdef INVARIANTS 725 MPASS(node->tn_vnode == NULL); 726 MPASS((node->tn_vpstate & TMPFS_VNODE_ALLOCATING) == 0); 727 728 /* 729 * Make sure this is a node type we can deal with. Everything 730 * is explicitly enumerated without the 'default' clause so 731 * the compiler can throw an error in case a new type is 732 * added. 733 */ 734 switch (node->tn_type) { 735 case VBLK: 736 case VCHR: 737 case VDIR: 738 case VFIFO: 739 case VSOCK: 740 case VLNK: 741 case VREG: 742 break; 743 case VNON: 744 case VBAD: 745 case VMARKER: 746 panic("%s: bad type %d for node %p", __func__, 747 (int)node->tn_type, node); 748 } 749 #endif 750 751 switch (node->tn_type) { 752 case VREG: 753 uobj = node->tn_reg.tn_aobj; 754 node->tn_reg.tn_aobj = NULL; 755 if (uobj != NULL) { 756 VM_OBJECT_WLOCK(uobj); 757 KASSERT((uobj->flags & OBJ_TMPFS) != 0, 758 ("tmpfs node %p uobj %p not tmpfs", node, uobj)); 759 vm_object_clear_flag(uobj, OBJ_TMPFS); 760 KASSERT(tmp->tm_pages_used >= node->tn_reg.tn_pages, 761 ("tmpfs tmp %p node %p pages %jd free %jd", tmp, 762 node, (uintmax_t)tmp->tm_pages_used, 763 (uintmax_t)node->tn_reg.tn_pages)); 764 atomic_add_long(&tmp->tm_pages_used, 765 -node->tn_reg.tn_pages); 766 VM_OBJECT_WUNLOCK(uobj); 767 } 768 tmpfs_free_tmp(tmp); 769 770 /* 771 * vm_object_deallocate() must not be called while 772 * owning tm_allnode_lock, because deallocate might 773 * sleep. Call it after tmpfs_free_tmp() does the 774 * unlock. 775 */ 776 if (uobj != NULL) 777 vm_object_deallocate(uobj); 778 779 break; 780 case VLNK: 781 tmpfs_free_tmp(tmp); 782 783 symlink = node->tn_link_target; 784 atomic_store_ptr(&node->tn_link_target, NULL); 785 if (atomic_load_char(&node->tn_link_smr)) { 786 cache_symlink_free(symlink, node->tn_size + 1); 787 } else { 788 free(symlink, M_TMPFSNAME); 789 } 790 break; 791 default: 792 tmpfs_free_tmp(tmp); 793 break; 794 } 795 796 uma_zfree_smr(tmpfs_node_pool, node); 797 return (true); 798 } 799 800 static __inline uint32_t 801 tmpfs_dirent_hash(const char *name, u_int len) 802 { 803 uint32_t hash; 804 805 hash = fnv_32_buf(name, len, FNV1_32_INIT + len) & TMPFS_DIRCOOKIE_MASK; 806 #ifdef TMPFS_DEBUG_DIRCOOKIE_DUP 807 hash &= 0xf; 808 #endif 809 if (hash < TMPFS_DIRCOOKIE_MIN) 810 hash += TMPFS_DIRCOOKIE_MIN; 811 812 return (hash); 813 } 814 815 static __inline off_t 816 tmpfs_dirent_cookie(struct tmpfs_dirent *de) 817 { 818 if (de == NULL) 819 return (TMPFS_DIRCOOKIE_EOF); 820 821 MPASS(de->td_cookie >= TMPFS_DIRCOOKIE_MIN); 822 823 return (de->td_cookie); 824 } 825 826 static __inline boolean_t 827 tmpfs_dirent_dup(struct tmpfs_dirent *de) 828 { 829 return ((de->td_cookie & TMPFS_DIRCOOKIE_DUP) != 0); 830 } 831 832 static __inline boolean_t 833 tmpfs_dirent_duphead(struct tmpfs_dirent *de) 834 { 835 return ((de->td_cookie & TMPFS_DIRCOOKIE_DUPHEAD) != 0); 836 } 837 838 void 839 tmpfs_dirent_init(struct tmpfs_dirent *de, const char *name, u_int namelen) 840 { 841 de->td_hash = de->td_cookie = tmpfs_dirent_hash(name, namelen); 842 memcpy(de->ud.td_name, name, namelen); 843 de->td_namelen = namelen; 844 } 845 846 /* 847 * Allocates a new directory entry for the node node with a name of name. 848 * The new directory entry is returned in *de. 849 * 850 * The link count of node is increased by one to reflect the new object 851 * referencing it. 852 * 853 * Returns zero on success or an appropriate error code on failure. 854 */ 855 int 856 tmpfs_alloc_dirent(struct tmpfs_mount *tmp, struct tmpfs_node *node, 857 const char *name, u_int len, struct tmpfs_dirent **de) 858 { 859 struct tmpfs_dirent *nde; 860 861 nde = malloc(sizeof(*nde), M_TMPFSDIR, M_WAITOK); 862 nde->td_node = node; 863 if (name != NULL) { 864 nde->ud.td_name = malloc(len, M_TMPFSNAME, M_WAITOK); 865 tmpfs_dirent_init(nde, name, len); 866 } else 867 nde->td_namelen = 0; 868 if (node != NULL) 869 node->tn_links++; 870 871 *de = nde; 872 873 return (0); 874 } 875 876 /* 877 * Frees a directory entry. It is the caller's responsibility to destroy 878 * the node referenced by it if needed. 879 * 880 * The link count of node is decreased by one to reflect the removal of an 881 * object that referenced it. This only happens if 'node_exists' is true; 882 * otherwise the function will not access the node referred to by the 883 * directory entry, as it may already have been released from the outside. 884 */ 885 void 886 tmpfs_free_dirent(struct tmpfs_mount *tmp, struct tmpfs_dirent *de) 887 { 888 struct tmpfs_node *node; 889 890 node = de->td_node; 891 if (node != NULL) { 892 MPASS(node->tn_links > 0); 893 node->tn_links--; 894 } 895 if (!tmpfs_dirent_duphead(de) && de->ud.td_name != NULL) 896 free(de->ud.td_name, M_TMPFSNAME); 897 free(de, M_TMPFSDIR); 898 } 899 900 void 901 tmpfs_destroy_vobject(struct vnode *vp, vm_object_t obj) 902 { 903 bool want_vrele; 904 905 ASSERT_VOP_ELOCKED(vp, "tmpfs_destroy_vobject"); 906 if (vp->v_type != VREG || obj == NULL) 907 return; 908 909 VM_OBJECT_WLOCK(obj); 910 VI_LOCK(vp); 911 /* 912 * May be going through forced unmount. 913 */ 914 want_vrele = false; 915 if ((obj->flags & OBJ_TMPFS_VREF) != 0) { 916 vm_object_clear_flag(obj, OBJ_TMPFS_VREF); 917 want_vrele = true; 918 } 919 920 if (vp->v_writecount < 0) 921 vp->v_writecount = 0; 922 VI_UNLOCK(vp); 923 VM_OBJECT_WUNLOCK(obj); 924 if (want_vrele) { 925 vrele(vp); 926 } 927 } 928 929 /* 930 * Allocates a new vnode for the node node or returns a new reference to 931 * an existing one if the node had already a vnode referencing it. The 932 * resulting locked vnode is returned in *vpp. 933 * 934 * Returns zero on success or an appropriate error code on failure. 935 */ 936 int 937 tmpfs_alloc_vp(struct mount *mp, struct tmpfs_node *node, int lkflag, 938 struct vnode **vpp) 939 { 940 struct vnode *vp; 941 enum vgetstate vs; 942 struct tmpfs_mount *tm; 943 vm_object_t object; 944 int error; 945 946 error = 0; 947 tm = VFS_TO_TMPFS(mp); 948 TMPFS_NODE_LOCK(node); 949 tmpfs_ref_node(node); 950 loop: 951 TMPFS_NODE_ASSERT_LOCKED(node); 952 if ((vp = node->tn_vnode) != NULL) { 953 MPASS((node->tn_vpstate & TMPFS_VNODE_DOOMED) == 0); 954 if ((node->tn_type == VDIR && node->tn_dir.tn_parent == NULL) || 955 (VN_IS_DOOMED(vp) && 956 (lkflag & LK_NOWAIT) != 0)) { 957 TMPFS_NODE_UNLOCK(node); 958 error = ENOENT; 959 vp = NULL; 960 goto out; 961 } 962 if (VN_IS_DOOMED(vp)) { 963 node->tn_vpstate |= TMPFS_VNODE_WRECLAIM; 964 while ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0) { 965 msleep(&node->tn_vnode, TMPFS_NODE_MTX(node), 966 0, "tmpfsE", 0); 967 } 968 goto loop; 969 } 970 vs = vget_prep(vp); 971 TMPFS_NODE_UNLOCK(node); 972 error = vget_finish(vp, lkflag, vs); 973 if (error == ENOENT) { 974 TMPFS_NODE_LOCK(node); 975 goto loop; 976 } 977 if (error != 0) { 978 vp = NULL; 979 goto out; 980 } 981 982 /* 983 * Make sure the vnode is still there after 984 * getting the interlock to avoid racing a free. 985 */ 986 if (node->tn_vnode != vp) { 987 vput(vp); 988 TMPFS_NODE_LOCK(node); 989 goto loop; 990 } 991 992 goto out; 993 } 994 995 if ((node->tn_vpstate & TMPFS_VNODE_DOOMED) || 996 (node->tn_type == VDIR && node->tn_dir.tn_parent == NULL)) { 997 TMPFS_NODE_UNLOCK(node); 998 error = ENOENT; 999 vp = NULL; 1000 goto out; 1001 } 1002 1003 /* 1004 * otherwise lock the vp list while we call getnewvnode 1005 * since that can block. 1006 */ 1007 if (node->tn_vpstate & TMPFS_VNODE_ALLOCATING) { 1008 node->tn_vpstate |= TMPFS_VNODE_WANT; 1009 error = msleep((caddr_t) &node->tn_vpstate, 1010 TMPFS_NODE_MTX(node), 0, "tmpfs_alloc_vp", 0); 1011 if (error != 0) 1012 goto out; 1013 goto loop; 1014 } else 1015 node->tn_vpstate |= TMPFS_VNODE_ALLOCATING; 1016 1017 TMPFS_NODE_UNLOCK(node); 1018 1019 /* Get a new vnode and associate it with our node. */ 1020 error = getnewvnode("tmpfs", mp, VFS_TO_TMPFS(mp)->tm_nonc ? 1021 &tmpfs_vnodeop_nonc_entries : &tmpfs_vnodeop_entries, &vp); 1022 if (error != 0) 1023 goto unlock; 1024 MPASS(vp != NULL); 1025 1026 /* lkflag is ignored, the lock is exclusive */ 1027 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1028 1029 vp->v_data = node; 1030 vp->v_type = node->tn_type; 1031 1032 /* Type-specific initialization. */ 1033 switch (node->tn_type) { 1034 case VBLK: 1035 /* FALLTHROUGH */ 1036 case VCHR: 1037 /* FALLTHROUGH */ 1038 case VLNK: 1039 /* FALLTHROUGH */ 1040 case VSOCK: 1041 break; 1042 case VFIFO: 1043 vp->v_op = &tmpfs_fifoop_entries; 1044 break; 1045 case VREG: 1046 object = node->tn_reg.tn_aobj; 1047 VM_OBJECT_WLOCK(object); 1048 KASSERT((object->flags & OBJ_TMPFS_VREF) == 0, 1049 ("%s: object %p with OBJ_TMPFS_VREF but without vnode", 1050 __func__, object)); 1051 KASSERT(object->un_pager.swp.writemappings == 0, 1052 ("%s: object %p has writemappings", 1053 __func__, object)); 1054 VI_LOCK(vp); 1055 KASSERT(vp->v_object == NULL, ("Not NULL v_object in tmpfs")); 1056 vp->v_object = object; 1057 vn_irflag_set_locked(vp, VIRF_PGREAD | VIRF_TEXT_REF); 1058 VI_UNLOCK(vp); 1059 VM_OBJECT_WUNLOCK(object); 1060 break; 1061 case VDIR: 1062 MPASS(node->tn_dir.tn_parent != NULL); 1063 if (node->tn_dir.tn_parent == node) 1064 vp->v_vflag |= VV_ROOT; 1065 break; 1066 1067 default: 1068 panic("tmpfs_alloc_vp: type %p %d", node, (int)node->tn_type); 1069 } 1070 if (vp->v_type != VFIFO) 1071 VN_LOCK_ASHARE(vp); 1072 1073 error = insmntque1(vp, mp); 1074 if (error != 0) { 1075 /* Need to clear v_object for insmntque failure. */ 1076 tmpfs_destroy_vobject(vp, vp->v_object); 1077 vp->v_object = NULL; 1078 vp->v_data = NULL; 1079 vp->v_op = &dead_vnodeops; 1080 vgone(vp); 1081 vput(vp); 1082 vp = NULL; 1083 } 1084 1085 unlock: 1086 TMPFS_NODE_LOCK(node); 1087 1088 MPASS(node->tn_vpstate & TMPFS_VNODE_ALLOCATING); 1089 node->tn_vpstate &= ~TMPFS_VNODE_ALLOCATING; 1090 node->tn_vnode = vp; 1091 1092 if (node->tn_vpstate & TMPFS_VNODE_WANT) { 1093 node->tn_vpstate &= ~TMPFS_VNODE_WANT; 1094 TMPFS_NODE_UNLOCK(node); 1095 wakeup((caddr_t) &node->tn_vpstate); 1096 } else 1097 TMPFS_NODE_UNLOCK(node); 1098 1099 out: 1100 if (error == 0) { 1101 *vpp = vp; 1102 1103 #ifdef INVARIANTS 1104 MPASS(*vpp != NULL && VOP_ISLOCKED(*vpp)); 1105 TMPFS_NODE_LOCK(node); 1106 MPASS(*vpp == node->tn_vnode); 1107 TMPFS_NODE_UNLOCK(node); 1108 #endif 1109 } 1110 tmpfs_free_node(tm, node); 1111 1112 return (error); 1113 } 1114 1115 /* 1116 * Destroys the association between the vnode vp and the node it 1117 * references. 1118 */ 1119 void 1120 tmpfs_free_vp(struct vnode *vp) 1121 { 1122 struct tmpfs_node *node; 1123 1124 node = VP_TO_TMPFS_NODE(vp); 1125 1126 TMPFS_NODE_ASSERT_LOCKED(node); 1127 node->tn_vnode = NULL; 1128 if ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0) 1129 wakeup(&node->tn_vnode); 1130 node->tn_vpstate &= ~TMPFS_VNODE_WRECLAIM; 1131 vp->v_data = NULL; 1132 } 1133 1134 /* 1135 * Allocates a new file of type 'type' and adds it to the parent directory 1136 * 'dvp'; this addition is done using the component name given in 'cnp'. 1137 * The ownership of the new file is automatically assigned based on the 1138 * credentials of the caller (through 'cnp'), the group is set based on 1139 * the parent directory and the mode is determined from the 'vap' argument. 1140 * If successful, *vpp holds a vnode to the newly created file and zero 1141 * is returned. Otherwise *vpp is NULL and the function returns an 1142 * appropriate error code. 1143 */ 1144 int 1145 tmpfs_alloc_file(struct vnode *dvp, struct vnode **vpp, struct vattr *vap, 1146 struct componentname *cnp, const char *target) 1147 { 1148 int error; 1149 struct tmpfs_dirent *de; 1150 struct tmpfs_mount *tmp; 1151 struct tmpfs_node *dnode; 1152 struct tmpfs_node *node; 1153 struct tmpfs_node *parent; 1154 1155 ASSERT_VOP_ELOCKED(dvp, "tmpfs_alloc_file"); 1156 1157 tmp = VFS_TO_TMPFS(dvp->v_mount); 1158 dnode = VP_TO_TMPFS_DIR(dvp); 1159 *vpp = NULL; 1160 1161 /* If the entry we are creating is a directory, we cannot overflow 1162 * the number of links of its parent, because it will get a new 1163 * link. */ 1164 if (vap->va_type == VDIR) { 1165 /* Ensure that we do not overflow the maximum number of links 1166 * imposed by the system. */ 1167 MPASS(dnode->tn_links <= TMPFS_LINK_MAX); 1168 if (dnode->tn_links == TMPFS_LINK_MAX) { 1169 return (EMLINK); 1170 } 1171 1172 parent = dnode; 1173 MPASS(parent != NULL); 1174 } else 1175 parent = NULL; 1176 1177 /* Allocate a node that represents the new file. */ 1178 error = tmpfs_alloc_node(dvp->v_mount, tmp, vap->va_type, 1179 cnp->cn_cred->cr_uid, dnode->tn_gid, vap->va_mode, parent, 1180 target, vap->va_rdev, &node); 1181 if (error != 0) 1182 return (error); 1183 1184 /* Allocate a directory entry that points to the new file. */ 1185 error = tmpfs_alloc_dirent(tmp, node, cnp->cn_nameptr, cnp->cn_namelen, 1186 &de); 1187 if (error != 0) { 1188 tmpfs_free_node(tmp, node); 1189 return (error); 1190 } 1191 1192 /* Allocate a vnode for the new file. */ 1193 error = tmpfs_alloc_vp(dvp->v_mount, node, LK_EXCLUSIVE, vpp); 1194 if (error != 0) { 1195 tmpfs_free_dirent(tmp, de); 1196 tmpfs_free_node(tmp, node); 1197 return (error); 1198 } 1199 1200 /* Now that all required items are allocated, we can proceed to 1201 * insert the new node into the directory, an operation that 1202 * cannot fail. */ 1203 if (cnp->cn_flags & ISWHITEOUT) 1204 tmpfs_dir_whiteout_remove(dvp, cnp); 1205 tmpfs_dir_attach(dvp, de); 1206 return (0); 1207 } 1208 1209 struct tmpfs_dirent * 1210 tmpfs_dir_first(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc) 1211 { 1212 struct tmpfs_dirent *de; 1213 1214 de = RB_MIN(tmpfs_dir, &dnode->tn_dir.tn_dirhead); 1215 dc->tdc_tree = de; 1216 if (de != NULL && tmpfs_dirent_duphead(de)) 1217 de = LIST_FIRST(&de->ud.td_duphead); 1218 dc->tdc_current = de; 1219 1220 return (dc->tdc_current); 1221 } 1222 1223 struct tmpfs_dirent * 1224 tmpfs_dir_next(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc) 1225 { 1226 struct tmpfs_dirent *de; 1227 1228 MPASS(dc->tdc_tree != NULL); 1229 if (tmpfs_dirent_dup(dc->tdc_current)) { 1230 dc->tdc_current = LIST_NEXT(dc->tdc_current, uh.td_dup.entries); 1231 if (dc->tdc_current != NULL) 1232 return (dc->tdc_current); 1233 } 1234 dc->tdc_tree = dc->tdc_current = RB_NEXT(tmpfs_dir, 1235 &dnode->tn_dir.tn_dirhead, dc->tdc_tree); 1236 if ((de = dc->tdc_current) != NULL && tmpfs_dirent_duphead(de)) { 1237 dc->tdc_current = LIST_FIRST(&de->ud.td_duphead); 1238 MPASS(dc->tdc_current != NULL); 1239 } 1240 1241 return (dc->tdc_current); 1242 } 1243 1244 /* Lookup directory entry in RB-Tree. Function may return duphead entry. */ 1245 static struct tmpfs_dirent * 1246 tmpfs_dir_xlookup_hash(struct tmpfs_node *dnode, uint32_t hash) 1247 { 1248 struct tmpfs_dirent *de, dekey; 1249 1250 dekey.td_hash = hash; 1251 de = RB_FIND(tmpfs_dir, &dnode->tn_dir.tn_dirhead, &dekey); 1252 return (de); 1253 } 1254 1255 /* Lookup directory entry by cookie, initialize directory cursor accordingly. */ 1256 static struct tmpfs_dirent * 1257 tmpfs_dir_lookup_cookie(struct tmpfs_node *node, off_t cookie, 1258 struct tmpfs_dir_cursor *dc) 1259 { 1260 struct tmpfs_dir *dirhead = &node->tn_dir.tn_dirhead; 1261 struct tmpfs_dirent *de, dekey; 1262 1263 MPASS(cookie >= TMPFS_DIRCOOKIE_MIN); 1264 1265 if (cookie == node->tn_dir.tn_readdir_lastn && 1266 (de = node->tn_dir.tn_readdir_lastp) != NULL) { 1267 /* Protect against possible race, tn_readdir_last[pn] 1268 * may be updated with only shared vnode lock held. */ 1269 if (cookie == tmpfs_dirent_cookie(de)) 1270 goto out; 1271 } 1272 1273 if ((cookie & TMPFS_DIRCOOKIE_DUP) != 0) { 1274 LIST_FOREACH(de, &node->tn_dir.tn_dupindex, 1275 uh.td_dup.index_entries) { 1276 MPASS(tmpfs_dirent_dup(de)); 1277 if (de->td_cookie == cookie) 1278 goto out; 1279 /* dupindex list is sorted. */ 1280 if (de->td_cookie < cookie) { 1281 de = NULL; 1282 goto out; 1283 } 1284 } 1285 MPASS(de == NULL); 1286 goto out; 1287 } 1288 1289 if ((cookie & TMPFS_DIRCOOKIE_MASK) != cookie) { 1290 de = NULL; 1291 } else { 1292 dekey.td_hash = cookie; 1293 /* Recover if direntry for cookie was removed */ 1294 de = RB_NFIND(tmpfs_dir, dirhead, &dekey); 1295 } 1296 dc->tdc_tree = de; 1297 dc->tdc_current = de; 1298 if (de != NULL && tmpfs_dirent_duphead(de)) { 1299 dc->tdc_current = LIST_FIRST(&de->ud.td_duphead); 1300 MPASS(dc->tdc_current != NULL); 1301 } 1302 return (dc->tdc_current); 1303 1304 out: 1305 dc->tdc_tree = de; 1306 dc->tdc_current = de; 1307 if (de != NULL && tmpfs_dirent_dup(de)) 1308 dc->tdc_tree = tmpfs_dir_xlookup_hash(node, 1309 de->td_hash); 1310 return (dc->tdc_current); 1311 } 1312 1313 /* 1314 * Looks for a directory entry in the directory represented by node. 1315 * 'cnp' describes the name of the entry to look for. Note that the . 1316 * and .. components are not allowed as they do not physically exist 1317 * within directories. 1318 * 1319 * Returns a pointer to the entry when found, otherwise NULL. 1320 */ 1321 struct tmpfs_dirent * 1322 tmpfs_dir_lookup(struct tmpfs_node *node, struct tmpfs_node *f, 1323 struct componentname *cnp) 1324 { 1325 struct tmpfs_dir_duphead *duphead; 1326 struct tmpfs_dirent *de; 1327 uint32_t hash; 1328 1329 MPASS(IMPLIES(cnp->cn_namelen == 1, cnp->cn_nameptr[0] != '.')); 1330 MPASS(IMPLIES(cnp->cn_namelen == 2, !(cnp->cn_nameptr[0] == '.' && 1331 cnp->cn_nameptr[1] == '.'))); 1332 TMPFS_VALIDATE_DIR(node); 1333 1334 hash = tmpfs_dirent_hash(cnp->cn_nameptr, cnp->cn_namelen); 1335 de = tmpfs_dir_xlookup_hash(node, hash); 1336 if (de != NULL && tmpfs_dirent_duphead(de)) { 1337 duphead = &de->ud.td_duphead; 1338 LIST_FOREACH(de, duphead, uh.td_dup.entries) { 1339 if (TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr, 1340 cnp->cn_namelen)) 1341 break; 1342 } 1343 } else if (de != NULL) { 1344 if (!TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr, 1345 cnp->cn_namelen)) 1346 de = NULL; 1347 } 1348 if (de != NULL && f != NULL && de->td_node != f) 1349 de = NULL; 1350 1351 return (de); 1352 } 1353 1354 /* 1355 * Attach duplicate-cookie directory entry nde to dnode and insert to dupindex 1356 * list, allocate new cookie value. 1357 */ 1358 static void 1359 tmpfs_dir_attach_dup(struct tmpfs_node *dnode, 1360 struct tmpfs_dir_duphead *duphead, struct tmpfs_dirent *nde) 1361 { 1362 struct tmpfs_dir_duphead *dupindex; 1363 struct tmpfs_dirent *de, *pde; 1364 1365 dupindex = &dnode->tn_dir.tn_dupindex; 1366 de = LIST_FIRST(dupindex); 1367 if (de == NULL || de->td_cookie < TMPFS_DIRCOOKIE_DUP_MAX) { 1368 if (de == NULL) 1369 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN; 1370 else 1371 nde->td_cookie = de->td_cookie + 1; 1372 MPASS(tmpfs_dirent_dup(nde)); 1373 LIST_INSERT_HEAD(dupindex, nde, uh.td_dup.index_entries); 1374 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); 1375 return; 1376 } 1377 1378 /* 1379 * Cookie numbers are near exhaustion. Scan dupindex list for unused 1380 * numbers. dupindex list is sorted in descending order. Keep it so 1381 * after inserting nde. 1382 */ 1383 while (1) { 1384 pde = de; 1385 de = LIST_NEXT(de, uh.td_dup.index_entries); 1386 if (de == NULL && pde->td_cookie != TMPFS_DIRCOOKIE_DUP_MIN) { 1387 /* 1388 * Last element of the index doesn't have minimal cookie 1389 * value, use it. 1390 */ 1391 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN; 1392 LIST_INSERT_AFTER(pde, nde, uh.td_dup.index_entries); 1393 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); 1394 return; 1395 } else if (de == NULL) { 1396 /* 1397 * We are so lucky have 2^30 hash duplicates in single 1398 * directory :) Return largest possible cookie value. 1399 * It should be fine except possible issues with 1400 * VOP_READDIR restart. 1401 */ 1402 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MAX; 1403 LIST_INSERT_HEAD(dupindex, nde, 1404 uh.td_dup.index_entries); 1405 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); 1406 return; 1407 } 1408 if (de->td_cookie + 1 == pde->td_cookie || 1409 de->td_cookie >= TMPFS_DIRCOOKIE_DUP_MAX) 1410 continue; /* No hole or invalid cookie. */ 1411 nde->td_cookie = de->td_cookie + 1; 1412 MPASS(tmpfs_dirent_dup(nde)); 1413 MPASS(pde->td_cookie > nde->td_cookie); 1414 MPASS(nde->td_cookie > de->td_cookie); 1415 LIST_INSERT_BEFORE(de, nde, uh.td_dup.index_entries); 1416 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); 1417 return; 1418 } 1419 } 1420 1421 /* 1422 * Attaches the directory entry de to the directory represented by vp. 1423 * Note that this does not change the link count of the node pointed by 1424 * the directory entry, as this is done by tmpfs_alloc_dirent. 1425 */ 1426 void 1427 tmpfs_dir_attach(struct vnode *vp, struct tmpfs_dirent *de) 1428 { 1429 struct tmpfs_node *dnode; 1430 struct tmpfs_dirent *xde, *nde; 1431 1432 ASSERT_VOP_ELOCKED(vp, __func__); 1433 MPASS(de->td_namelen > 0); 1434 MPASS(de->td_hash >= TMPFS_DIRCOOKIE_MIN); 1435 MPASS(de->td_cookie == de->td_hash); 1436 1437 dnode = VP_TO_TMPFS_DIR(vp); 1438 dnode->tn_dir.tn_readdir_lastn = 0; 1439 dnode->tn_dir.tn_readdir_lastp = NULL; 1440 1441 MPASS(!tmpfs_dirent_dup(de)); 1442 xde = RB_INSERT(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de); 1443 if (xde != NULL && tmpfs_dirent_duphead(xde)) 1444 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de); 1445 else if (xde != NULL) { 1446 /* 1447 * Allocate new duphead. Swap xde with duphead to avoid 1448 * adding/removing elements with the same hash. 1449 */ 1450 MPASS(!tmpfs_dirent_dup(xde)); 1451 tmpfs_alloc_dirent(VFS_TO_TMPFS(vp->v_mount), NULL, NULL, 0, 1452 &nde); 1453 /* *nde = *xde; XXX gcc 4.2.1 may generate invalid code. */ 1454 memcpy(nde, xde, sizeof(*xde)); 1455 xde->td_cookie |= TMPFS_DIRCOOKIE_DUPHEAD; 1456 LIST_INIT(&xde->ud.td_duphead); 1457 xde->td_namelen = 0; 1458 xde->td_node = NULL; 1459 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, nde); 1460 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de); 1461 } 1462 dnode->tn_size += sizeof(struct tmpfs_dirent); 1463 dnode->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED; 1464 dnode->tn_accessed = true; 1465 tmpfs_update(vp); 1466 } 1467 1468 /* 1469 * Detaches the directory entry de from the directory represented by vp. 1470 * Note that this does not change the link count of the node pointed by 1471 * the directory entry, as this is done by tmpfs_free_dirent. 1472 */ 1473 void 1474 tmpfs_dir_detach(struct vnode *vp, struct tmpfs_dirent *de) 1475 { 1476 struct tmpfs_mount *tmp; 1477 struct tmpfs_dir *head; 1478 struct tmpfs_node *dnode; 1479 struct tmpfs_dirent *xde; 1480 1481 ASSERT_VOP_ELOCKED(vp, __func__); 1482 1483 dnode = VP_TO_TMPFS_DIR(vp); 1484 head = &dnode->tn_dir.tn_dirhead; 1485 dnode->tn_dir.tn_readdir_lastn = 0; 1486 dnode->tn_dir.tn_readdir_lastp = NULL; 1487 1488 if (tmpfs_dirent_dup(de)) { 1489 /* Remove duphead if de was last entry. */ 1490 if (LIST_NEXT(de, uh.td_dup.entries) == NULL) { 1491 xde = tmpfs_dir_xlookup_hash(dnode, de->td_hash); 1492 MPASS(tmpfs_dirent_duphead(xde)); 1493 } else 1494 xde = NULL; 1495 LIST_REMOVE(de, uh.td_dup.entries); 1496 LIST_REMOVE(de, uh.td_dup.index_entries); 1497 if (xde != NULL) { 1498 if (LIST_EMPTY(&xde->ud.td_duphead)) { 1499 RB_REMOVE(tmpfs_dir, head, xde); 1500 tmp = VFS_TO_TMPFS(vp->v_mount); 1501 MPASS(xde->td_node == NULL); 1502 tmpfs_free_dirent(tmp, xde); 1503 } 1504 } 1505 de->td_cookie = de->td_hash; 1506 } else 1507 RB_REMOVE(tmpfs_dir, head, de); 1508 1509 dnode->tn_size -= sizeof(struct tmpfs_dirent); 1510 dnode->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED; 1511 dnode->tn_accessed = true; 1512 tmpfs_update(vp); 1513 } 1514 1515 void 1516 tmpfs_dir_destroy(struct tmpfs_mount *tmp, struct tmpfs_node *dnode) 1517 { 1518 struct tmpfs_dirent *de, *dde, *nde; 1519 1520 RB_FOREACH_SAFE(de, tmpfs_dir, &dnode->tn_dir.tn_dirhead, nde) { 1521 RB_REMOVE(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de); 1522 /* Node may already be destroyed. */ 1523 de->td_node = NULL; 1524 if (tmpfs_dirent_duphead(de)) { 1525 while ((dde = LIST_FIRST(&de->ud.td_duphead)) != NULL) { 1526 LIST_REMOVE(dde, uh.td_dup.entries); 1527 dde->td_node = NULL; 1528 tmpfs_free_dirent(tmp, dde); 1529 } 1530 } 1531 tmpfs_free_dirent(tmp, de); 1532 } 1533 } 1534 1535 /* 1536 * Helper function for tmpfs_readdir. Creates a '.' entry for the given 1537 * directory and returns it in the uio space. The function returns 0 1538 * on success, -1 if there was not enough space in the uio structure to 1539 * hold the directory entry or an appropriate error code if another 1540 * error happens. 1541 */ 1542 static int 1543 tmpfs_dir_getdotdent(struct tmpfs_mount *tm, struct tmpfs_node *node, 1544 struct uio *uio) 1545 { 1546 int error; 1547 struct dirent dent; 1548 1549 TMPFS_VALIDATE_DIR(node); 1550 MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOT); 1551 1552 dent.d_fileno = node->tn_id; 1553 dent.d_off = TMPFS_DIRCOOKIE_DOTDOT; 1554 dent.d_type = DT_DIR; 1555 dent.d_namlen = 1; 1556 dent.d_name[0] = '.'; 1557 dent.d_reclen = GENERIC_DIRSIZ(&dent); 1558 dirent_terminate(&dent); 1559 1560 if (dent.d_reclen > uio->uio_resid) 1561 error = EJUSTRETURN; 1562 else 1563 error = uiomove(&dent, dent.d_reclen, uio); 1564 1565 tmpfs_set_accessed(tm, node); 1566 1567 return (error); 1568 } 1569 1570 /* 1571 * Helper function for tmpfs_readdir. Creates a '..' entry for the given 1572 * directory and returns it in the uio space. The function returns 0 1573 * on success, -1 if there was not enough space in the uio structure to 1574 * hold the directory entry or an appropriate error code if another 1575 * error happens. 1576 */ 1577 static int 1578 tmpfs_dir_getdotdotdent(struct tmpfs_mount *tm, struct tmpfs_node *node, 1579 struct uio *uio, off_t next) 1580 { 1581 struct tmpfs_node *parent; 1582 struct dirent dent; 1583 int error; 1584 1585 TMPFS_VALIDATE_DIR(node); 1586 MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOTDOT); 1587 1588 /* 1589 * Return ENOENT if the current node is already removed. 1590 */ 1591 TMPFS_ASSERT_LOCKED(node); 1592 parent = node->tn_dir.tn_parent; 1593 if (parent == NULL) 1594 return (ENOENT); 1595 1596 dent.d_fileno = parent->tn_id; 1597 dent.d_off = next; 1598 dent.d_type = DT_DIR; 1599 dent.d_namlen = 2; 1600 dent.d_name[0] = '.'; 1601 dent.d_name[1] = '.'; 1602 dent.d_reclen = GENERIC_DIRSIZ(&dent); 1603 dirent_terminate(&dent); 1604 1605 if (dent.d_reclen > uio->uio_resid) 1606 error = EJUSTRETURN; 1607 else 1608 error = uiomove(&dent, dent.d_reclen, uio); 1609 1610 tmpfs_set_accessed(tm, node); 1611 1612 return (error); 1613 } 1614 1615 /* 1616 * Helper function for tmpfs_readdir. Returns as much directory entries 1617 * as can fit in the uio space. The read starts at uio->uio_offset. 1618 * The function returns 0 on success, -1 if there was not enough space 1619 * in the uio structure to hold the directory entry or an appropriate 1620 * error code if another error happens. 1621 */ 1622 int 1623 tmpfs_dir_getdents(struct tmpfs_mount *tm, struct tmpfs_node *node, 1624 struct uio *uio, int maxcookies, uint64_t *cookies, int *ncookies) 1625 { 1626 struct tmpfs_dir_cursor dc; 1627 struct tmpfs_dirent *de, *nde; 1628 off_t off; 1629 int error; 1630 1631 TMPFS_VALIDATE_DIR(node); 1632 1633 off = 0; 1634 1635 /* 1636 * Lookup the node from the current offset. The starting offset of 1637 * 0 will lookup both '.' and '..', and then the first real entry, 1638 * or EOF if there are none. Then find all entries for the dir that 1639 * fit into the buffer. Once no more entries are found (de == NULL), 1640 * the offset is set to TMPFS_DIRCOOKIE_EOF, which will cause the next 1641 * call to return 0. 1642 */ 1643 switch (uio->uio_offset) { 1644 case TMPFS_DIRCOOKIE_DOT: 1645 error = tmpfs_dir_getdotdent(tm, node, uio); 1646 if (error != 0) 1647 return (error); 1648 uio->uio_offset = off = TMPFS_DIRCOOKIE_DOTDOT; 1649 if (cookies != NULL) 1650 cookies[(*ncookies)++] = off; 1651 /* FALLTHROUGH */ 1652 case TMPFS_DIRCOOKIE_DOTDOT: 1653 de = tmpfs_dir_first(node, &dc); 1654 off = tmpfs_dirent_cookie(de); 1655 error = tmpfs_dir_getdotdotdent(tm, node, uio, off); 1656 if (error != 0) 1657 return (error); 1658 uio->uio_offset = off; 1659 if (cookies != NULL) 1660 cookies[(*ncookies)++] = off; 1661 /* EOF. */ 1662 if (de == NULL) 1663 return (0); 1664 break; 1665 case TMPFS_DIRCOOKIE_EOF: 1666 return (0); 1667 default: 1668 de = tmpfs_dir_lookup_cookie(node, uio->uio_offset, &dc); 1669 if (de == NULL) 1670 return (EINVAL); 1671 if (cookies != NULL) 1672 off = tmpfs_dirent_cookie(de); 1673 } 1674 1675 /* 1676 * Read as much entries as possible; i.e., until we reach the end of the 1677 * directory or we exhaust uio space. 1678 */ 1679 do { 1680 struct dirent d; 1681 1682 /* 1683 * Create a dirent structure representing the current tmpfs_node 1684 * and fill it. 1685 */ 1686 if (de->td_node == NULL) { 1687 d.d_fileno = 1; 1688 d.d_type = DT_WHT; 1689 } else { 1690 d.d_fileno = de->td_node->tn_id; 1691 switch (de->td_node->tn_type) { 1692 case VBLK: 1693 d.d_type = DT_BLK; 1694 break; 1695 1696 case VCHR: 1697 d.d_type = DT_CHR; 1698 break; 1699 1700 case VDIR: 1701 d.d_type = DT_DIR; 1702 break; 1703 1704 case VFIFO: 1705 d.d_type = DT_FIFO; 1706 break; 1707 1708 case VLNK: 1709 d.d_type = DT_LNK; 1710 break; 1711 1712 case VREG: 1713 d.d_type = DT_REG; 1714 break; 1715 1716 case VSOCK: 1717 d.d_type = DT_SOCK; 1718 break; 1719 1720 default: 1721 panic("tmpfs_dir_getdents: type %p %d", 1722 de->td_node, (int)de->td_node->tn_type); 1723 } 1724 } 1725 d.d_namlen = de->td_namelen; 1726 MPASS(de->td_namelen < sizeof(d.d_name)); 1727 (void)memcpy(d.d_name, de->ud.td_name, de->td_namelen); 1728 d.d_reclen = GENERIC_DIRSIZ(&d); 1729 1730 /* 1731 * Stop reading if the directory entry we are treating is bigger 1732 * than the amount of data that can be returned. 1733 */ 1734 if (d.d_reclen > uio->uio_resid) { 1735 error = EJUSTRETURN; 1736 break; 1737 } 1738 1739 nde = tmpfs_dir_next(node, &dc); 1740 d.d_off = tmpfs_dirent_cookie(nde); 1741 dirent_terminate(&d); 1742 1743 /* 1744 * Copy the new dirent structure into the output buffer and 1745 * advance pointers. 1746 */ 1747 error = uiomove(&d, d.d_reclen, uio); 1748 if (error == 0) { 1749 de = nde; 1750 if (cookies != NULL) { 1751 off = tmpfs_dirent_cookie(de); 1752 MPASS(*ncookies < maxcookies); 1753 cookies[(*ncookies)++] = off; 1754 } 1755 } 1756 } while (error == 0 && uio->uio_resid > 0 && de != NULL); 1757 1758 /* Skip setting off when using cookies as it is already done above. */ 1759 if (cookies == NULL) 1760 off = tmpfs_dirent_cookie(de); 1761 1762 /* Update the offset and cache. */ 1763 uio->uio_offset = off; 1764 node->tn_dir.tn_readdir_lastn = off; 1765 node->tn_dir.tn_readdir_lastp = de; 1766 1767 tmpfs_set_accessed(tm, node); 1768 return (error); 1769 } 1770 1771 int 1772 tmpfs_dir_whiteout_add(struct vnode *dvp, struct componentname *cnp) 1773 { 1774 struct tmpfs_dirent *de; 1775 int error; 1776 1777 error = tmpfs_alloc_dirent(VFS_TO_TMPFS(dvp->v_mount), NULL, 1778 cnp->cn_nameptr, cnp->cn_namelen, &de); 1779 if (error != 0) 1780 return (error); 1781 tmpfs_dir_attach(dvp, de); 1782 return (0); 1783 } 1784 1785 void 1786 tmpfs_dir_whiteout_remove(struct vnode *dvp, struct componentname *cnp) 1787 { 1788 struct tmpfs_dirent *de; 1789 1790 de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(dvp), NULL, cnp); 1791 MPASS(de != NULL && de->td_node == NULL); 1792 tmpfs_dir_detach(dvp, de); 1793 tmpfs_free_dirent(VFS_TO_TMPFS(dvp->v_mount), de); 1794 } 1795 1796 /* 1797 * Resizes the aobj associated with the regular file pointed to by 'vp' to the 1798 * size 'newsize'. 'vp' must point to a vnode that represents a regular file. 1799 * 'newsize' must be positive. 1800 * 1801 * Returns zero on success or an appropriate error code on failure. 1802 */ 1803 int 1804 tmpfs_reg_resize(struct vnode *vp, off_t newsize, boolean_t ignerr) 1805 { 1806 struct tmpfs_node *node; 1807 vm_object_t uobj; 1808 vm_pindex_t idx, newpages, oldpages; 1809 off_t oldsize; 1810 int base, error; 1811 1812 MPASS(vp->v_type == VREG); 1813 MPASS(newsize >= 0); 1814 1815 node = VP_TO_TMPFS_NODE(vp); 1816 uobj = node->tn_reg.tn_aobj; 1817 1818 /* 1819 * Convert the old and new sizes to the number of pages needed to 1820 * store them. It may happen that we do not need to do anything 1821 * because the last allocated page can accommodate the change on 1822 * its own. 1823 */ 1824 oldsize = node->tn_size; 1825 oldpages = OFF_TO_IDX(oldsize + PAGE_MASK); 1826 MPASS(oldpages == uobj->size); 1827 newpages = OFF_TO_IDX(newsize + PAGE_MASK); 1828 1829 if (__predict_true(newpages == oldpages && newsize >= oldsize)) { 1830 node->tn_size = newsize; 1831 return (0); 1832 } 1833 1834 VM_OBJECT_WLOCK(uobj); 1835 if (newsize < oldsize) { 1836 /* 1837 * Zero the truncated part of the last page. 1838 */ 1839 base = newsize & PAGE_MASK; 1840 if (base != 0) { 1841 idx = OFF_TO_IDX(newsize); 1842 error = tmpfs_partial_page_invalidate(uobj, idx, base, 1843 PAGE_SIZE, ignerr); 1844 if (error != 0) { 1845 VM_OBJECT_WUNLOCK(uobj); 1846 return (error); 1847 } 1848 } 1849 1850 /* 1851 * Release any swap space and free any whole pages. 1852 */ 1853 if (newpages < oldpages) 1854 vm_object_page_remove(uobj, newpages, 0, 0); 1855 } 1856 uobj->size = newpages; 1857 VM_OBJECT_WUNLOCK(uobj); 1858 1859 node->tn_size = newsize; 1860 return (0); 1861 } 1862 1863 /* 1864 * Punch hole in the aobj associated with the regular file pointed to by 'vp'. 1865 * Requests completely beyond the end-of-file are converted to no-op. 1866 * 1867 * Returns 0 on success or error code from tmpfs_partial_page_invalidate() on 1868 * failure. 1869 */ 1870 int 1871 tmpfs_reg_punch_hole(struct vnode *vp, off_t *offset, off_t *length) 1872 { 1873 struct tmpfs_node *node; 1874 vm_object_t object; 1875 vm_pindex_t pistart, pi, piend; 1876 int startofs, endofs, end; 1877 off_t off, len; 1878 int error; 1879 1880 KASSERT(*length <= OFF_MAX - *offset, ("%s: offset + length overflows", 1881 __func__)); 1882 node = VP_TO_TMPFS_NODE(vp); 1883 KASSERT(node->tn_type == VREG, ("%s: node is not regular file", 1884 __func__)); 1885 object = node->tn_reg.tn_aobj; 1886 off = *offset; 1887 len = omin(node->tn_size - off, *length); 1888 startofs = off & PAGE_MASK; 1889 endofs = (off + len) & PAGE_MASK; 1890 pistart = OFF_TO_IDX(off); 1891 piend = OFF_TO_IDX(off + len); 1892 pi = OFF_TO_IDX((vm_ooffset_t)off + PAGE_MASK); 1893 error = 0; 1894 1895 /* Handle the case when offset is on or beyond file size. */ 1896 if (len <= 0) { 1897 *length = 0; 1898 return (0); 1899 } 1900 1901 VM_OBJECT_WLOCK(object); 1902 1903 /* 1904 * If there is a partial page at the beginning of the hole-punching 1905 * request, fill the partial page with zeroes. 1906 */ 1907 if (startofs != 0) { 1908 end = pistart != piend ? PAGE_SIZE : endofs; 1909 error = tmpfs_partial_page_invalidate(object, pistart, startofs, 1910 end, FALSE); 1911 if (error != 0) 1912 goto out; 1913 off += end - startofs; 1914 len -= end - startofs; 1915 } 1916 1917 /* 1918 * Toss away the full pages in the affected area. 1919 */ 1920 if (pi < piend) { 1921 vm_object_page_remove(object, pi, piend, 0); 1922 off += IDX_TO_OFF(piend - pi); 1923 len -= IDX_TO_OFF(piend - pi); 1924 } 1925 1926 /* 1927 * If there is a partial page at the end of the hole-punching request, 1928 * fill the partial page with zeroes. 1929 */ 1930 if (endofs != 0 && pistart != piend) { 1931 error = tmpfs_partial_page_invalidate(object, piend, 0, endofs, 1932 FALSE); 1933 if (error != 0) 1934 goto out; 1935 off += endofs; 1936 len -= endofs; 1937 } 1938 1939 out: 1940 VM_OBJECT_WUNLOCK(object); 1941 *offset = off; 1942 *length = len; 1943 return (error); 1944 } 1945 1946 void 1947 tmpfs_check_mtime(struct vnode *vp) 1948 { 1949 struct tmpfs_node *node; 1950 struct vm_object *obj; 1951 1952 ASSERT_VOP_ELOCKED(vp, "check_mtime"); 1953 if (vp->v_type != VREG) 1954 return; 1955 obj = vp->v_object; 1956 KASSERT(obj->type == tmpfs_pager_type && 1957 (obj->flags & (OBJ_SWAP | OBJ_TMPFS)) == 1958 (OBJ_SWAP | OBJ_TMPFS), ("non-tmpfs obj")); 1959 /* unlocked read */ 1960 if (obj->generation != obj->cleangeneration) { 1961 VM_OBJECT_WLOCK(obj); 1962 if (obj->generation != obj->cleangeneration) { 1963 obj->cleangeneration = obj->generation; 1964 node = VP_TO_TMPFS_NODE(vp); 1965 node->tn_status |= TMPFS_NODE_MODIFIED | 1966 TMPFS_NODE_CHANGED; 1967 } 1968 VM_OBJECT_WUNLOCK(obj); 1969 } 1970 } 1971 1972 /* 1973 * Change flags of the given vnode. 1974 * Caller should execute tmpfs_update on vp after a successful execution. 1975 * The vnode must be locked on entry and remain locked on exit. 1976 */ 1977 int 1978 tmpfs_chflags(struct vnode *vp, u_long flags, struct ucred *cred, 1979 struct thread *td) 1980 { 1981 int error; 1982 struct tmpfs_node *node; 1983 1984 ASSERT_VOP_ELOCKED(vp, "chflags"); 1985 1986 node = VP_TO_TMPFS_NODE(vp); 1987 1988 if ((flags & ~(SF_APPEND | SF_ARCHIVED | SF_IMMUTABLE | SF_NOUNLINK | 1989 UF_APPEND | UF_ARCHIVE | UF_HIDDEN | UF_IMMUTABLE | UF_NODUMP | 1990 UF_NOUNLINK | UF_OFFLINE | UF_OPAQUE | UF_READONLY | UF_REPARSE | 1991 UF_SPARSE | UF_SYSTEM)) != 0) 1992 return (EOPNOTSUPP); 1993 1994 /* Disallow this operation if the file system is mounted read-only. */ 1995 if (vp->v_mount->mnt_flag & MNT_RDONLY) 1996 return (EROFS); 1997 1998 /* 1999 * Callers may only modify the file flags on objects they 2000 * have VADMIN rights for. 2001 */ 2002 if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) 2003 return (error); 2004 /* 2005 * Unprivileged processes are not permitted to unset system 2006 * flags, or modify flags if any system flags are set. 2007 */ 2008 if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS)) { 2009 if (node->tn_flags & 2010 (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) { 2011 error = securelevel_gt(cred, 0); 2012 if (error) 2013 return (error); 2014 } 2015 } else { 2016 if (node->tn_flags & 2017 (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) || 2018 ((flags ^ node->tn_flags) & SF_SETTABLE)) 2019 return (EPERM); 2020 } 2021 node->tn_flags = flags; 2022 node->tn_status |= TMPFS_NODE_CHANGED; 2023 2024 ASSERT_VOP_ELOCKED(vp, "chflags2"); 2025 2026 return (0); 2027 } 2028 2029 /* 2030 * Change access mode on the given vnode. 2031 * Caller should execute tmpfs_update on vp after a successful execution. 2032 * The vnode must be locked on entry and remain locked on exit. 2033 */ 2034 int 2035 tmpfs_chmod(struct vnode *vp, mode_t mode, struct ucred *cred, 2036 struct thread *td) 2037 { 2038 int error; 2039 struct tmpfs_node *node; 2040 mode_t newmode; 2041 2042 ASSERT_VOP_ELOCKED(vp, "chmod"); 2043 ASSERT_VOP_IN_SEQC(vp); 2044 2045 node = VP_TO_TMPFS_NODE(vp); 2046 2047 /* Disallow this operation if the file system is mounted read-only. */ 2048 if (vp->v_mount->mnt_flag & MNT_RDONLY) 2049 return (EROFS); 2050 2051 /* Immutable or append-only files cannot be modified, either. */ 2052 if (node->tn_flags & (IMMUTABLE | APPEND)) 2053 return (EPERM); 2054 2055 /* 2056 * To modify the permissions on a file, must possess VADMIN 2057 * for that file. 2058 */ 2059 if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) 2060 return (error); 2061 2062 /* 2063 * Privileged processes may set the sticky bit on non-directories, 2064 * as well as set the setgid bit on a file with a group that the 2065 * process is not a member of. 2066 */ 2067 if (vp->v_type != VDIR && (mode & S_ISTXT)) { 2068 if (priv_check_cred(cred, PRIV_VFS_STICKYFILE)) 2069 return (EFTYPE); 2070 } 2071 if (!groupmember(node->tn_gid, cred) && (mode & S_ISGID)) { 2072 error = priv_check_cred(cred, PRIV_VFS_SETGID); 2073 if (error) 2074 return (error); 2075 } 2076 2077 newmode = node->tn_mode & ~ALLPERMS; 2078 newmode |= mode & ALLPERMS; 2079 atomic_store_short(&node->tn_mode, newmode); 2080 2081 node->tn_status |= TMPFS_NODE_CHANGED; 2082 2083 ASSERT_VOP_ELOCKED(vp, "chmod2"); 2084 2085 return (0); 2086 } 2087 2088 /* 2089 * Change ownership of the given vnode. At least one of uid or gid must 2090 * be different than VNOVAL. If one is set to that value, the attribute 2091 * is unchanged. 2092 * Caller should execute tmpfs_update on vp after a successful execution. 2093 * The vnode must be locked on entry and remain locked on exit. 2094 */ 2095 int 2096 tmpfs_chown(struct vnode *vp, uid_t uid, gid_t gid, struct ucred *cred, 2097 struct thread *td) 2098 { 2099 int error; 2100 struct tmpfs_node *node; 2101 uid_t ouid; 2102 gid_t ogid; 2103 mode_t newmode; 2104 2105 ASSERT_VOP_ELOCKED(vp, "chown"); 2106 ASSERT_VOP_IN_SEQC(vp); 2107 2108 node = VP_TO_TMPFS_NODE(vp); 2109 2110 /* Assign default values if they are unknown. */ 2111 MPASS(uid != VNOVAL || gid != VNOVAL); 2112 if (uid == VNOVAL) 2113 uid = node->tn_uid; 2114 if (gid == VNOVAL) 2115 gid = node->tn_gid; 2116 MPASS(uid != VNOVAL && gid != VNOVAL); 2117 2118 /* Disallow this operation if the file system is mounted read-only. */ 2119 if (vp->v_mount->mnt_flag & MNT_RDONLY) 2120 return (EROFS); 2121 2122 /* Immutable or append-only files cannot be modified, either. */ 2123 if (node->tn_flags & (IMMUTABLE | APPEND)) 2124 return (EPERM); 2125 2126 /* 2127 * To modify the ownership of a file, must possess VADMIN for that 2128 * file. 2129 */ 2130 if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) 2131 return (error); 2132 2133 /* 2134 * To change the owner of a file, or change the group of a file to a 2135 * group of which we are not a member, the caller must have 2136 * privilege. 2137 */ 2138 if ((uid != node->tn_uid || 2139 (gid != node->tn_gid && !groupmember(gid, cred))) && 2140 (error = priv_check_cred(cred, PRIV_VFS_CHOWN))) 2141 return (error); 2142 2143 ogid = node->tn_gid; 2144 ouid = node->tn_uid; 2145 2146 node->tn_uid = uid; 2147 node->tn_gid = gid; 2148 2149 node->tn_status |= TMPFS_NODE_CHANGED; 2150 2151 if ((node->tn_mode & (S_ISUID | S_ISGID)) != 0 && 2152 (ouid != uid || ogid != gid)) { 2153 if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID)) { 2154 newmode = node->tn_mode & ~(S_ISUID | S_ISGID); 2155 atomic_store_short(&node->tn_mode, newmode); 2156 } 2157 } 2158 2159 ASSERT_VOP_ELOCKED(vp, "chown2"); 2160 2161 return (0); 2162 } 2163 2164 /* 2165 * Change size of the given vnode. 2166 * Caller should execute tmpfs_update on vp after a successful execution. 2167 * The vnode must be locked on entry and remain locked on exit. 2168 */ 2169 int 2170 tmpfs_chsize(struct vnode *vp, u_quad_t size, struct ucred *cred, 2171 struct thread *td) 2172 { 2173 int error; 2174 struct tmpfs_node *node; 2175 2176 ASSERT_VOP_ELOCKED(vp, "chsize"); 2177 2178 node = VP_TO_TMPFS_NODE(vp); 2179 2180 /* Decide whether this is a valid operation based on the file type. */ 2181 error = 0; 2182 switch (vp->v_type) { 2183 case VDIR: 2184 return (EISDIR); 2185 2186 case VREG: 2187 if (vp->v_mount->mnt_flag & MNT_RDONLY) 2188 return (EROFS); 2189 break; 2190 2191 case VBLK: 2192 /* FALLTHROUGH */ 2193 case VCHR: 2194 /* FALLTHROUGH */ 2195 case VFIFO: 2196 /* 2197 * Allow modifications of special files even if in the file 2198 * system is mounted read-only (we are not modifying the 2199 * files themselves, but the objects they represent). 2200 */ 2201 return (0); 2202 2203 default: 2204 /* Anything else is unsupported. */ 2205 return (EOPNOTSUPP); 2206 } 2207 2208 /* Immutable or append-only files cannot be modified, either. */ 2209 if (node->tn_flags & (IMMUTABLE | APPEND)) 2210 return (EPERM); 2211 2212 error = vn_rlimit_trunc(size, td); 2213 if (error != 0) 2214 return (error); 2215 2216 error = tmpfs_truncate(vp, size); 2217 /* 2218 * tmpfs_truncate will raise the NOTE_EXTEND and NOTE_ATTRIB kevents 2219 * for us, as will update tn_status; no need to do that here. 2220 */ 2221 2222 ASSERT_VOP_ELOCKED(vp, "chsize2"); 2223 2224 return (error); 2225 } 2226 2227 /* 2228 * Change access and modification times of the given vnode. 2229 * Caller should execute tmpfs_update on vp after a successful execution. 2230 * The vnode must be locked on entry and remain locked on exit. 2231 */ 2232 int 2233 tmpfs_chtimes(struct vnode *vp, struct vattr *vap, 2234 struct ucred *cred, struct thread *td) 2235 { 2236 int error; 2237 struct tmpfs_node *node; 2238 2239 ASSERT_VOP_ELOCKED(vp, "chtimes"); 2240 2241 node = VP_TO_TMPFS_NODE(vp); 2242 2243 /* Disallow this operation if the file system is mounted read-only. */ 2244 if (vp->v_mount->mnt_flag & MNT_RDONLY) 2245 return (EROFS); 2246 2247 /* Immutable or append-only files cannot be modified, either. */ 2248 if (node->tn_flags & (IMMUTABLE | APPEND)) 2249 return (EPERM); 2250 2251 error = vn_utimes_perm(vp, vap, cred, td); 2252 if (error != 0) 2253 return (error); 2254 2255 if (vap->va_atime.tv_sec != VNOVAL) 2256 node->tn_accessed = true; 2257 if (vap->va_mtime.tv_sec != VNOVAL) 2258 node->tn_status |= TMPFS_NODE_MODIFIED; 2259 if (vap->va_birthtime.tv_sec != VNOVAL) 2260 node->tn_status |= TMPFS_NODE_MODIFIED; 2261 tmpfs_itimes(vp, &vap->va_atime, &vap->va_mtime); 2262 if (vap->va_birthtime.tv_sec != VNOVAL) 2263 node->tn_birthtime = vap->va_birthtime; 2264 ASSERT_VOP_ELOCKED(vp, "chtimes2"); 2265 2266 return (0); 2267 } 2268 2269 void 2270 tmpfs_set_status(struct tmpfs_mount *tm, struct tmpfs_node *node, int status) 2271 { 2272 2273 if ((node->tn_status & status) == status || tm->tm_ronly) 2274 return; 2275 TMPFS_NODE_LOCK(node); 2276 node->tn_status |= status; 2277 TMPFS_NODE_UNLOCK(node); 2278 } 2279 2280 void 2281 tmpfs_set_accessed(struct tmpfs_mount *tm, struct tmpfs_node *node) 2282 { 2283 if (node->tn_accessed || tm->tm_ronly) 2284 return; 2285 atomic_store_8(&node->tn_accessed, true); 2286 } 2287 2288 /* Sync timestamps */ 2289 void 2290 tmpfs_itimes(struct vnode *vp, const struct timespec *acc, 2291 const struct timespec *mod) 2292 { 2293 struct tmpfs_node *node; 2294 struct timespec now; 2295 2296 ASSERT_VOP_LOCKED(vp, "tmpfs_itimes"); 2297 node = VP_TO_TMPFS_NODE(vp); 2298 2299 if (!node->tn_accessed && 2300 (node->tn_status & (TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED)) == 0) 2301 return; 2302 2303 vfs_timestamp(&now); 2304 TMPFS_NODE_LOCK(node); 2305 if (node->tn_accessed) { 2306 if (acc == NULL) 2307 acc = &now; 2308 node->tn_atime = *acc; 2309 } 2310 if (node->tn_status & TMPFS_NODE_MODIFIED) { 2311 if (mod == NULL) 2312 mod = &now; 2313 node->tn_mtime = *mod; 2314 } 2315 if (node->tn_status & TMPFS_NODE_CHANGED) 2316 node->tn_ctime = now; 2317 node->tn_status &= ~(TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED); 2318 node->tn_accessed = false; 2319 TMPFS_NODE_UNLOCK(node); 2320 2321 /* XXX: FIX? The entropy here is desirable, but the harvesting may be expensive */ 2322 random_harvest_queue(node, sizeof(*node), RANDOM_FS_ATIME); 2323 } 2324 2325 int 2326 tmpfs_truncate(struct vnode *vp, off_t length) 2327 { 2328 int error; 2329 struct tmpfs_node *node; 2330 2331 node = VP_TO_TMPFS_NODE(vp); 2332 2333 if (length < 0) { 2334 error = EINVAL; 2335 goto out; 2336 } 2337 2338 if (node->tn_size == length) { 2339 error = 0; 2340 goto out; 2341 } 2342 2343 if (length > VFS_TO_TMPFS(vp->v_mount)->tm_maxfilesize) 2344 return (EFBIG); 2345 2346 error = tmpfs_reg_resize(vp, length, FALSE); 2347 if (error == 0) 2348 node->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED; 2349 2350 out: 2351 tmpfs_update(vp); 2352 2353 return (error); 2354 } 2355 2356 static __inline int 2357 tmpfs_dirtree_cmp(struct tmpfs_dirent *a, struct tmpfs_dirent *b) 2358 { 2359 if (a->td_hash > b->td_hash) 2360 return (1); 2361 else if (a->td_hash < b->td_hash) 2362 return (-1); 2363 return (0); 2364 } 2365 2366 RB_GENERATE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp); 2367