1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 24 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 25 */ 26 27 #include <assert.h> 28 #include <fcntl.h> 29 #include <libgen.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <limits.h> 35 #include <libzutil.h> 36 #include <sys/crypto/icp.h> 37 #include <sys/processor.h> 38 #include <sys/rrwlock.h> 39 #include <sys/spa.h> 40 #include <sys/stat.h> 41 #include <sys/systeminfo.h> 42 #include <sys/time.h> 43 #include <sys/utsname.h> 44 #include <sys/zfs_context.h> 45 #include <sys/zfs_onexit.h> 46 #include <sys/zfs_vfsops.h> 47 #include <sys/zstd/zstd.h> 48 #include <sys/zvol.h> 49 #include <zfs_fletcher.h> 50 #include <zlib.h> 51 52 /* 53 * Emulation of kernel services in userland. 54 */ 55 56 uint64_t physmem; 57 uint32_t hostid; 58 struct utsname hw_utsname; 59 60 /* If set, all blocks read will be copied to the specified directory. */ 61 char *vn_dumpdir = NULL; 62 63 /* this only exists to have its address taken */ 64 struct proc p0; 65 66 /* 67 * ========================================================================= 68 * threads 69 * ========================================================================= 70 * 71 * TS_STACK_MIN is dictated by the minimum allowed pthread stack size. While 72 * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for 73 * the expected stack depth while small enough to avoid exhausting address 74 * space with high thread counts. 75 */ 76 #define TS_STACK_MIN MAX(PTHREAD_STACK_MIN, 32768) 77 #define TS_STACK_MAX (256 * 1024) 78 79 struct zk_thread_wrapper { 80 void (*func)(void *); 81 void *arg; 82 }; 83 84 static void * 85 zk_thread_wrapper(void *arg) 86 { 87 struct zk_thread_wrapper ztw; 88 memcpy(&ztw, arg, sizeof (ztw)); 89 free(arg); 90 ztw.func(ztw.arg); 91 return (NULL); 92 } 93 94 kthread_t * 95 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state) 96 { 97 pthread_attr_t attr; 98 pthread_t tid; 99 char *stkstr; 100 struct zk_thread_wrapper *ztw; 101 int detachstate = PTHREAD_CREATE_DETACHED; 102 103 VERIFY0(pthread_attr_init(&attr)); 104 105 if (state & TS_JOINABLE) 106 detachstate = PTHREAD_CREATE_JOINABLE; 107 108 VERIFY0(pthread_attr_setdetachstate(&attr, detachstate)); 109 110 /* 111 * We allow the default stack size in user space to be specified by 112 * setting the ZFS_STACK_SIZE environment variable. This allows us 113 * the convenience of observing and debugging stack overruns in 114 * user space. Explicitly specified stack sizes will be honored. 115 * The usage of ZFS_STACK_SIZE is discussed further in the 116 * ENVIRONMENT VARIABLES sections of the ztest(1) man page. 117 */ 118 if (stksize == 0) { 119 stkstr = getenv("ZFS_STACK_SIZE"); 120 121 if (stkstr == NULL) 122 stksize = TS_STACK_MAX; 123 else 124 stksize = MAX(atoi(stkstr), TS_STACK_MIN); 125 } 126 127 VERIFY3S(stksize, >, 0); 128 stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE); 129 130 /* 131 * If this ever fails, it may be because the stack size is not a 132 * multiple of system page size. 133 */ 134 VERIFY0(pthread_attr_setstacksize(&attr, stksize)); 135 VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE)); 136 137 VERIFY(ztw = malloc(sizeof (*ztw))); 138 ztw->func = func; 139 ztw->arg = arg; 140 VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw)); 141 VERIFY0(pthread_attr_destroy(&attr)); 142 143 return ((void *)(uintptr_t)tid); 144 } 145 146 /* 147 * ========================================================================= 148 * kstats 149 * ========================================================================= 150 */ 151 kstat_t * 152 kstat_create(const char *module, int instance, const char *name, 153 const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag) 154 { 155 (void) module, (void) instance, (void) name, (void) class, (void) type, 156 (void) ndata, (void) ks_flag; 157 return (NULL); 158 } 159 160 void 161 kstat_install(kstat_t *ksp) 162 { 163 (void) ksp; 164 } 165 166 void 167 kstat_delete(kstat_t *ksp) 168 { 169 (void) ksp; 170 } 171 172 void 173 kstat_set_raw_ops(kstat_t *ksp, 174 int (*headers)(char *buf, size_t size), 175 int (*data)(char *buf, size_t size, void *data), 176 void *(*addr)(kstat_t *ksp, loff_t index)) 177 { 178 (void) ksp, (void) headers, (void) data, (void) addr; 179 } 180 181 /* 182 * ========================================================================= 183 * mutexes 184 * ========================================================================= 185 */ 186 187 void 188 mutex_init(kmutex_t *mp, char *name, int type, void *cookie) 189 { 190 (void) name, (void) type, (void) cookie; 191 VERIFY0(pthread_mutex_init(&mp->m_lock, NULL)); 192 memset(&mp->m_owner, 0, sizeof (pthread_t)); 193 } 194 195 void 196 mutex_destroy(kmutex_t *mp) 197 { 198 VERIFY0(pthread_mutex_destroy(&mp->m_lock)); 199 } 200 201 void 202 mutex_enter(kmutex_t *mp) 203 { 204 VERIFY0(pthread_mutex_lock(&mp->m_lock)); 205 mp->m_owner = pthread_self(); 206 } 207 208 int 209 mutex_enter_check_return(kmutex_t *mp) 210 { 211 int error = pthread_mutex_lock(&mp->m_lock); 212 if (error == 0) 213 mp->m_owner = pthread_self(); 214 return (error); 215 } 216 217 int 218 mutex_tryenter(kmutex_t *mp) 219 { 220 int error = pthread_mutex_trylock(&mp->m_lock); 221 if (error == 0) { 222 mp->m_owner = pthread_self(); 223 return (1); 224 } else { 225 VERIFY3S(error, ==, EBUSY); 226 return (0); 227 } 228 } 229 230 void 231 mutex_exit(kmutex_t *mp) 232 { 233 memset(&mp->m_owner, 0, sizeof (pthread_t)); 234 VERIFY0(pthread_mutex_unlock(&mp->m_lock)); 235 } 236 237 /* 238 * ========================================================================= 239 * rwlocks 240 * ========================================================================= 241 */ 242 243 void 244 rw_init(krwlock_t *rwlp, char *name, int type, void *arg) 245 { 246 (void) name, (void) type, (void) arg; 247 VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL)); 248 rwlp->rw_readers = 0; 249 rwlp->rw_owner = 0; 250 } 251 252 void 253 rw_destroy(krwlock_t *rwlp) 254 { 255 VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock)); 256 } 257 258 void 259 rw_enter(krwlock_t *rwlp, krw_t rw) 260 { 261 if (rw == RW_READER) { 262 VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock)); 263 atomic_inc_uint(&rwlp->rw_readers); 264 } else { 265 VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock)); 266 rwlp->rw_owner = pthread_self(); 267 } 268 } 269 270 void 271 rw_exit(krwlock_t *rwlp) 272 { 273 if (RW_READ_HELD(rwlp)) 274 atomic_dec_uint(&rwlp->rw_readers); 275 else 276 rwlp->rw_owner = 0; 277 278 VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock)); 279 } 280 281 int 282 rw_tryenter(krwlock_t *rwlp, krw_t rw) 283 { 284 int error; 285 286 if (rw == RW_READER) 287 error = pthread_rwlock_tryrdlock(&rwlp->rw_lock); 288 else 289 error = pthread_rwlock_trywrlock(&rwlp->rw_lock); 290 291 if (error == 0) { 292 if (rw == RW_READER) 293 atomic_inc_uint(&rwlp->rw_readers); 294 else 295 rwlp->rw_owner = pthread_self(); 296 297 return (1); 298 } 299 300 VERIFY3S(error, ==, EBUSY); 301 302 return (0); 303 } 304 305 uint32_t 306 zone_get_hostid(void *zonep) 307 { 308 /* 309 * We're emulating the system's hostid in userland. 310 */ 311 (void) zonep; 312 return (hostid); 313 } 314 315 int 316 rw_tryupgrade(krwlock_t *rwlp) 317 { 318 (void) rwlp; 319 return (0); 320 } 321 322 /* 323 * ========================================================================= 324 * condition variables 325 * ========================================================================= 326 */ 327 328 void 329 cv_init(kcondvar_t *cv, char *name, int type, void *arg) 330 { 331 (void) name, (void) type, (void) arg; 332 VERIFY0(pthread_cond_init(cv, NULL)); 333 } 334 335 void 336 cv_destroy(kcondvar_t *cv) 337 { 338 VERIFY0(pthread_cond_destroy(cv)); 339 } 340 341 void 342 cv_wait(kcondvar_t *cv, kmutex_t *mp) 343 { 344 memset(&mp->m_owner, 0, sizeof (pthread_t)); 345 VERIFY0(pthread_cond_wait(cv, &mp->m_lock)); 346 mp->m_owner = pthread_self(); 347 } 348 349 int 350 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp) 351 { 352 cv_wait(cv, mp); 353 return (1); 354 } 355 356 int 357 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime) 358 { 359 int error; 360 struct timeval tv; 361 struct timespec ts; 362 clock_t delta; 363 364 delta = abstime - ddi_get_lbolt(); 365 if (delta <= 0) 366 return (-1); 367 368 VERIFY(gettimeofday(&tv, NULL) == 0); 369 370 ts.tv_sec = tv.tv_sec + delta / hz; 371 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz); 372 if (ts.tv_nsec >= NANOSEC) { 373 ts.tv_sec++; 374 ts.tv_nsec -= NANOSEC; 375 } 376 377 memset(&mp->m_owner, 0, sizeof (pthread_t)); 378 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 379 mp->m_owner = pthread_self(); 380 381 if (error == ETIMEDOUT) 382 return (-1); 383 384 VERIFY0(error); 385 386 return (1); 387 } 388 389 int 390 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res, 391 int flag) 392 { 393 (void) res; 394 int error; 395 struct timeval tv; 396 struct timespec ts; 397 hrtime_t delta; 398 399 ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE); 400 401 delta = tim; 402 if (flag & CALLOUT_FLAG_ABSOLUTE) 403 delta -= gethrtime(); 404 405 if (delta <= 0) 406 return (-1); 407 408 VERIFY0(gettimeofday(&tv, NULL)); 409 410 ts.tv_sec = tv.tv_sec + delta / NANOSEC; 411 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC); 412 if (ts.tv_nsec >= NANOSEC) { 413 ts.tv_sec++; 414 ts.tv_nsec -= NANOSEC; 415 } 416 417 memset(&mp->m_owner, 0, sizeof (pthread_t)); 418 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 419 mp->m_owner = pthread_self(); 420 421 if (error == ETIMEDOUT) 422 return (-1); 423 424 VERIFY0(error); 425 426 return (1); 427 } 428 429 void 430 cv_signal(kcondvar_t *cv) 431 { 432 VERIFY0(pthread_cond_signal(cv)); 433 } 434 435 void 436 cv_broadcast(kcondvar_t *cv) 437 { 438 VERIFY0(pthread_cond_broadcast(cv)); 439 } 440 441 /* 442 * ========================================================================= 443 * procfs list 444 * ========================================================================= 445 */ 446 447 void 448 seq_printf(struct seq_file *m, const char *fmt, ...) 449 { 450 (void) m, (void) fmt; 451 } 452 453 void 454 procfs_list_install(const char *module, 455 const char *submodule, 456 const char *name, 457 mode_t mode, 458 procfs_list_t *procfs_list, 459 int (*show)(struct seq_file *f, void *p), 460 int (*show_header)(struct seq_file *f), 461 int (*clear)(procfs_list_t *procfs_list), 462 size_t procfs_list_node_off) 463 { 464 (void) module, (void) submodule, (void) name, (void) mode, (void) show, 465 (void) show_header, (void) clear; 466 mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL); 467 list_create(&procfs_list->pl_list, 468 procfs_list_node_off + sizeof (procfs_list_node_t), 469 procfs_list_node_off + offsetof(procfs_list_node_t, pln_link)); 470 procfs_list->pl_next_id = 1; 471 procfs_list->pl_node_offset = procfs_list_node_off; 472 } 473 474 void 475 procfs_list_uninstall(procfs_list_t *procfs_list) 476 { 477 (void) procfs_list; 478 } 479 480 void 481 procfs_list_destroy(procfs_list_t *procfs_list) 482 { 483 ASSERT(list_is_empty(&procfs_list->pl_list)); 484 list_destroy(&procfs_list->pl_list); 485 mutex_destroy(&procfs_list->pl_lock); 486 } 487 488 #define NODE_ID(procfs_list, obj) \ 489 (((procfs_list_node_t *)(((char *)obj) + \ 490 (procfs_list)->pl_node_offset))->pln_id) 491 492 void 493 procfs_list_add(procfs_list_t *procfs_list, void *p) 494 { 495 ASSERT(MUTEX_HELD(&procfs_list->pl_lock)); 496 NODE_ID(procfs_list, p) = procfs_list->pl_next_id++; 497 list_insert_tail(&procfs_list->pl_list, p); 498 } 499 500 /* 501 * ========================================================================= 502 * vnode operations 503 * ========================================================================= 504 */ 505 506 /* 507 * ========================================================================= 508 * Figure out which debugging statements to print 509 * ========================================================================= 510 */ 511 512 static char *dprintf_string; 513 static int dprintf_print_all; 514 515 int 516 dprintf_find_string(const char *string) 517 { 518 char *tmp_str = dprintf_string; 519 int len = strlen(string); 520 521 /* 522 * Find out if this is a string we want to print. 523 * String format: file1.c,function_name1,file2.c,file3.c 524 */ 525 526 while (tmp_str != NULL) { 527 if (strncmp(tmp_str, string, len) == 0 && 528 (tmp_str[len] == ',' || tmp_str[len] == '\0')) 529 return (1); 530 tmp_str = strchr(tmp_str, ','); 531 if (tmp_str != NULL) 532 tmp_str++; /* Get rid of , */ 533 } 534 return (0); 535 } 536 537 void 538 dprintf_setup(int *argc, char **argv) 539 { 540 int i, j; 541 542 /* 543 * Debugging can be specified two ways: by setting the 544 * environment variable ZFS_DEBUG, or by including a 545 * "debug=..." argument on the command line. The command 546 * line setting overrides the environment variable. 547 */ 548 549 for (i = 1; i < *argc; i++) { 550 int len = strlen("debug="); 551 /* First look for a command line argument */ 552 if (strncmp("debug=", argv[i], len) == 0) { 553 dprintf_string = argv[i] + len; 554 /* Remove from args */ 555 for (j = i; j < *argc; j++) 556 argv[j] = argv[j+1]; 557 argv[j] = NULL; 558 (*argc)--; 559 } 560 } 561 562 if (dprintf_string == NULL) { 563 /* Look for ZFS_DEBUG environment variable */ 564 dprintf_string = getenv("ZFS_DEBUG"); 565 } 566 567 /* 568 * Are we just turning on all debugging? 569 */ 570 if (dprintf_find_string("on")) 571 dprintf_print_all = 1; 572 573 if (dprintf_string != NULL) 574 zfs_flags |= ZFS_DEBUG_DPRINTF; 575 } 576 577 /* 578 * ========================================================================= 579 * debug printfs 580 * ========================================================================= 581 */ 582 void 583 __dprintf(boolean_t dprint, const char *file, const char *func, 584 int line, const char *fmt, ...) 585 { 586 /* Get rid of annoying "../common/" prefix to filename. */ 587 const char *newfile = zfs_basename(file); 588 589 va_list adx; 590 if (dprint) { 591 /* dprintf messages are printed immediately */ 592 593 if (!dprintf_print_all && 594 !dprintf_find_string(newfile) && 595 !dprintf_find_string(func)) 596 return; 597 598 /* Print out just the function name if requested */ 599 flockfile(stdout); 600 if (dprintf_find_string("pid")) 601 (void) printf("%d ", getpid()); 602 if (dprintf_find_string("tid")) 603 (void) printf("%ju ", 604 (uintmax_t)(uintptr_t)pthread_self()); 605 if (dprintf_find_string("cpu")) 606 (void) printf("%u ", getcpuid()); 607 if (dprintf_find_string("time")) 608 (void) printf("%llu ", gethrtime()); 609 if (dprintf_find_string("long")) 610 (void) printf("%s, line %d: ", newfile, line); 611 (void) printf("dprintf: %s: ", func); 612 va_start(adx, fmt); 613 (void) vprintf(fmt, adx); 614 va_end(adx); 615 funlockfile(stdout); 616 } else { 617 /* zfs_dbgmsg is logged for dumping later */ 618 size_t size; 619 char *buf; 620 int i; 621 622 size = 1024; 623 buf = umem_alloc(size, UMEM_NOFAIL); 624 i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func); 625 626 if (i < size) { 627 va_start(adx, fmt); 628 (void) vsnprintf(buf + i, size - i, fmt, adx); 629 va_end(adx); 630 } 631 632 __zfs_dbgmsg(buf); 633 634 umem_free(buf, size); 635 } 636 } 637 638 /* 639 * ========================================================================= 640 * cmn_err() and panic() 641 * ========================================================================= 642 */ 643 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" }; 644 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" }; 645 646 __attribute__((noreturn)) void 647 vpanic(const char *fmt, va_list adx) 648 { 649 (void) fprintf(stderr, "error: "); 650 (void) vfprintf(stderr, fmt, adx); 651 (void) fprintf(stderr, "\n"); 652 653 abort(); /* think of it as a "user-level crash dump" */ 654 } 655 656 __attribute__((noreturn)) void 657 panic(const char *fmt, ...) 658 { 659 va_list adx; 660 661 va_start(adx, fmt); 662 vpanic(fmt, adx); 663 va_end(adx); 664 } 665 666 void 667 vcmn_err(int ce, const char *fmt, va_list adx) 668 { 669 if (ce == CE_PANIC) 670 vpanic(fmt, adx); 671 if (ce != CE_NOTE) { /* suppress noise in userland stress testing */ 672 (void) fprintf(stderr, "%s", ce_prefix[ce]); 673 (void) vfprintf(stderr, fmt, adx); 674 (void) fprintf(stderr, "%s", ce_suffix[ce]); 675 } 676 } 677 678 void 679 cmn_err(int ce, const char *fmt, ...) 680 { 681 va_list adx; 682 683 va_start(adx, fmt); 684 vcmn_err(ce, fmt, adx); 685 va_end(adx); 686 } 687 688 /* 689 * ========================================================================= 690 * misc routines 691 * ========================================================================= 692 */ 693 694 void 695 delay(clock_t ticks) 696 { 697 (void) poll(0, 0, ticks * (1000 / hz)); 698 } 699 700 /* 701 * Find highest one bit set. 702 * Returns bit number + 1 of highest bit that is set, otherwise returns 0. 703 * The __builtin_clzll() function is supported by both GCC and Clang. 704 */ 705 int 706 highbit64(uint64_t i) 707 { 708 if (i == 0) 709 return (0); 710 711 return (NBBY * sizeof (uint64_t) - __builtin_clzll(i)); 712 } 713 714 /* 715 * Find lowest one bit set. 716 * Returns bit number + 1 of lowest bit that is set, otherwise returns 0. 717 * The __builtin_ffsll() function is supported by both GCC and Clang. 718 */ 719 int 720 lowbit64(uint64_t i) 721 { 722 if (i == 0) 723 return (0); 724 725 return (__builtin_ffsll(i)); 726 } 727 728 const char *random_path = "/dev/random"; 729 const char *urandom_path = "/dev/urandom"; 730 static int random_fd = -1, urandom_fd = -1; 731 732 void 733 random_init(void) 734 { 735 VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1); 736 VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1); 737 } 738 739 void 740 random_fini(void) 741 { 742 close(random_fd); 743 close(urandom_fd); 744 745 random_fd = -1; 746 urandom_fd = -1; 747 } 748 749 static int 750 random_get_bytes_common(uint8_t *ptr, size_t len, int fd) 751 { 752 size_t resid = len; 753 ssize_t bytes; 754 755 ASSERT(fd != -1); 756 757 while (resid != 0) { 758 bytes = read(fd, ptr, resid); 759 ASSERT3S(bytes, >=, 0); 760 ptr += bytes; 761 resid -= bytes; 762 } 763 764 return (0); 765 } 766 767 int 768 random_get_bytes(uint8_t *ptr, size_t len) 769 { 770 return (random_get_bytes_common(ptr, len, random_fd)); 771 } 772 773 int 774 random_get_pseudo_bytes(uint8_t *ptr, size_t len) 775 { 776 return (random_get_bytes_common(ptr, len, urandom_fd)); 777 } 778 779 int 780 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result) 781 { 782 errno = 0; 783 *result = strtoull(str, nptr, base); 784 if (*result == 0) 785 return (errno); 786 return (0); 787 } 788 789 utsname_t * 790 utsname(void) 791 { 792 return (&hw_utsname); 793 } 794 795 /* 796 * ========================================================================= 797 * kernel emulation setup & teardown 798 * ========================================================================= 799 */ 800 static int 801 umem_out_of_memory(void) 802 { 803 char errmsg[] = "out of memory -- generating core dump\n"; 804 805 (void) fprintf(stderr, "%s", errmsg); 806 abort(); 807 return (0); 808 } 809 810 void 811 kernel_init(int mode) 812 { 813 extern uint_t rrw_tsd_key; 814 815 umem_nofail_callback(umem_out_of_memory); 816 817 physmem = sysconf(_SC_PHYS_PAGES); 818 819 dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem, 820 (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30)); 821 822 hostid = (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0; 823 824 random_init(); 825 826 VERIFY0(uname(&hw_utsname)); 827 828 system_taskq_init(); 829 icp_init(); 830 831 zstd_init(); 832 833 spa_init((spa_mode_t)mode); 834 835 fletcher_4_init(); 836 837 tsd_create(&rrw_tsd_key, rrw_tsd_destroy); 838 } 839 840 void 841 kernel_fini(void) 842 { 843 fletcher_4_fini(); 844 spa_fini(); 845 846 zstd_fini(); 847 848 icp_fini(); 849 system_taskq_fini(); 850 851 random_fini(); 852 } 853 854 uid_t 855 crgetuid(cred_t *cr) 856 { 857 (void) cr; 858 return (0); 859 } 860 861 uid_t 862 crgetruid(cred_t *cr) 863 { 864 (void) cr; 865 return (0); 866 } 867 868 gid_t 869 crgetgid(cred_t *cr) 870 { 871 (void) cr; 872 return (0); 873 } 874 875 int 876 crgetngroups(cred_t *cr) 877 { 878 (void) cr; 879 return (0); 880 } 881 882 gid_t * 883 crgetgroups(cred_t *cr) 884 { 885 (void) cr; 886 return (NULL); 887 } 888 889 int 890 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr) 891 { 892 (void) name, (void) cr; 893 return (0); 894 } 895 896 int 897 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr) 898 { 899 (void) from, (void) to, (void) cr; 900 return (0); 901 } 902 903 int 904 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr) 905 { 906 (void) name, (void) cr; 907 return (0); 908 } 909 910 int 911 secpolicy_zfs(const cred_t *cr) 912 { 913 (void) cr; 914 return (0); 915 } 916 917 int 918 secpolicy_zfs_proc(const cred_t *cr, proc_t *proc) 919 { 920 (void) cr, (void) proc; 921 return (0); 922 } 923 924 ksiddomain_t * 925 ksid_lookupdomain(const char *dom) 926 { 927 ksiddomain_t *kd; 928 929 kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL); 930 kd->kd_name = spa_strdup(dom); 931 return (kd); 932 } 933 934 void 935 ksiddomain_rele(ksiddomain_t *ksid) 936 { 937 spa_strfree(ksid->kd_name); 938 umem_free(ksid, sizeof (ksiddomain_t)); 939 } 940 941 char * 942 kmem_vasprintf(const char *fmt, va_list adx) 943 { 944 char *buf = NULL; 945 va_list adx_copy; 946 947 va_copy(adx_copy, adx); 948 VERIFY(vasprintf(&buf, fmt, adx_copy) != -1); 949 va_end(adx_copy); 950 951 return (buf); 952 } 953 954 char * 955 kmem_asprintf(const char *fmt, ...) 956 { 957 char *buf = NULL; 958 va_list adx; 959 960 va_start(adx, fmt); 961 VERIFY(vasprintf(&buf, fmt, adx) != -1); 962 va_end(adx); 963 964 return (buf); 965 } 966 967 /* 968 * kmem_scnprintf() will return the number of characters that it would have 969 * printed whenever it is limited by value of the size variable, rather than 970 * the number of characters that it did print. This can cause misbehavior on 971 * subsequent uses of the return value, so we define a safe version that will 972 * return the number of characters actually printed, minus the NULL format 973 * character. Subsequent use of this by the safe string functions is safe 974 * whether it is snprintf(), strlcat() or strlcpy(). 975 */ 976 int 977 kmem_scnprintf(char *restrict str, size_t size, const char *restrict fmt, ...) 978 { 979 int n; 980 va_list ap; 981 982 /* Make the 0 case a no-op so that we do not return -1 */ 983 if (size == 0) 984 return (0); 985 986 va_start(ap, fmt); 987 n = vsnprintf(str, size, fmt, ap); 988 va_end(ap); 989 990 if (n >= size) 991 n = size - 1; 992 993 return (n); 994 } 995 996 zfs_file_t * 997 zfs_onexit_fd_hold(int fd, minor_t *minorp) 998 { 999 (void) fd; 1000 *minorp = 0; 1001 return (NULL); 1002 } 1003 1004 void 1005 zfs_onexit_fd_rele(zfs_file_t *fp) 1006 { 1007 (void) fp; 1008 } 1009 1010 int 1011 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, 1012 uintptr_t *action_handle) 1013 { 1014 (void) minor, (void) func, (void) data, (void) action_handle; 1015 return (0); 1016 } 1017 1018 fstrans_cookie_t 1019 spl_fstrans_mark(void) 1020 { 1021 return ((fstrans_cookie_t)0); 1022 } 1023 1024 void 1025 spl_fstrans_unmark(fstrans_cookie_t cookie) 1026 { 1027 (void) cookie; 1028 } 1029 1030 int 1031 __spl_pf_fstrans_check(void) 1032 { 1033 return (0); 1034 } 1035 1036 int 1037 kmem_cache_reap_active(void) 1038 { 1039 return (0); 1040 } 1041 1042 void 1043 zvol_create_minor(const char *name) 1044 { 1045 (void) name; 1046 } 1047 1048 void 1049 zvol_create_minors_recursive(const char *name) 1050 { 1051 (void) name; 1052 } 1053 1054 void 1055 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 1056 { 1057 (void) spa, (void) name, (void) async; 1058 } 1059 1060 void 1061 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname, 1062 boolean_t async) 1063 { 1064 (void) spa, (void) oldname, (void) newname, (void) async; 1065 } 1066 1067 /* 1068 * Open file 1069 * 1070 * path - fully qualified path to file 1071 * flags - file attributes O_READ / O_WRITE / O_EXCL 1072 * fpp - pointer to return file pointer 1073 * 1074 * Returns 0 on success underlying error on failure. 1075 */ 1076 int 1077 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp) 1078 { 1079 int fd = -1; 1080 int dump_fd = -1; 1081 int err; 1082 int old_umask = 0; 1083 zfs_file_t *fp; 1084 struct stat64 st; 1085 1086 if (!(flags & O_CREAT) && stat64(path, &st) == -1) 1087 return (errno); 1088 1089 if (!(flags & O_CREAT) && S_ISBLK(st.st_mode)) 1090 flags |= O_DIRECT; 1091 1092 if (flags & O_CREAT) 1093 old_umask = umask(0); 1094 1095 fd = open64(path, flags, mode); 1096 if (fd == -1) 1097 return (errno); 1098 1099 if (flags & O_CREAT) 1100 (void) umask(old_umask); 1101 1102 if (vn_dumpdir != NULL) { 1103 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL); 1104 const char *inpath = zfs_basename(path); 1105 1106 (void) snprintf(dumppath, MAXPATHLEN, 1107 "%s/%s", vn_dumpdir, inpath); 1108 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666); 1109 umem_free(dumppath, MAXPATHLEN); 1110 if (dump_fd == -1) { 1111 err = errno; 1112 close(fd); 1113 return (err); 1114 } 1115 } else { 1116 dump_fd = -1; 1117 } 1118 1119 (void) fcntl(fd, F_SETFD, FD_CLOEXEC); 1120 1121 fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL); 1122 fp->f_fd = fd; 1123 fp->f_dump_fd = dump_fd; 1124 *fpp = fp; 1125 1126 return (0); 1127 } 1128 1129 void 1130 zfs_file_close(zfs_file_t *fp) 1131 { 1132 close(fp->f_fd); 1133 if (fp->f_dump_fd != -1) 1134 close(fp->f_dump_fd); 1135 1136 umem_free(fp, sizeof (zfs_file_t)); 1137 } 1138 1139 /* 1140 * Stateful write - use os internal file pointer to determine where to 1141 * write and update on successful completion. 1142 * 1143 * fp - pointer to file (pipe, socket, etc) to write to 1144 * buf - buffer to write 1145 * count - # of bytes to write 1146 * resid - pointer to count of unwritten bytes (if short write) 1147 * 1148 * Returns 0 on success errno on failure. 1149 */ 1150 int 1151 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid) 1152 { 1153 ssize_t rc; 1154 1155 rc = write(fp->f_fd, buf, count); 1156 if (rc < 0) 1157 return (errno); 1158 1159 if (resid) { 1160 *resid = count - rc; 1161 } else if (rc != count) { 1162 return (EIO); 1163 } 1164 1165 return (0); 1166 } 1167 1168 /* 1169 * Stateless write - os internal file pointer is not updated. 1170 * 1171 * fp - pointer to file (pipe, socket, etc) to write to 1172 * buf - buffer to write 1173 * count - # of bytes to write 1174 * off - file offset to write to (only valid for seekable types) 1175 * resid - pointer to count of unwritten bytes 1176 * 1177 * Returns 0 on success errno on failure. 1178 */ 1179 int 1180 zfs_file_pwrite(zfs_file_t *fp, const void *buf, 1181 size_t count, loff_t pos, ssize_t *resid) 1182 { 1183 ssize_t rc, split, done; 1184 int sectors; 1185 1186 /* 1187 * To simulate partial disk writes, we split writes into two 1188 * system calls so that the process can be killed in between. 1189 * This is used by ztest to simulate realistic failure modes. 1190 */ 1191 sectors = count >> SPA_MINBLOCKSHIFT; 1192 split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT; 1193 rc = pwrite64(fp->f_fd, buf, split, pos); 1194 if (rc != -1) { 1195 done = rc; 1196 rc = pwrite64(fp->f_fd, (char *)buf + split, 1197 count - split, pos + split); 1198 } 1199 #ifdef __linux__ 1200 if (rc == -1 && errno == EINVAL) { 1201 /* 1202 * Under Linux, this most likely means an alignment issue 1203 * (memory or disk) due to O_DIRECT, so we abort() in order 1204 * to catch the offender. 1205 */ 1206 abort(); 1207 } 1208 #endif 1209 1210 if (rc < 0) 1211 return (errno); 1212 1213 done += rc; 1214 1215 if (resid) { 1216 *resid = count - done; 1217 } else if (done != count) { 1218 return (EIO); 1219 } 1220 1221 return (0); 1222 } 1223 1224 /* 1225 * Stateful read - use os internal file pointer to determine where to 1226 * read and update on successful completion. 1227 * 1228 * fp - pointer to file (pipe, socket, etc) to read from 1229 * buf - buffer to write 1230 * count - # of bytes to read 1231 * resid - pointer to count of unread bytes (if short read) 1232 * 1233 * Returns 0 on success errno on failure. 1234 */ 1235 int 1236 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid) 1237 { 1238 int rc; 1239 1240 rc = read(fp->f_fd, buf, count); 1241 if (rc < 0) 1242 return (errno); 1243 1244 if (resid) { 1245 *resid = count - rc; 1246 } else if (rc != count) { 1247 return (EIO); 1248 } 1249 1250 return (0); 1251 } 1252 1253 /* 1254 * Stateless read - os internal file pointer is not updated. 1255 * 1256 * fp - pointer to file (pipe, socket, etc) to read from 1257 * buf - buffer to write 1258 * count - # of bytes to write 1259 * off - file offset to read from (only valid for seekable types) 1260 * resid - pointer to count of unwritten bytes (if short write) 1261 * 1262 * Returns 0 on success errno on failure. 1263 */ 1264 int 1265 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off, 1266 ssize_t *resid) 1267 { 1268 ssize_t rc; 1269 1270 rc = pread64(fp->f_fd, buf, count, off); 1271 if (rc < 0) { 1272 #ifdef __linux__ 1273 /* 1274 * Under Linux, this most likely means an alignment issue 1275 * (memory or disk) due to O_DIRECT, so we abort() in order to 1276 * catch the offender. 1277 */ 1278 if (errno == EINVAL) 1279 abort(); 1280 #endif 1281 return (errno); 1282 } 1283 1284 if (fp->f_dump_fd != -1) { 1285 int status; 1286 1287 status = pwrite64(fp->f_dump_fd, buf, rc, off); 1288 ASSERT(status != -1); 1289 } 1290 1291 if (resid) { 1292 *resid = count - rc; 1293 } else if (rc != count) { 1294 return (EIO); 1295 } 1296 1297 return (0); 1298 } 1299 1300 /* 1301 * lseek - set / get file pointer 1302 * 1303 * fp - pointer to file (pipe, socket, etc) to read from 1304 * offp - value to seek to, returns current value plus passed offset 1305 * whence - see man pages for standard lseek whence values 1306 * 1307 * Returns 0 on success errno on failure (ESPIPE for non seekable types) 1308 */ 1309 int 1310 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence) 1311 { 1312 loff_t rc; 1313 1314 rc = lseek(fp->f_fd, *offp, whence); 1315 if (rc < 0) 1316 return (errno); 1317 1318 *offp = rc; 1319 1320 return (0); 1321 } 1322 1323 /* 1324 * Get file attributes 1325 * 1326 * filp - file pointer 1327 * zfattr - pointer to file attr structure 1328 * 1329 * Currently only used for fetching size and file mode 1330 * 1331 * Returns 0 on success or error code of underlying getattr call on failure. 1332 */ 1333 int 1334 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr) 1335 { 1336 struct stat64 st; 1337 1338 if (fstat64_blk(fp->f_fd, &st) == -1) 1339 return (errno); 1340 1341 zfattr->zfa_size = st.st_size; 1342 zfattr->zfa_mode = st.st_mode; 1343 1344 return (0); 1345 } 1346 1347 /* 1348 * Sync file to disk 1349 * 1350 * filp - file pointer 1351 * flags - O_SYNC and or O_DSYNC 1352 * 1353 * Returns 0 on success or error code of underlying sync call on failure. 1354 */ 1355 int 1356 zfs_file_fsync(zfs_file_t *fp, int flags) 1357 { 1358 (void) flags; 1359 1360 if (fsync(fp->f_fd) < 0) 1361 return (errno); 1362 1363 return (0); 1364 } 1365 1366 /* 1367 * fallocate - allocate or free space on disk 1368 * 1369 * fp - file pointer 1370 * mode (non-standard options for hole punching etc) 1371 * offset - offset to start allocating or freeing from 1372 * len - length to free / allocate 1373 * 1374 * OPTIONAL 1375 */ 1376 int 1377 zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len) 1378 { 1379 #ifdef __linux__ 1380 return (fallocate(fp->f_fd, mode, offset, len)); 1381 #else 1382 (void) fp, (void) mode, (void) offset, (void) len; 1383 return (EOPNOTSUPP); 1384 #endif 1385 } 1386 1387 /* 1388 * Request current file pointer offset 1389 * 1390 * fp - pointer to file 1391 * 1392 * Returns current file offset. 1393 */ 1394 loff_t 1395 zfs_file_off(zfs_file_t *fp) 1396 { 1397 return (lseek(fp->f_fd, SEEK_CUR, 0)); 1398 } 1399 1400 /* 1401 * unlink file 1402 * 1403 * path - fully qualified file path 1404 * 1405 * Returns 0 on success. 1406 * 1407 * OPTIONAL 1408 */ 1409 int 1410 zfs_file_unlink(const char *path) 1411 { 1412 return (remove(path)); 1413 } 1414 1415 /* 1416 * Get reference to file pointer 1417 * 1418 * fd - input file descriptor 1419 * 1420 * Returns pointer to file struct or NULL. 1421 * Unsupported in user space. 1422 */ 1423 zfs_file_t * 1424 zfs_file_get(int fd) 1425 { 1426 (void) fd; 1427 abort(); 1428 return (NULL); 1429 } 1430 /* 1431 * Drop reference to file pointer 1432 * 1433 * fp - pointer to file struct 1434 * 1435 * Unsupported in user space. 1436 */ 1437 void 1438 zfs_file_put(zfs_file_t *fp) 1439 { 1440 abort(); 1441 (void) fp; 1442 } 1443 1444 void 1445 zfsvfs_update_fromname(const char *oldname, const char *newname) 1446 { 1447 (void) oldname, (void) newname; 1448 } 1449 1450 void 1451 spa_import_os(spa_t *spa) 1452 { 1453 (void) spa; 1454 } 1455 1456 void 1457 spa_export_os(spa_t *spa) 1458 { 1459 (void) spa; 1460 } 1461 1462 void 1463 spa_activate_os(spa_t *spa) 1464 { 1465 (void) spa; 1466 } 1467 1468 void 1469 spa_deactivate_os(spa_t *spa) 1470 { 1471 (void) spa; 1472 } 1473