xref: /freebsd/sys/contrib/openzfs/lib/libzpool/kernel.c (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
24  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
25  */
26 
27 #include <assert.h>
28 #include <fcntl.h>
29 #include <libgen.h>
30 #include <poll.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <libzutil.h>
35 #include <sys/crypto/icp.h>
36 #include <sys/processor.h>
37 #include <sys/rrwlock.h>
38 #include <sys/spa.h>
39 #include <sys/stat.h>
40 #include <sys/systeminfo.h>
41 #include <sys/time.h>
42 #include <sys/utsname.h>
43 #include <sys/zfs_context.h>
44 #include <sys/zfs_onexit.h>
45 #include <sys/zfs_vfsops.h>
46 #include <sys/zstd/zstd.h>
47 #include <sys/zvol.h>
48 #include <zfs_fletcher.h>
49 #include <zlib.h>
50 
51 /*
52  * Emulation of kernel services in userland.
53  */
54 
55 uint64_t physmem;
56 uint32_t hostid;
57 struct utsname hw_utsname;
58 
59 /* If set, all blocks read will be copied to the specified directory. */
60 char *vn_dumpdir = NULL;
61 
62 /* this only exists to have its address taken */
63 struct proc p0;
64 
65 /*
66  * =========================================================================
67  * threads
68  * =========================================================================
69  *
70  * TS_STACK_MIN is dictated by the minimum allowed pthread stack size.  While
71  * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
72  * the expected stack depth while small enough to avoid exhausting address
73  * space with high thread counts.
74  */
75 #define	TS_STACK_MIN	MAX(PTHREAD_STACK_MIN, 32768)
76 #define	TS_STACK_MAX	(256 * 1024)
77 
78 struct zk_thread_wrapper {
79 	void (*func)(void *);
80 	void *arg;
81 };
82 
83 static void *
84 zk_thread_wrapper(void *arg)
85 {
86 	struct zk_thread_wrapper ztw;
87 	memcpy(&ztw, arg, sizeof (ztw));
88 	free(arg);
89 	ztw.func(ztw.arg);
90 	return (NULL);
91 }
92 
93 kthread_t *
94 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
95 {
96 	pthread_attr_t attr;
97 	pthread_t tid;
98 	char *stkstr;
99 	struct zk_thread_wrapper *ztw;
100 	int detachstate = PTHREAD_CREATE_DETACHED;
101 
102 	VERIFY0(pthread_attr_init(&attr));
103 
104 	if (state & TS_JOINABLE)
105 		detachstate = PTHREAD_CREATE_JOINABLE;
106 
107 	VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
108 
109 	/*
110 	 * We allow the default stack size in user space to be specified by
111 	 * setting the ZFS_STACK_SIZE environment variable.  This allows us
112 	 * the convenience of observing and debugging stack overruns in
113 	 * user space.  Explicitly specified stack sizes will be honored.
114 	 * The usage of ZFS_STACK_SIZE is discussed further in the
115 	 * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
116 	 */
117 	if (stksize == 0) {
118 		stkstr = getenv("ZFS_STACK_SIZE");
119 
120 		if (stkstr == NULL)
121 			stksize = TS_STACK_MAX;
122 		else
123 			stksize = MAX(atoi(stkstr), TS_STACK_MIN);
124 	}
125 
126 	VERIFY3S(stksize, >, 0);
127 	stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
128 
129 	/*
130 	 * If this ever fails, it may be because the stack size is not a
131 	 * multiple of system page size.
132 	 */
133 	VERIFY0(pthread_attr_setstacksize(&attr, stksize));
134 	VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
135 
136 	VERIFY(ztw = malloc(sizeof (*ztw)));
137 	ztw->func = func;
138 	ztw->arg = arg;
139 	VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw));
140 	VERIFY0(pthread_attr_destroy(&attr));
141 
142 	return ((void *)(uintptr_t)tid);
143 }
144 
145 /*
146  * =========================================================================
147  * kstats
148  * =========================================================================
149  */
150 kstat_t *
151 kstat_create(const char *module, int instance, const char *name,
152     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
153 {
154 	(void) module, (void) instance, (void) name, (void) class, (void) type,
155 	    (void) ndata, (void) ks_flag;
156 	return (NULL);
157 }
158 
159 void
160 kstat_install(kstat_t *ksp)
161 {
162 	(void) ksp;
163 }
164 
165 void
166 kstat_delete(kstat_t *ksp)
167 {
168 	(void) ksp;
169 }
170 
171 void
172 kstat_set_raw_ops(kstat_t *ksp,
173     int (*headers)(char *buf, size_t size),
174     int (*data)(char *buf, size_t size, void *data),
175     void *(*addr)(kstat_t *ksp, loff_t index))
176 {
177 	(void) ksp, (void) headers, (void) data, (void) addr;
178 }
179 
180 /*
181  * =========================================================================
182  * mutexes
183  * =========================================================================
184  */
185 
186 void
187 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
188 {
189 	(void) name, (void) type, (void) cookie;
190 	VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
191 	memset(&mp->m_owner, 0, sizeof (pthread_t));
192 }
193 
194 void
195 mutex_destroy(kmutex_t *mp)
196 {
197 	VERIFY0(pthread_mutex_destroy(&mp->m_lock));
198 }
199 
200 void
201 mutex_enter(kmutex_t *mp)
202 {
203 	VERIFY0(pthread_mutex_lock(&mp->m_lock));
204 	mp->m_owner = pthread_self();
205 }
206 
207 int
208 mutex_tryenter(kmutex_t *mp)
209 {
210 	int error = pthread_mutex_trylock(&mp->m_lock);
211 	if (error == 0) {
212 		mp->m_owner = pthread_self();
213 		return (1);
214 	} else {
215 		VERIFY3S(error, ==, EBUSY);
216 		return (0);
217 	}
218 }
219 
220 void
221 mutex_exit(kmutex_t *mp)
222 {
223 	memset(&mp->m_owner, 0, sizeof (pthread_t));
224 	VERIFY0(pthread_mutex_unlock(&mp->m_lock));
225 }
226 
227 /*
228  * =========================================================================
229  * rwlocks
230  * =========================================================================
231  */
232 
233 void
234 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
235 {
236 	(void) name, (void) type, (void) arg;
237 	VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
238 	rwlp->rw_readers = 0;
239 	rwlp->rw_owner = 0;
240 }
241 
242 void
243 rw_destroy(krwlock_t *rwlp)
244 {
245 	VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
246 }
247 
248 void
249 rw_enter(krwlock_t *rwlp, krw_t rw)
250 {
251 	if (rw == RW_READER) {
252 		VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
253 		atomic_inc_uint(&rwlp->rw_readers);
254 	} else {
255 		VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
256 		rwlp->rw_owner = pthread_self();
257 	}
258 }
259 
260 void
261 rw_exit(krwlock_t *rwlp)
262 {
263 	if (RW_READ_HELD(rwlp))
264 		atomic_dec_uint(&rwlp->rw_readers);
265 	else
266 		rwlp->rw_owner = 0;
267 
268 	VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
269 }
270 
271 int
272 rw_tryenter(krwlock_t *rwlp, krw_t rw)
273 {
274 	int error;
275 
276 	if (rw == RW_READER)
277 		error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
278 	else
279 		error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
280 
281 	if (error == 0) {
282 		if (rw == RW_READER)
283 			atomic_inc_uint(&rwlp->rw_readers);
284 		else
285 			rwlp->rw_owner = pthread_self();
286 
287 		return (1);
288 	}
289 
290 	VERIFY3S(error, ==, EBUSY);
291 
292 	return (0);
293 }
294 
295 uint32_t
296 zone_get_hostid(void *zonep)
297 {
298 	/*
299 	 * We're emulating the system's hostid in userland.
300 	 */
301 	(void) zonep;
302 	return (hostid);
303 }
304 
305 int
306 rw_tryupgrade(krwlock_t *rwlp)
307 {
308 	(void) rwlp;
309 	return (0);
310 }
311 
312 /*
313  * =========================================================================
314  * condition variables
315  * =========================================================================
316  */
317 
318 void
319 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
320 {
321 	(void) name, (void) type, (void) arg;
322 	VERIFY0(pthread_cond_init(cv, NULL));
323 }
324 
325 void
326 cv_destroy(kcondvar_t *cv)
327 {
328 	VERIFY0(pthread_cond_destroy(cv));
329 }
330 
331 void
332 cv_wait(kcondvar_t *cv, kmutex_t *mp)
333 {
334 	memset(&mp->m_owner, 0, sizeof (pthread_t));
335 	VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
336 	mp->m_owner = pthread_self();
337 }
338 
339 int
340 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp)
341 {
342 	cv_wait(cv, mp);
343 	return (1);
344 }
345 
346 int
347 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
348 {
349 	int error;
350 	struct timeval tv;
351 	struct timespec ts;
352 	clock_t delta;
353 
354 	delta = abstime - ddi_get_lbolt();
355 	if (delta <= 0)
356 		return (-1);
357 
358 	VERIFY(gettimeofday(&tv, NULL) == 0);
359 
360 	ts.tv_sec = tv.tv_sec + delta / hz;
361 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz);
362 	if (ts.tv_nsec >= NANOSEC) {
363 		ts.tv_sec++;
364 		ts.tv_nsec -= NANOSEC;
365 	}
366 
367 	memset(&mp->m_owner, 0, sizeof (pthread_t));
368 	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
369 	mp->m_owner = pthread_self();
370 
371 	if (error == ETIMEDOUT)
372 		return (-1);
373 
374 	VERIFY0(error);
375 
376 	return (1);
377 }
378 
379 int
380 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
381     int flag)
382 {
383 	(void) res;
384 	int error;
385 	struct timeval tv;
386 	struct timespec ts;
387 	hrtime_t delta;
388 
389 	ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
390 
391 	delta = tim;
392 	if (flag & CALLOUT_FLAG_ABSOLUTE)
393 		delta -= gethrtime();
394 
395 	if (delta <= 0)
396 		return (-1);
397 
398 	VERIFY0(gettimeofday(&tv, NULL));
399 
400 	ts.tv_sec = tv.tv_sec + delta / NANOSEC;
401 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
402 	if (ts.tv_nsec >= NANOSEC) {
403 		ts.tv_sec++;
404 		ts.tv_nsec -= NANOSEC;
405 	}
406 
407 	memset(&mp->m_owner, 0, sizeof (pthread_t));
408 	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
409 	mp->m_owner = pthread_self();
410 
411 	if (error == ETIMEDOUT)
412 		return (-1);
413 
414 	VERIFY0(error);
415 
416 	return (1);
417 }
418 
419 void
420 cv_signal(kcondvar_t *cv)
421 {
422 	VERIFY0(pthread_cond_signal(cv));
423 }
424 
425 void
426 cv_broadcast(kcondvar_t *cv)
427 {
428 	VERIFY0(pthread_cond_broadcast(cv));
429 }
430 
431 /*
432  * =========================================================================
433  * procfs list
434  * =========================================================================
435  */
436 
437 void
438 seq_printf(struct seq_file *m, const char *fmt, ...)
439 {
440 	(void) m, (void) fmt;
441 }
442 
443 void
444 procfs_list_install(const char *module,
445     const char *submodule,
446     const char *name,
447     mode_t mode,
448     procfs_list_t *procfs_list,
449     int (*show)(struct seq_file *f, void *p),
450     int (*show_header)(struct seq_file *f),
451     int (*clear)(procfs_list_t *procfs_list),
452     size_t procfs_list_node_off)
453 {
454 	(void) module, (void) submodule, (void) name, (void) mode, (void) show,
455 	    (void) show_header, (void) clear;
456 	mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL);
457 	list_create(&procfs_list->pl_list,
458 	    procfs_list_node_off + sizeof (procfs_list_node_t),
459 	    procfs_list_node_off + offsetof(procfs_list_node_t, pln_link));
460 	procfs_list->pl_next_id = 1;
461 	procfs_list->pl_node_offset = procfs_list_node_off;
462 }
463 
464 void
465 procfs_list_uninstall(procfs_list_t *procfs_list)
466 {
467 	(void) procfs_list;
468 }
469 
470 void
471 procfs_list_destroy(procfs_list_t *procfs_list)
472 {
473 	ASSERT(list_is_empty(&procfs_list->pl_list));
474 	list_destroy(&procfs_list->pl_list);
475 	mutex_destroy(&procfs_list->pl_lock);
476 }
477 
478 #define	NODE_ID(procfs_list, obj) \
479 		(((procfs_list_node_t *)(((char *)obj) + \
480 		(procfs_list)->pl_node_offset))->pln_id)
481 
482 void
483 procfs_list_add(procfs_list_t *procfs_list, void *p)
484 {
485 	ASSERT(MUTEX_HELD(&procfs_list->pl_lock));
486 	NODE_ID(procfs_list, p) = procfs_list->pl_next_id++;
487 	list_insert_tail(&procfs_list->pl_list, p);
488 }
489 
490 /*
491  * =========================================================================
492  * vnode operations
493  * =========================================================================
494  */
495 
496 /*
497  * =========================================================================
498  * Figure out which debugging statements to print
499  * =========================================================================
500  */
501 
502 static char *dprintf_string;
503 static int dprintf_print_all;
504 
505 int
506 dprintf_find_string(const char *string)
507 {
508 	char *tmp_str = dprintf_string;
509 	int len = strlen(string);
510 
511 	/*
512 	 * Find out if this is a string we want to print.
513 	 * String format: file1.c,function_name1,file2.c,file3.c
514 	 */
515 
516 	while (tmp_str != NULL) {
517 		if (strncmp(tmp_str, string, len) == 0 &&
518 		    (tmp_str[len] == ',' || tmp_str[len] == '\0'))
519 			return (1);
520 		tmp_str = strchr(tmp_str, ',');
521 		if (tmp_str != NULL)
522 			tmp_str++; /* Get rid of , */
523 	}
524 	return (0);
525 }
526 
527 void
528 dprintf_setup(int *argc, char **argv)
529 {
530 	int i, j;
531 
532 	/*
533 	 * Debugging can be specified two ways: by setting the
534 	 * environment variable ZFS_DEBUG, or by including a
535 	 * "debug=..."  argument on the command line.  The command
536 	 * line setting overrides the environment variable.
537 	 */
538 
539 	for (i = 1; i < *argc; i++) {
540 		int len = strlen("debug=");
541 		/* First look for a command line argument */
542 		if (strncmp("debug=", argv[i], len) == 0) {
543 			dprintf_string = argv[i] + len;
544 			/* Remove from args */
545 			for (j = i; j < *argc; j++)
546 				argv[j] = argv[j+1];
547 			argv[j] = NULL;
548 			(*argc)--;
549 		}
550 	}
551 
552 	if (dprintf_string == NULL) {
553 		/* Look for ZFS_DEBUG environment variable */
554 		dprintf_string = getenv("ZFS_DEBUG");
555 	}
556 
557 	/*
558 	 * Are we just turning on all debugging?
559 	 */
560 	if (dprintf_find_string("on"))
561 		dprintf_print_all = 1;
562 
563 	if (dprintf_string != NULL)
564 		zfs_flags |= ZFS_DEBUG_DPRINTF;
565 }
566 
567 /*
568  * =========================================================================
569  * debug printfs
570  * =========================================================================
571  */
572 void
573 __dprintf(boolean_t dprint, const char *file, const char *func,
574     int line, const char *fmt, ...)
575 {
576 	/* Get rid of annoying "../common/" prefix to filename. */
577 	const char *newfile = zfs_basename(file);
578 
579 	va_list adx;
580 	if (dprint) {
581 		/* dprintf messages are printed immediately */
582 
583 		if (!dprintf_print_all &&
584 		    !dprintf_find_string(newfile) &&
585 		    !dprintf_find_string(func))
586 			return;
587 
588 		/* Print out just the function name if requested */
589 		flockfile(stdout);
590 		if (dprintf_find_string("pid"))
591 			(void) printf("%d ", getpid());
592 		if (dprintf_find_string("tid"))
593 			(void) printf("%ju ",
594 			    (uintmax_t)(uintptr_t)pthread_self());
595 		if (dprintf_find_string("cpu"))
596 			(void) printf("%u ", getcpuid());
597 		if (dprintf_find_string("time"))
598 			(void) printf("%llu ", gethrtime());
599 		if (dprintf_find_string("long"))
600 			(void) printf("%s, line %d: ", newfile, line);
601 		(void) printf("dprintf: %s: ", func);
602 		va_start(adx, fmt);
603 		(void) vprintf(fmt, adx);
604 		va_end(adx);
605 		funlockfile(stdout);
606 	} else {
607 		/* zfs_dbgmsg is logged for dumping later */
608 		size_t size;
609 		char *buf;
610 		int i;
611 
612 		size = 1024;
613 		buf = umem_alloc(size, UMEM_NOFAIL);
614 		i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func);
615 
616 		if (i < size) {
617 			va_start(adx, fmt);
618 			(void) vsnprintf(buf + i, size - i, fmt, adx);
619 			va_end(adx);
620 		}
621 
622 		__zfs_dbgmsg(buf);
623 
624 		umem_free(buf, size);
625 	}
626 }
627 
628 /*
629  * =========================================================================
630  * cmn_err() and panic()
631  * =========================================================================
632  */
633 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
634 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
635 
636 __attribute__((noreturn)) void
637 vpanic(const char *fmt, va_list adx)
638 {
639 	(void) fprintf(stderr, "error: ");
640 	(void) vfprintf(stderr, fmt, adx);
641 	(void) fprintf(stderr, "\n");
642 
643 	abort();	/* think of it as a "user-level crash dump" */
644 }
645 
646 __attribute__((noreturn)) void
647 panic(const char *fmt, ...)
648 {
649 	va_list adx;
650 
651 	va_start(adx, fmt);
652 	vpanic(fmt, adx);
653 	va_end(adx);
654 }
655 
656 void
657 vcmn_err(int ce, const char *fmt, va_list adx)
658 {
659 	if (ce == CE_PANIC)
660 		vpanic(fmt, adx);
661 	if (ce != CE_NOTE) {	/* suppress noise in userland stress testing */
662 		(void) fprintf(stderr, "%s", ce_prefix[ce]);
663 		(void) vfprintf(stderr, fmt, adx);
664 		(void) fprintf(stderr, "%s", ce_suffix[ce]);
665 	}
666 }
667 
668 void
669 cmn_err(int ce, const char *fmt, ...)
670 {
671 	va_list adx;
672 
673 	va_start(adx, fmt);
674 	vcmn_err(ce, fmt, adx);
675 	va_end(adx);
676 }
677 
678 /*
679  * =========================================================================
680  * misc routines
681  * =========================================================================
682  */
683 
684 void
685 delay(clock_t ticks)
686 {
687 	(void) poll(0, 0, ticks * (1000 / hz));
688 }
689 
690 /*
691  * Find highest one bit set.
692  * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
693  * The __builtin_clzll() function is supported by both GCC and Clang.
694  */
695 int
696 highbit64(uint64_t i)
697 {
698 	if (i == 0)
699 	return (0);
700 
701 	return (NBBY * sizeof (uint64_t) - __builtin_clzll(i));
702 }
703 
704 /*
705  * Find lowest one bit set.
706  * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
707  * The __builtin_ffsll() function is supported by both GCC and Clang.
708  */
709 int
710 lowbit64(uint64_t i)
711 {
712 	if (i == 0)
713 		return (0);
714 
715 	return (__builtin_ffsll(i));
716 }
717 
718 const char *random_path = "/dev/random";
719 const char *urandom_path = "/dev/urandom";
720 static int random_fd = -1, urandom_fd = -1;
721 
722 void
723 random_init(void)
724 {
725 	VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1);
726 	VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1);
727 }
728 
729 void
730 random_fini(void)
731 {
732 	close(random_fd);
733 	close(urandom_fd);
734 
735 	random_fd = -1;
736 	urandom_fd = -1;
737 }
738 
739 static int
740 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
741 {
742 	size_t resid = len;
743 	ssize_t bytes;
744 
745 	ASSERT(fd != -1);
746 
747 	while (resid != 0) {
748 		bytes = read(fd, ptr, resid);
749 		ASSERT3S(bytes, >=, 0);
750 		ptr += bytes;
751 		resid -= bytes;
752 	}
753 
754 	return (0);
755 }
756 
757 int
758 random_get_bytes(uint8_t *ptr, size_t len)
759 {
760 	return (random_get_bytes_common(ptr, len, random_fd));
761 }
762 
763 int
764 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
765 {
766 	return (random_get_bytes_common(ptr, len, urandom_fd));
767 }
768 
769 int
770 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
771 {
772 	(void) nptr;
773 	char *end;
774 
775 	*result = strtoull(str, &end, base);
776 	if (*result == 0)
777 		return (errno);
778 	return (0);
779 }
780 
781 utsname_t *
782 utsname(void)
783 {
784 	return (&hw_utsname);
785 }
786 
787 /*
788  * =========================================================================
789  * kernel emulation setup & teardown
790  * =========================================================================
791  */
792 static int
793 umem_out_of_memory(void)
794 {
795 	char errmsg[] = "out of memory -- generating core dump\n";
796 
797 	(void) fprintf(stderr, "%s", errmsg);
798 	abort();
799 	return (0);
800 }
801 
802 void
803 kernel_init(int mode)
804 {
805 	extern uint_t rrw_tsd_key;
806 
807 	umem_nofail_callback(umem_out_of_memory);
808 
809 	physmem = sysconf(_SC_PHYS_PAGES);
810 
811 	dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem,
812 	    (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
813 
814 	hostid = (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0;
815 
816 	random_init();
817 
818 	VERIFY0(uname(&hw_utsname));
819 
820 	system_taskq_init();
821 	icp_init();
822 
823 	zstd_init();
824 
825 	spa_init((spa_mode_t)mode);
826 
827 	fletcher_4_init();
828 
829 	tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
830 }
831 
832 void
833 kernel_fini(void)
834 {
835 	fletcher_4_fini();
836 	spa_fini();
837 
838 	zstd_fini();
839 
840 	icp_fini();
841 	system_taskq_fini();
842 
843 	random_fini();
844 }
845 
846 uid_t
847 crgetuid(cred_t *cr)
848 {
849 	(void) cr;
850 	return (0);
851 }
852 
853 uid_t
854 crgetruid(cred_t *cr)
855 {
856 	(void) cr;
857 	return (0);
858 }
859 
860 gid_t
861 crgetgid(cred_t *cr)
862 {
863 	(void) cr;
864 	return (0);
865 }
866 
867 int
868 crgetngroups(cred_t *cr)
869 {
870 	(void) cr;
871 	return (0);
872 }
873 
874 gid_t *
875 crgetgroups(cred_t *cr)
876 {
877 	(void) cr;
878 	return (NULL);
879 }
880 
881 int
882 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
883 {
884 	(void) name, (void) cr;
885 	return (0);
886 }
887 
888 int
889 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
890 {
891 	(void) from, (void) to, (void) cr;
892 	return (0);
893 }
894 
895 int
896 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
897 {
898 	(void) name, (void) cr;
899 	return (0);
900 }
901 
902 int
903 secpolicy_zfs(const cred_t *cr)
904 {
905 	(void) cr;
906 	return (0);
907 }
908 
909 int
910 secpolicy_zfs_proc(const cred_t *cr, proc_t *proc)
911 {
912 	(void) cr, (void) proc;
913 	return (0);
914 }
915 
916 ksiddomain_t *
917 ksid_lookupdomain(const char *dom)
918 {
919 	ksiddomain_t *kd;
920 
921 	kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
922 	kd->kd_name = spa_strdup(dom);
923 	return (kd);
924 }
925 
926 void
927 ksiddomain_rele(ksiddomain_t *ksid)
928 {
929 	spa_strfree(ksid->kd_name);
930 	umem_free(ksid, sizeof (ksiddomain_t));
931 }
932 
933 char *
934 kmem_vasprintf(const char *fmt, va_list adx)
935 {
936 	char *buf = NULL;
937 	va_list adx_copy;
938 
939 	va_copy(adx_copy, adx);
940 	VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
941 	va_end(adx_copy);
942 
943 	return (buf);
944 }
945 
946 char *
947 kmem_asprintf(const char *fmt, ...)
948 {
949 	char *buf = NULL;
950 	va_list adx;
951 
952 	va_start(adx, fmt);
953 	VERIFY(vasprintf(&buf, fmt, adx) != -1);
954 	va_end(adx);
955 
956 	return (buf);
957 }
958 
959 zfs_file_t *
960 zfs_onexit_fd_hold(int fd, minor_t *minorp)
961 {
962 	(void) fd;
963 	*minorp = 0;
964 	return (NULL);
965 }
966 
967 void
968 zfs_onexit_fd_rele(zfs_file_t *fp)
969 {
970 	(void) fp;
971 }
972 
973 int
974 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
975     uint64_t *action_handle)
976 {
977 	(void) minor, (void) func, (void) data, (void) action_handle;
978 	return (0);
979 }
980 
981 fstrans_cookie_t
982 spl_fstrans_mark(void)
983 {
984 	return ((fstrans_cookie_t)0);
985 }
986 
987 void
988 spl_fstrans_unmark(fstrans_cookie_t cookie)
989 {
990 	(void) cookie;
991 }
992 
993 int
994 __spl_pf_fstrans_check(void)
995 {
996 	return (0);
997 }
998 
999 int
1000 kmem_cache_reap_active(void)
1001 {
1002 	return (0);
1003 }
1004 
1005 void *zvol_tag = "zvol_tag";
1006 
1007 void
1008 zvol_create_minor(const char *name)
1009 {
1010 	(void) name;
1011 }
1012 
1013 void
1014 zvol_create_minors_recursive(const char *name)
1015 {
1016 	(void) name;
1017 }
1018 
1019 void
1020 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1021 {
1022 	(void) spa, (void) name, (void) async;
1023 }
1024 
1025 void
1026 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,
1027     boolean_t async)
1028 {
1029 	(void) spa, (void) oldname, (void) newname, (void) async;
1030 }
1031 
1032 /*
1033  * Open file
1034  *
1035  * path - fully qualified path to file
1036  * flags - file attributes O_READ / O_WRITE / O_EXCL
1037  * fpp - pointer to return file pointer
1038  *
1039  * Returns 0 on success underlying error on failure.
1040  */
1041 int
1042 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp)
1043 {
1044 	int fd = -1;
1045 	int dump_fd = -1;
1046 	int err;
1047 	int old_umask = 0;
1048 	zfs_file_t *fp;
1049 	struct stat64 st;
1050 
1051 	if (!(flags & O_CREAT) && stat64(path, &st) == -1)
1052 		return (errno);
1053 
1054 	if (!(flags & O_CREAT) && S_ISBLK(st.st_mode))
1055 		flags |= O_DIRECT;
1056 
1057 	if (flags & O_CREAT)
1058 		old_umask = umask(0);
1059 
1060 	fd = open64(path, flags, mode);
1061 	if (fd == -1)
1062 		return (errno);
1063 
1064 	if (flags & O_CREAT)
1065 		(void) umask(old_umask);
1066 
1067 	if (vn_dumpdir != NULL) {
1068 		char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
1069 		const char *inpath = zfs_basename(path);
1070 
1071 		(void) snprintf(dumppath, MAXPATHLEN,
1072 		    "%s/%s", vn_dumpdir, inpath);
1073 		dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
1074 		umem_free(dumppath, MAXPATHLEN);
1075 		if (dump_fd == -1) {
1076 			err = errno;
1077 			close(fd);
1078 			return (err);
1079 		}
1080 	} else {
1081 		dump_fd = -1;
1082 	}
1083 
1084 	(void) fcntl(fd, F_SETFD, FD_CLOEXEC);
1085 
1086 	fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL);
1087 	fp->f_fd = fd;
1088 	fp->f_dump_fd = dump_fd;
1089 	*fpp = fp;
1090 
1091 	return (0);
1092 }
1093 
1094 void
1095 zfs_file_close(zfs_file_t *fp)
1096 {
1097 	close(fp->f_fd);
1098 	if (fp->f_dump_fd != -1)
1099 		close(fp->f_dump_fd);
1100 
1101 	umem_free(fp, sizeof (zfs_file_t));
1102 }
1103 
1104 /*
1105  * Stateful write - use os internal file pointer to determine where to
1106  * write and update on successful completion.
1107  *
1108  * fp -  pointer to file (pipe, socket, etc) to write to
1109  * buf - buffer to write
1110  * count - # of bytes to write
1111  * resid -  pointer to count of unwritten bytes  (if short write)
1112  *
1113  * Returns 0 on success errno on failure.
1114  */
1115 int
1116 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
1117 {
1118 	ssize_t rc;
1119 
1120 	rc = write(fp->f_fd, buf, count);
1121 	if (rc < 0)
1122 		return (errno);
1123 
1124 	if (resid) {
1125 		*resid = count - rc;
1126 	} else if (rc != count) {
1127 		return (EIO);
1128 	}
1129 
1130 	return (0);
1131 }
1132 
1133 /*
1134  * Stateless write - os internal file pointer is not updated.
1135  *
1136  * fp -  pointer to file (pipe, socket, etc) to write to
1137  * buf - buffer to write
1138  * count - # of bytes to write
1139  * off - file offset to write to (only valid for seekable types)
1140  * resid -  pointer to count of unwritten bytes
1141  *
1142  * Returns 0 on success errno on failure.
1143  */
1144 int
1145 zfs_file_pwrite(zfs_file_t *fp, const void *buf,
1146     size_t count, loff_t pos, ssize_t *resid)
1147 {
1148 	ssize_t rc, split, done;
1149 	int sectors;
1150 
1151 	/*
1152 	 * To simulate partial disk writes, we split writes into two
1153 	 * system calls so that the process can be killed in between.
1154 	 * This is used by ztest to simulate realistic failure modes.
1155 	 */
1156 	sectors = count >> SPA_MINBLOCKSHIFT;
1157 	split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT;
1158 	rc = pwrite64(fp->f_fd, buf, split, pos);
1159 	if (rc != -1) {
1160 		done = rc;
1161 		rc = pwrite64(fp->f_fd, (char *)buf + split,
1162 		    count - split, pos + split);
1163 	}
1164 #ifdef __linux__
1165 	if (rc == -1 && errno == EINVAL) {
1166 		/*
1167 		 * Under Linux, this most likely means an alignment issue
1168 		 * (memory or disk) due to O_DIRECT, so we abort() in order
1169 		 * to catch the offender.
1170 		 */
1171 		abort();
1172 	}
1173 #endif
1174 
1175 	if (rc < 0)
1176 		return (errno);
1177 
1178 	done += rc;
1179 
1180 	if (resid) {
1181 		*resid = count - done;
1182 	} else if (done != count) {
1183 		return (EIO);
1184 	}
1185 
1186 	return (0);
1187 }
1188 
1189 /*
1190  * Stateful read - use os internal file pointer to determine where to
1191  * read and update on successful completion.
1192  *
1193  * fp -  pointer to file (pipe, socket, etc) to read from
1194  * buf - buffer to write
1195  * count - # of bytes to read
1196  * resid -  pointer to count of unread bytes (if short read)
1197  *
1198  * Returns 0 on success errno on failure.
1199  */
1200 int
1201 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid)
1202 {
1203 	int rc;
1204 
1205 	rc = read(fp->f_fd, buf, count);
1206 	if (rc < 0)
1207 		return (errno);
1208 
1209 	if (resid) {
1210 		*resid = count - rc;
1211 	} else if (rc != count) {
1212 		return (EIO);
1213 	}
1214 
1215 	return (0);
1216 }
1217 
1218 /*
1219  * Stateless read - os internal file pointer is not updated.
1220  *
1221  * fp -  pointer to file (pipe, socket, etc) to read from
1222  * buf - buffer to write
1223  * count - # of bytes to write
1224  * off - file offset to read from (only valid for seekable types)
1225  * resid -  pointer to count of unwritten bytes (if short write)
1226  *
1227  * Returns 0 on success errno on failure.
1228  */
1229 int
1230 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off,
1231     ssize_t *resid)
1232 {
1233 	ssize_t rc;
1234 
1235 	rc = pread64(fp->f_fd, buf, count, off);
1236 	if (rc < 0) {
1237 #ifdef __linux__
1238 		/*
1239 		 * Under Linux, this most likely means an alignment issue
1240 		 * (memory or disk) due to O_DIRECT, so we abort() in order to
1241 		 * catch the offender.
1242 		 */
1243 		if (errno == EINVAL)
1244 			abort();
1245 #endif
1246 		return (errno);
1247 	}
1248 
1249 	if (fp->f_dump_fd != -1) {
1250 		int status;
1251 
1252 		status = pwrite64(fp->f_dump_fd, buf, rc, off);
1253 		ASSERT(status != -1);
1254 	}
1255 
1256 	if (resid) {
1257 		*resid = count - rc;
1258 	} else if (rc != count) {
1259 		return (EIO);
1260 	}
1261 
1262 	return (0);
1263 }
1264 
1265 /*
1266  * lseek - set / get file pointer
1267  *
1268  * fp -  pointer to file (pipe, socket, etc) to read from
1269  * offp - value to seek to, returns current value plus passed offset
1270  * whence - see man pages for standard lseek whence values
1271  *
1272  * Returns 0 on success errno on failure (ESPIPE for non seekable types)
1273  */
1274 int
1275 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence)
1276 {
1277 	loff_t rc;
1278 
1279 	rc = lseek(fp->f_fd, *offp, whence);
1280 	if (rc < 0)
1281 		return (errno);
1282 
1283 	*offp = rc;
1284 
1285 	return (0);
1286 }
1287 
1288 /*
1289  * Get file attributes
1290  *
1291  * filp - file pointer
1292  * zfattr - pointer to file attr structure
1293  *
1294  * Currently only used for fetching size and file mode
1295  *
1296  * Returns 0 on success or error code of underlying getattr call on failure.
1297  */
1298 int
1299 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr)
1300 {
1301 	struct stat64 st;
1302 
1303 	if (fstat64_blk(fp->f_fd, &st) == -1)
1304 		return (errno);
1305 
1306 	zfattr->zfa_size = st.st_size;
1307 	zfattr->zfa_mode = st.st_mode;
1308 
1309 	return (0);
1310 }
1311 
1312 /*
1313  * Sync file to disk
1314  *
1315  * filp - file pointer
1316  * flags - O_SYNC and or O_DSYNC
1317  *
1318  * Returns 0 on success or error code of underlying sync call on failure.
1319  */
1320 int
1321 zfs_file_fsync(zfs_file_t *fp, int flags)
1322 {
1323 	(void) flags;
1324 
1325 	if (fsync(fp->f_fd) < 0)
1326 		return (errno);
1327 
1328 	return (0);
1329 }
1330 
1331 /*
1332  * fallocate - allocate or free space on disk
1333  *
1334  * fp - file pointer
1335  * mode (non-standard options for hole punching etc)
1336  * offset - offset to start allocating or freeing from
1337  * len - length to free / allocate
1338  *
1339  * OPTIONAL
1340  */
1341 int
1342 zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len)
1343 {
1344 #ifdef __linux__
1345 	return (fallocate(fp->f_fd, mode, offset, len));
1346 #else
1347 	(void) fp, (void) mode, (void) offset, (void) len;
1348 	return (EOPNOTSUPP);
1349 #endif
1350 }
1351 
1352 /*
1353  * Request current file pointer offset
1354  *
1355  * fp - pointer to file
1356  *
1357  * Returns current file offset.
1358  */
1359 loff_t
1360 zfs_file_off(zfs_file_t *fp)
1361 {
1362 	return (lseek(fp->f_fd, SEEK_CUR, 0));
1363 }
1364 
1365 /*
1366  * unlink file
1367  *
1368  * path - fully qualified file path
1369  *
1370  * Returns 0 on success.
1371  *
1372  * OPTIONAL
1373  */
1374 int
1375 zfs_file_unlink(const char *path)
1376 {
1377 	return (remove(path));
1378 }
1379 
1380 /*
1381  * Get reference to file pointer
1382  *
1383  * fd - input file descriptor
1384  *
1385  * Returns pointer to file struct or NULL.
1386  * Unsupported in user space.
1387  */
1388 zfs_file_t *
1389 zfs_file_get(int fd)
1390 {
1391 	(void) fd;
1392 	abort();
1393 	return (NULL);
1394 }
1395 /*
1396  * Drop reference to file pointer
1397  *
1398  * fp - pointer to file struct
1399  *
1400  * Unsupported in user space.
1401  */
1402 void
1403 zfs_file_put(zfs_file_t *fp)
1404 {
1405 	abort();
1406 	(void) fp;
1407 }
1408 
1409 void
1410 zfsvfs_update_fromname(const char *oldname, const char *newname)
1411 {
1412 	(void) oldname, (void) newname;
1413 }
1414 
1415 void
1416 spa_import_os(spa_t *spa)
1417 {
1418 	(void) spa;
1419 }
1420 
1421 void
1422 spa_export_os(spa_t *spa)
1423 {
1424 	(void) spa;
1425 }
1426 
1427 void
1428 spa_activate_os(spa_t *spa)
1429 {
1430 	(void) spa;
1431 }
1432 
1433 void
1434 spa_deactivate_os(spa_t *spa)
1435 {
1436 	(void) spa;
1437 }
1438