xref: /freebsd/sys/dev/md/md.c (revision 734e78dfcb0b1c724a336101db3edd5519ef342f)
1098ca2bdSWarner Losh /*-
200a6a3c6SPoul-Henning Kamp  * ----------------------------------------------------------------------------
300a6a3c6SPoul-Henning Kamp  * "THE BEER-WARE LICENSE" (Revision 42):
400a6a3c6SPoul-Henning Kamp  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
500a6a3c6SPoul-Henning Kamp  * can do whatever you want with this stuff. If we meet some day, and you think
600a6a3c6SPoul-Henning Kamp  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
700a6a3c6SPoul-Henning Kamp  * ----------------------------------------------------------------------------
800a6a3c6SPoul-Henning Kamp  *
900a6a3c6SPoul-Henning Kamp  * $FreeBSD$
1000a6a3c6SPoul-Henning Kamp  *
1100a6a3c6SPoul-Henning Kamp  */
1200a6a3c6SPoul-Henning Kamp 
13098ca2bdSWarner Losh /*-
14637f671aSPoul-Henning Kamp  * The following functions are based in the vn(4) driver: mdstart_swap(),
15637f671aSPoul-Henning Kamp  * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(),
16637f671aSPoul-Henning Kamp  * and as such under the following copyright:
17637f671aSPoul-Henning Kamp  *
18637f671aSPoul-Henning Kamp  * Copyright (c) 1988 University of Utah.
19637f671aSPoul-Henning Kamp  * Copyright (c) 1990, 1993
20637f671aSPoul-Henning Kamp  *	The Regents of the University of California.  All rights reserved.
21637f671aSPoul-Henning Kamp  *
22ed010cdfSWarner Losh  * This code is derived from software contributed to Berkeley by
23ed010cdfSWarner Losh  * the Systems Programming Group of the University of Utah Computer
24ed010cdfSWarner Losh  * Science Department.
25ed010cdfSWarner Losh  *
26637f671aSPoul-Henning Kamp  * Redistribution and use in source and binary forms, with or without
27637f671aSPoul-Henning Kamp  * modification, are permitted provided that the following conditions
28637f671aSPoul-Henning Kamp  * are met:
29637f671aSPoul-Henning Kamp  * 1. Redistributions of source code must retain the above copyright
30637f671aSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer.
31637f671aSPoul-Henning Kamp  * 2. Redistributions in binary form must reproduce the above copyright
32637f671aSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer in the
33637f671aSPoul-Henning Kamp  *    documentation and/or other materials provided with the distribution.
34637f671aSPoul-Henning Kamp  * 4. Neither the name of the University nor the names of its contributors
35637f671aSPoul-Henning Kamp  *    may be used to endorse or promote products derived from this software
36637f671aSPoul-Henning Kamp  *    without specific prior written permission.
37637f671aSPoul-Henning Kamp  *
38637f671aSPoul-Henning Kamp  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
39637f671aSPoul-Henning Kamp  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40637f671aSPoul-Henning Kamp  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
41637f671aSPoul-Henning Kamp  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
42637f671aSPoul-Henning Kamp  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
43637f671aSPoul-Henning Kamp  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
44637f671aSPoul-Henning Kamp  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
45637f671aSPoul-Henning Kamp  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
46637f671aSPoul-Henning Kamp  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
47637f671aSPoul-Henning Kamp  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
48637f671aSPoul-Henning Kamp  * SUCH DAMAGE.
49637f671aSPoul-Henning Kamp  *
50637f671aSPoul-Henning Kamp  * from: Utah Hdr: vn.c 1.13 94/04/02
51637f671aSPoul-Henning Kamp  *
52637f671aSPoul-Henning Kamp  *	from: @(#)vn.c	8.6 (Berkeley) 4/1/94
53637f671aSPoul-Henning Kamp  * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03
54637f671aSPoul-Henning Kamp  */
55637f671aSPoul-Henning Kamp 
566f4f00f1SPoul-Henning Kamp #include "opt_geom.h"
573f54a085SPoul-Henning Kamp #include "opt_md.h"
5871e4fff8SPoul-Henning Kamp 
5900a6a3c6SPoul-Henning Kamp #include <sys/param.h>
6000a6a3c6SPoul-Henning Kamp #include <sys/systm.h>
619626b608SPoul-Henning Kamp #include <sys/bio.h>
6200a6a3c6SPoul-Henning Kamp #include <sys/conf.h>
63a03be42dSMaxim Sobolev #include <sys/devicestat.h>
648f8def9eSPoul-Henning Kamp #include <sys/fcntl.h>
65fb919e4dSMark Murray #include <sys/kernel.h>
665c97ca54SIan Dowse #include <sys/kthread.h>
6706d425f9SEd Schouten #include <sys/limits.h>
68fb919e4dSMark Murray #include <sys/linker.h>
69fb919e4dSMark Murray #include <sys/lock.h>
70fb919e4dSMark Murray #include <sys/malloc.h>
71fb919e4dSMark Murray #include <sys/mdioctl.h>
72a08d2e7fSJohn Baldwin #include <sys/mount.h>
739dceb26bSJohn Baldwin #include <sys/mutex.h>
749b00ca19SPoul-Henning Kamp #include <sys/sx.h>
75fb919e4dSMark Murray #include <sys/namei.h>
768f8def9eSPoul-Henning Kamp #include <sys/proc.h>
77fb919e4dSMark Murray #include <sys/queue.h>
78657bd8b1SAndrey V. Elsukov #include <sys/sbuf.h>
7963710c4dSJohn Baldwin #include <sys/sched.h>
807cd53fddSAlan Cox #include <sys/sf_buf.h>
81fb919e4dSMark Murray #include <sys/sysctl.h>
82fb919e4dSMark Murray #include <sys/vnode.h>
83fb919e4dSMark Murray 
846f4f00f1SPoul-Henning Kamp #include <geom/geom.h>
856f4f00f1SPoul-Henning Kamp 
868f8def9eSPoul-Henning Kamp #include <vm/vm.h>
871c771f92SKonstantin Belousov #include <vm/vm_param.h>
888f8def9eSPoul-Henning Kamp #include <vm/vm_object.h>
898f8def9eSPoul-Henning Kamp #include <vm/vm_page.h>
908f8def9eSPoul-Henning Kamp #include <vm/vm_pager.h>
918f8def9eSPoul-Henning Kamp #include <vm/swap_pager.h>
92f43b2bacSPoul-Henning Kamp #include <vm/uma.h>
933f54a085SPoul-Henning Kamp 
9457e9624eSPoul-Henning Kamp #define MD_MODVER 1
9557e9624eSPoul-Henning Kamp 
965c97ca54SIan Dowse #define MD_SHUTDOWN	0x10000		/* Tell worker thread to terminate. */
97a08d2e7fSJohn Baldwin #define	MD_EXITING	0x20000		/* Worker thread is exiting. */
985c97ca54SIan Dowse 
99f2744793SSheldon Hearn #ifndef MD_NSECT
100f2744793SSheldon Hearn #define MD_NSECT (10000 * 2)
10133edfabeSPoul-Henning Kamp #endif
10233edfabeSPoul-Henning Kamp 
1035bb84bc8SRobert Watson static MALLOC_DEFINE(M_MD, "md_disk", "Memory Disk");
1045bb84bc8SRobert Watson static MALLOC_DEFINE(M_MDSECT, "md_sectors", "Memory Disk Sectors");
10500a6a3c6SPoul-Henning Kamp 
10671e4fff8SPoul-Henning Kamp static int md_debug;
1073eb9ab52SEitan Adler SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0,
1083eb9ab52SEitan Adler     "Enable md(4) debug messages");
109c44d423eSKonstantin Belousov static int md_malloc_wait;
1103eb9ab52SEitan Adler SYSCTL_INT(_vm, OID_AUTO, md_malloc_wait, CTLFLAG_RW, &md_malloc_wait, 0,
1113eb9ab52SEitan Adler     "Allow malloc to wait for memory allocations");
11200a6a3c6SPoul-Henning Kamp 
11322ff74b2SMarcel Moolenaar #if defined(MD_ROOT) && !defined(MD_ROOT_FSTYPE)
11422ff74b2SMarcel Moolenaar #define	MD_ROOT_FSTYPE	"ufs"
11522ff74b2SMarcel Moolenaar #endif
11622ff74b2SMarcel Moolenaar 
11771e4fff8SPoul-Henning Kamp #if defined(MD_ROOT) && defined(MD_ROOT_SIZE)
118de64f22aSLuigi Rizzo /*
119de64f22aSLuigi Rizzo  * Preloaded image gets put here.
120de64f22aSLuigi Rizzo  * Applications that patch the object with the image can determine
121de64f22aSLuigi Rizzo  * the size looking at the start and end markers (strings),
122de64f22aSLuigi Rizzo  * so we want them contiguous.
123de64f22aSLuigi Rizzo  */
124de64f22aSLuigi Rizzo static struct {
125de64f22aSLuigi Rizzo 	u_char start[MD_ROOT_SIZE*1024];
126de64f22aSLuigi Rizzo 	u_char end[128];
127de64f22aSLuigi Rizzo } mfs_root = {
128de64f22aSLuigi Rizzo 	.start = "MFS Filesystem goes here",
129de64f22aSLuigi Rizzo 	.end = "MFS Filesystem had better STOP here",
130de64f22aSLuigi Rizzo };
13171e4fff8SPoul-Henning Kamp #endif
13271e4fff8SPoul-Henning Kamp 
13319945697SPoul-Henning Kamp static g_init_t g_md_init;
13419945697SPoul-Henning Kamp static g_fini_t g_md_fini;
13519945697SPoul-Henning Kamp static g_start_t g_md_start;
13619945697SPoul-Henning Kamp static g_access_t g_md_access;
137b42f40b8SJaakko Heinonen static void g_md_dumpconf(struct sbuf *sb, const char *indent,
138b42f40b8SJaakko Heinonen     struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp);
1390eb14309SPoul-Henning Kamp 
1408f8def9eSPoul-Henning Kamp static int mdunits;
14189c9c53dSPoul-Henning Kamp static struct cdev *status_dev = 0;
1429b00ca19SPoul-Henning Kamp static struct sx md_sx;
143f4e7c5a8SJaakko Heinonen static struct unrhdr *md_uh;
14457e9624eSPoul-Henning Kamp 
145a522a159SPoul-Henning Kamp static d_ioctl_t mdctlioctl;
1468f8def9eSPoul-Henning Kamp 
1478f8def9eSPoul-Henning Kamp static struct cdevsw mdctl_cdevsw = {
148dc08ffecSPoul-Henning Kamp 	.d_version =	D_VERSION,
1497ac40f5fSPoul-Henning Kamp 	.d_ioctl =	mdctlioctl,
1507ac40f5fSPoul-Henning Kamp 	.d_name =	MD_NAME,
15100a6a3c6SPoul-Henning Kamp };
15200a6a3c6SPoul-Henning Kamp 
15319945697SPoul-Henning Kamp struct g_class g_md_class = {
15419945697SPoul-Henning Kamp 	.name = "MD",
1555721c9c7SPoul-Henning Kamp 	.version = G_VERSION,
15619945697SPoul-Henning Kamp 	.init = g_md_init,
15719945697SPoul-Henning Kamp 	.fini = g_md_fini,
15819945697SPoul-Henning Kamp 	.start = g_md_start,
15919945697SPoul-Henning Kamp 	.access = g_md_access,
160c27a8954SWojciech A. Koszek 	.dumpconf = g_md_dumpconf,
16119945697SPoul-Henning Kamp };
16219945697SPoul-Henning Kamp 
16319945697SPoul-Henning Kamp DECLARE_GEOM_CLASS(g_md_class, g_md);
16419945697SPoul-Henning Kamp 
1650cfaeeeeSPoul-Henning Kamp 
16613e403fdSAntoine Brodin static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(md_softc_list);
1673f54a085SPoul-Henning Kamp 
168c6517568SPoul-Henning Kamp #define NINDIR	(PAGE_SIZE / sizeof(uintptr_t))
169c6517568SPoul-Henning Kamp #define NMASK	(NINDIR-1)
170c6517568SPoul-Henning Kamp static int nshift;
171c6517568SPoul-Henning Kamp 
172c6517568SPoul-Henning Kamp struct indir {
173c6517568SPoul-Henning Kamp 	uintptr_t	*array;
1748b149b51SJohn Baldwin 	u_int		total;
1758b149b51SJohn Baldwin 	u_int		used;
1768b149b51SJohn Baldwin 	u_int		shift;
177c6517568SPoul-Henning Kamp };
178c6517568SPoul-Henning Kamp 
17900a6a3c6SPoul-Henning Kamp struct md_s {
18000a6a3c6SPoul-Henning Kamp 	int unit;
1813f54a085SPoul-Henning Kamp 	LIST_ENTRY(md_s) list;
1828177437dSPoul-Henning Kamp 	struct bio_queue_head bio_queue;
1830f8500a5SPoul-Henning Kamp 	struct mtx queue_mtx;
18489c9c53dSPoul-Henning Kamp 	struct cdev *dev;
1858f8def9eSPoul-Henning Kamp 	enum md_types type;
186b830359bSPawel Jakub Dawidek 	off_t mediasize;
187b830359bSPawel Jakub Dawidek 	unsigned sectorsize;
188fe603109SMaxim Sobolev 	unsigned opencount;
1894e8bfe14SPoul-Henning Kamp 	unsigned fwheads;
1904e8bfe14SPoul-Henning Kamp 	unsigned fwsectors;
1918f8def9eSPoul-Henning Kamp 	unsigned flags;
192f43b2bacSPoul-Henning Kamp 	char name[20];
1935c97ca54SIan Dowse 	struct proc *procp;
1946f4f00f1SPoul-Henning Kamp 	struct g_geom *gp;
1956f4f00f1SPoul-Henning Kamp 	struct g_provider *pp;
1969b00ca19SPoul-Henning Kamp 	int (*start)(struct md_s *sc, struct bio *bp);
197a03be42dSMaxim Sobolev 	struct devstat *devstat;
19895f1a897SPoul-Henning Kamp 
19995f1a897SPoul-Henning Kamp 	/* MD_MALLOC related fields */
200c6517568SPoul-Henning Kamp 	struct indir *indir;
201f43b2bacSPoul-Henning Kamp 	uma_zone_t uma;
20200a6a3c6SPoul-Henning Kamp 
20395f1a897SPoul-Henning Kamp 	/* MD_PRELOAD related fields */
20495f1a897SPoul-Henning Kamp 	u_char *pl_ptr;
205b830359bSPawel Jakub Dawidek 	size_t pl_len;
20600a6a3c6SPoul-Henning Kamp 
2078f8def9eSPoul-Henning Kamp 	/* MD_VNODE related fields */
2088f8def9eSPoul-Henning Kamp 	struct vnode *vnode;
20961a6eb62SPawel Jakub Dawidek 	char file[PATH_MAX];
2108f8def9eSPoul-Henning Kamp 	struct ucred *cred;
2118f8def9eSPoul-Henning Kamp 
212e0cebb40SDima Dorfman 	/* MD_SWAP related fields */
2138f8def9eSPoul-Henning Kamp 	vm_object_t object;
2148f8def9eSPoul-Henning Kamp };
21500a6a3c6SPoul-Henning Kamp 
216c6517568SPoul-Henning Kamp static struct indir *
2178b149b51SJohn Baldwin new_indir(u_int shift)
218c6517568SPoul-Henning Kamp {
219c6517568SPoul-Henning Kamp 	struct indir *ip;
220c6517568SPoul-Henning Kamp 
221c44d423eSKonstantin Belousov 	ip = malloc(sizeof *ip, M_MD, (md_malloc_wait ? M_WAITOK : M_NOWAIT)
222c44d423eSKonstantin Belousov 	    | M_ZERO);
223c6517568SPoul-Henning Kamp 	if (ip == NULL)
224c6517568SPoul-Henning Kamp 		return (NULL);
225c6517568SPoul-Henning Kamp 	ip->array = malloc(sizeof(uintptr_t) * NINDIR,
226c44d423eSKonstantin Belousov 	    M_MDSECT, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO);
227c6517568SPoul-Henning Kamp 	if (ip->array == NULL) {
228c6517568SPoul-Henning Kamp 		free(ip, M_MD);
229c6517568SPoul-Henning Kamp 		return (NULL);
230c6517568SPoul-Henning Kamp 	}
231c6517568SPoul-Henning Kamp 	ip->total = NINDIR;
232c6517568SPoul-Henning Kamp 	ip->shift = shift;
233c6517568SPoul-Henning Kamp 	return (ip);
234c6517568SPoul-Henning Kamp }
235c6517568SPoul-Henning Kamp 
236c6517568SPoul-Henning Kamp static void
237c6517568SPoul-Henning Kamp del_indir(struct indir *ip)
238c6517568SPoul-Henning Kamp {
239c6517568SPoul-Henning Kamp 
240f43b2bacSPoul-Henning Kamp 	free(ip->array, M_MDSECT);
241c6517568SPoul-Henning Kamp 	free(ip, M_MD);
242c6517568SPoul-Henning Kamp }
243c6517568SPoul-Henning Kamp 
244f43b2bacSPoul-Henning Kamp static void
245f43b2bacSPoul-Henning Kamp destroy_indir(struct md_s *sc, struct indir *ip)
246f43b2bacSPoul-Henning Kamp {
247f43b2bacSPoul-Henning Kamp 	int i;
248f43b2bacSPoul-Henning Kamp 
249f43b2bacSPoul-Henning Kamp 	for (i = 0; i < NINDIR; i++) {
250f43b2bacSPoul-Henning Kamp 		if (!ip->array[i])
251f43b2bacSPoul-Henning Kamp 			continue;
252f43b2bacSPoul-Henning Kamp 		if (ip->shift)
253f43b2bacSPoul-Henning Kamp 			destroy_indir(sc, (struct indir*)(ip->array[i]));
254f43b2bacSPoul-Henning Kamp 		else if (ip->array[i] > 255)
255f43b2bacSPoul-Henning Kamp 			uma_zfree(sc->uma, (void *)(ip->array[i]));
256f43b2bacSPoul-Henning Kamp 	}
257f43b2bacSPoul-Henning Kamp 	del_indir(ip);
258f43b2bacSPoul-Henning Kamp }
259f43b2bacSPoul-Henning Kamp 
260c6517568SPoul-Henning Kamp /*
2616c3cd0e2SMaxim Konovalov  * This function does the math and allocates the top level "indir" structure
262c6517568SPoul-Henning Kamp  * for a device of "size" sectors.
263c6517568SPoul-Henning Kamp  */
264c6517568SPoul-Henning Kamp 
265c6517568SPoul-Henning Kamp static struct indir *
266c6517568SPoul-Henning Kamp dimension(off_t size)
267c6517568SPoul-Henning Kamp {
268c6517568SPoul-Henning Kamp 	off_t rcnt;
269c6517568SPoul-Henning Kamp 	struct indir *ip;
270d12fc952SKonstantin Belousov 	int layer;
271c6517568SPoul-Henning Kamp 
272c6517568SPoul-Henning Kamp 	rcnt = size;
273c6517568SPoul-Henning Kamp 	layer = 0;
274c6517568SPoul-Henning Kamp 	while (rcnt > NINDIR) {
275c6517568SPoul-Henning Kamp 		rcnt /= NINDIR;
276c6517568SPoul-Henning Kamp 		layer++;
277c6517568SPoul-Henning Kamp 	}
278c6517568SPoul-Henning Kamp 
279c6517568SPoul-Henning Kamp 	/*
280c6517568SPoul-Henning Kamp 	 * XXX: the top layer is probably not fully populated, so we allocate
28183e13864SPoul-Henning Kamp 	 * too much space for ip->array in here.
282c6517568SPoul-Henning Kamp 	 */
28383e13864SPoul-Henning Kamp 	ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO);
28483e13864SPoul-Henning Kamp 	ip->array = malloc(sizeof(uintptr_t) * NINDIR,
28583e13864SPoul-Henning Kamp 	    M_MDSECT, M_WAITOK | M_ZERO);
28683e13864SPoul-Henning Kamp 	ip->total = NINDIR;
28783e13864SPoul-Henning Kamp 	ip->shift = layer * nshift;
288c6517568SPoul-Henning Kamp 	return (ip);
289c6517568SPoul-Henning Kamp }
290c6517568SPoul-Henning Kamp 
291c6517568SPoul-Henning Kamp /*
292c6517568SPoul-Henning Kamp  * Read a given sector
293c6517568SPoul-Henning Kamp  */
294c6517568SPoul-Henning Kamp 
295c6517568SPoul-Henning Kamp static uintptr_t
296c6517568SPoul-Henning Kamp s_read(struct indir *ip, off_t offset)
297c6517568SPoul-Henning Kamp {
298c6517568SPoul-Henning Kamp 	struct indir *cip;
299c6517568SPoul-Henning Kamp 	int idx;
300c6517568SPoul-Henning Kamp 	uintptr_t up;
301c6517568SPoul-Henning Kamp 
302c6517568SPoul-Henning Kamp 	if (md_debug > 1)
3036569d6f3SMaxime Henrion 		printf("s_read(%jd)\n", (intmax_t)offset);
304c6517568SPoul-Henning Kamp 	up = 0;
305c6517568SPoul-Henning Kamp 	for (cip = ip; cip != NULL;) {
306c6517568SPoul-Henning Kamp 		if (cip->shift) {
307c6517568SPoul-Henning Kamp 			idx = (offset >> cip->shift) & NMASK;
308c6517568SPoul-Henning Kamp 			up = cip->array[idx];
309c6517568SPoul-Henning Kamp 			cip = (struct indir *)up;
310c6517568SPoul-Henning Kamp 			continue;
311c6517568SPoul-Henning Kamp 		}
312c6517568SPoul-Henning Kamp 		idx = offset & NMASK;
313c6517568SPoul-Henning Kamp 		return (cip->array[idx]);
314c6517568SPoul-Henning Kamp 	}
315c6517568SPoul-Henning Kamp 	return (0);
316c6517568SPoul-Henning Kamp }
317c6517568SPoul-Henning Kamp 
318c6517568SPoul-Henning Kamp /*
319c6517568SPoul-Henning Kamp  * Write a given sector, prune the tree if the value is 0
320c6517568SPoul-Henning Kamp  */
321c6517568SPoul-Henning Kamp 
322c6517568SPoul-Henning Kamp static int
323fde2a2e4SPoul-Henning Kamp s_write(struct indir *ip, off_t offset, uintptr_t ptr)
324c6517568SPoul-Henning Kamp {
325c6517568SPoul-Henning Kamp 	struct indir *cip, *lip[10];
326c6517568SPoul-Henning Kamp 	int idx, li;
327c6517568SPoul-Henning Kamp 	uintptr_t up;
328c6517568SPoul-Henning Kamp 
329c6517568SPoul-Henning Kamp 	if (md_debug > 1)
3306569d6f3SMaxime Henrion 		printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr);
331c6517568SPoul-Henning Kamp 	up = 0;
332c6517568SPoul-Henning Kamp 	li = 0;
333c6517568SPoul-Henning Kamp 	cip = ip;
334c6517568SPoul-Henning Kamp 	for (;;) {
335c6517568SPoul-Henning Kamp 		lip[li++] = cip;
336c6517568SPoul-Henning Kamp 		if (cip->shift) {
337c6517568SPoul-Henning Kamp 			idx = (offset >> cip->shift) & NMASK;
338c6517568SPoul-Henning Kamp 			up = cip->array[idx];
339c6517568SPoul-Henning Kamp 			if (up != 0) {
340c6517568SPoul-Henning Kamp 				cip = (struct indir *)up;
341c6517568SPoul-Henning Kamp 				continue;
342c6517568SPoul-Henning Kamp 			}
343c6517568SPoul-Henning Kamp 			/* Allocate branch */
344c6517568SPoul-Henning Kamp 			cip->array[idx] =
345c6517568SPoul-Henning Kamp 			    (uintptr_t)new_indir(cip->shift - nshift);
346c6517568SPoul-Henning Kamp 			if (cip->array[idx] == 0)
347975b628fSPoul-Henning Kamp 				return (ENOSPC);
348c6517568SPoul-Henning Kamp 			cip->used++;
349c6517568SPoul-Henning Kamp 			up = cip->array[idx];
350c6517568SPoul-Henning Kamp 			cip = (struct indir *)up;
351c6517568SPoul-Henning Kamp 			continue;
352c6517568SPoul-Henning Kamp 		}
353c6517568SPoul-Henning Kamp 		/* leafnode */
354c6517568SPoul-Henning Kamp 		idx = offset & NMASK;
355c6517568SPoul-Henning Kamp 		up = cip->array[idx];
356c6517568SPoul-Henning Kamp 		if (up != 0)
357c6517568SPoul-Henning Kamp 			cip->used--;
358c6517568SPoul-Henning Kamp 		cip->array[idx] = ptr;
359c6517568SPoul-Henning Kamp 		if (ptr != 0)
360c6517568SPoul-Henning Kamp 			cip->used++;
361c6517568SPoul-Henning Kamp 		break;
362c6517568SPoul-Henning Kamp 	}
363c6517568SPoul-Henning Kamp 	if (cip->used != 0 || li == 1)
364c6517568SPoul-Henning Kamp 		return (0);
365c6517568SPoul-Henning Kamp 	li--;
366c6517568SPoul-Henning Kamp 	while (cip->used == 0 && cip != ip) {
367c6517568SPoul-Henning Kamp 		li--;
368c6517568SPoul-Henning Kamp 		idx = (offset >> lip[li]->shift) & NMASK;
369c6517568SPoul-Henning Kamp 		up = lip[li]->array[idx];
370c6517568SPoul-Henning Kamp 		KASSERT(up == (uintptr_t)cip, ("md screwed up"));
371c6517568SPoul-Henning Kamp 		del_indir(cip);
3724a6a94d8SArchie Cobbs 		lip[li]->array[idx] = 0;
373c6517568SPoul-Henning Kamp 		lip[li]->used--;
374c6517568SPoul-Henning Kamp 		cip = lip[li];
375c6517568SPoul-Henning Kamp 	}
376c6517568SPoul-Henning Kamp 	return (0);
377c6517568SPoul-Henning Kamp }
378c6517568SPoul-Henning Kamp 
3796f4f00f1SPoul-Henning Kamp 
3806f4f00f1SPoul-Henning Kamp static int
3816f4f00f1SPoul-Henning Kamp g_md_access(struct g_provider *pp, int r, int w, int e)
3826f4f00f1SPoul-Henning Kamp {
3836f4f00f1SPoul-Henning Kamp 	struct md_s *sc;
3846f4f00f1SPoul-Henning Kamp 
3856f4f00f1SPoul-Henning Kamp 	sc = pp->geom->softc;
38641c8b468SEdward Tomasz Napierala 	if (sc == NULL) {
38741c8b468SEdward Tomasz Napierala 		if (r <= 0 && w <= 0 && e <= 0)
38841c8b468SEdward Tomasz Napierala 			return (0);
3896b60a2cdSPoul-Henning Kamp 		return (ENXIO);
39041c8b468SEdward Tomasz Napierala 	}
3916f4f00f1SPoul-Henning Kamp 	r += pp->acr;
3926f4f00f1SPoul-Henning Kamp 	w += pp->acw;
3936f4f00f1SPoul-Henning Kamp 	e += pp->ace;
39486776891SChristian S.J. Peron 	if ((sc->flags & MD_READONLY) != 0 && w > 0)
39586776891SChristian S.J. Peron 		return (EROFS);
3966f4f00f1SPoul-Henning Kamp 	if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) {
3976f4f00f1SPoul-Henning Kamp 		sc->opencount = 1;
3986f4f00f1SPoul-Henning Kamp 	} else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) {
3996f4f00f1SPoul-Henning Kamp 		sc->opencount = 0;
4006f4f00f1SPoul-Henning Kamp 	}
4016f4f00f1SPoul-Henning Kamp 	return (0);
4026f4f00f1SPoul-Henning Kamp }
4036f4f00f1SPoul-Henning Kamp 
4046f4f00f1SPoul-Henning Kamp static void
4056f4f00f1SPoul-Henning Kamp g_md_start(struct bio *bp)
4066f4f00f1SPoul-Henning Kamp {
4076f4f00f1SPoul-Henning Kamp 	struct md_s *sc;
4086f4f00f1SPoul-Henning Kamp 
4096f4f00f1SPoul-Henning Kamp 	sc = bp->bio_to->geom->softc;
410a03be42dSMaxim Sobolev 	if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE))
411a03be42dSMaxim Sobolev 		devstat_start_transaction_bio(sc->devstat, bp);
4120f8500a5SPoul-Henning Kamp 	mtx_lock(&sc->queue_mtx);
413891619a6SPoul-Henning Kamp 	bioq_disksort(&sc->bio_queue, bp);
4144b07ede4SPawel Jakub Dawidek 	mtx_unlock(&sc->queue_mtx);
415e4cdd0d4SPawel Jakub Dawidek 	wakeup(sc);
4166f4f00f1SPoul-Henning Kamp }
4176f4f00f1SPoul-Henning Kamp 
418b4a4f93cSPoul-Henning Kamp static int
419b4a4f93cSPoul-Henning Kamp mdstart_malloc(struct md_s *sc, struct bio *bp)
42000a6a3c6SPoul-Henning Kamp {
421c6517568SPoul-Henning Kamp 	int i, error;
422c6517568SPoul-Henning Kamp 	u_char *dst;
423b830359bSPawel Jakub Dawidek 	off_t secno, nsec, uc;
424c6517568SPoul-Henning Kamp 	uintptr_t sp, osp;
42500a6a3c6SPoul-Henning Kamp 
4265541f25eSPawel Jakub Dawidek 	switch (bp->bio_cmd) {
4275541f25eSPawel Jakub Dawidek 	case BIO_READ:
4285541f25eSPawel Jakub Dawidek 	case BIO_WRITE:
4295541f25eSPawel Jakub Dawidek 	case BIO_DELETE:
4305541f25eSPawel Jakub Dawidek 		break;
4315541f25eSPawel Jakub Dawidek 	default:
4325541f25eSPawel Jakub Dawidek 		return (EOPNOTSUPP);
4335541f25eSPawel Jakub Dawidek 	}
4345541f25eSPawel Jakub Dawidek 
435b830359bSPawel Jakub Dawidek 	nsec = bp->bio_length / sc->sectorsize;
436b830359bSPawel Jakub Dawidek 	secno = bp->bio_offset / sc->sectorsize;
4378177437dSPoul-Henning Kamp 	dst = bp->bio_data;
438c6517568SPoul-Henning Kamp 	error = 0;
43900a6a3c6SPoul-Henning Kamp 	while (nsec--) {
440fde2a2e4SPoul-Henning Kamp 		osp = s_read(sc->indir, secno);
4418177437dSPoul-Henning Kamp 		if (bp->bio_cmd == BIO_DELETE) {
442fde2a2e4SPoul-Henning Kamp 			if (osp != 0)
443fde2a2e4SPoul-Henning Kamp 				error = s_write(sc->indir, secno, 0);
4448177437dSPoul-Henning Kamp 		} else if (bp->bio_cmd == BIO_READ) {
445fde2a2e4SPoul-Henning Kamp 			if (osp == 0)
446b830359bSPawel Jakub Dawidek 				bzero(dst, sc->sectorsize);
447fde2a2e4SPoul-Henning Kamp 			else if (osp <= 255)
448dbb95048SMarcel Moolenaar 				memset(dst, osp, sc->sectorsize);
449dbb95048SMarcel Moolenaar 			else {
450b830359bSPawel Jakub Dawidek 				bcopy((void *)osp, dst, sc->sectorsize);
451dbb95048SMarcel Moolenaar 				cpu_flush_dcache(dst, sc->sectorsize);
452dbb95048SMarcel Moolenaar 			}
453fde2a2e4SPoul-Henning Kamp 			osp = 0;
454c6517568SPoul-Henning Kamp 		} else if (bp->bio_cmd == BIO_WRITE) {
4558f8def9eSPoul-Henning Kamp 			if (sc->flags & MD_COMPRESS) {
45600a6a3c6SPoul-Henning Kamp 				uc = dst[0];
457b830359bSPawel Jakub Dawidek 				for (i = 1; i < sc->sectorsize; i++)
45800a6a3c6SPoul-Henning Kamp 					if (dst[i] != uc)
45900a6a3c6SPoul-Henning Kamp 						break;
4608f8def9eSPoul-Henning Kamp 			} else {
4618f8def9eSPoul-Henning Kamp 				i = 0;
4628f8def9eSPoul-Henning Kamp 				uc = 0;
4638f8def9eSPoul-Henning Kamp 			}
464b830359bSPawel Jakub Dawidek 			if (i == sc->sectorsize) {
465fde2a2e4SPoul-Henning Kamp 				if (osp != uc)
466fde2a2e4SPoul-Henning Kamp 					error = s_write(sc->indir, secno, uc);
46700a6a3c6SPoul-Henning Kamp 			} else {
468fde2a2e4SPoul-Henning Kamp 				if (osp <= 255) {
469b830359bSPawel Jakub Dawidek 					sp = (uintptr_t)uma_zalloc(sc->uma,
470c44d423eSKonstantin Belousov 					    md_malloc_wait ? M_WAITOK :
471b830359bSPawel Jakub Dawidek 					    M_NOWAIT);
472c6517568SPoul-Henning Kamp 					if (sp == 0) {
473fde2a2e4SPoul-Henning Kamp 						error = ENOSPC;
474fde2a2e4SPoul-Henning Kamp 						break;
475fde2a2e4SPoul-Henning Kamp 					}
476b830359bSPawel Jakub Dawidek 					bcopy(dst, (void *)sp, sc->sectorsize);
477fde2a2e4SPoul-Henning Kamp 					error = s_write(sc->indir, secno, sp);
478c6517568SPoul-Henning Kamp 				} else {
479b830359bSPawel Jakub Dawidek 					bcopy(dst, (void *)osp, sc->sectorsize);
480fde2a2e4SPoul-Henning Kamp 					osp = 0;
48100a6a3c6SPoul-Henning Kamp 				}
48200a6a3c6SPoul-Henning Kamp 			}
483c6517568SPoul-Henning Kamp 		} else {
484c6517568SPoul-Henning Kamp 			error = EOPNOTSUPP;
485c6517568SPoul-Henning Kamp 		}
486c6517568SPoul-Henning Kamp 		if (osp > 255)
487f43b2bacSPoul-Henning Kamp 			uma_zfree(sc->uma, (void*)osp);
488e3ed29a7SPawel Jakub Dawidek 		if (error != 0)
489c6517568SPoul-Henning Kamp 			break;
49000a6a3c6SPoul-Henning Kamp 		secno++;
491b830359bSPawel Jakub Dawidek 		dst += sc->sectorsize;
49200a6a3c6SPoul-Henning Kamp 	}
4938177437dSPoul-Henning Kamp 	bp->bio_resid = 0;
494c6517568SPoul-Henning Kamp 	return (error);
49500a6a3c6SPoul-Henning Kamp }
49600a6a3c6SPoul-Henning Kamp 
497b4a4f93cSPoul-Henning Kamp static int
498b4a4f93cSPoul-Henning Kamp mdstart_preload(struct md_s *sc, struct bio *bp)
49971e4fff8SPoul-Henning Kamp {
50071e4fff8SPoul-Henning Kamp 
501a8a58d03SPawel Jakub Dawidek 	switch (bp->bio_cmd) {
502a8a58d03SPawel Jakub Dawidek 	case BIO_READ:
503a8a58d03SPawel Jakub Dawidek 		bcopy(sc->pl_ptr + bp->bio_offset, bp->bio_data,
504a8a58d03SPawel Jakub Dawidek 		    bp->bio_length);
505dbb95048SMarcel Moolenaar 		cpu_flush_dcache(bp->bio_data, bp->bio_length);
506a8a58d03SPawel Jakub Dawidek 		break;
507a8a58d03SPawel Jakub Dawidek 	case BIO_WRITE:
508a8a58d03SPawel Jakub Dawidek 		bcopy(bp->bio_data, sc->pl_ptr + bp->bio_offset,
509a8a58d03SPawel Jakub Dawidek 		    bp->bio_length);
510a8a58d03SPawel Jakub Dawidek 		break;
51171e4fff8SPoul-Henning Kamp 	}
5128177437dSPoul-Henning Kamp 	bp->bio_resid = 0;
513b4a4f93cSPoul-Henning Kamp 	return (0);
51471e4fff8SPoul-Henning Kamp }
51571e4fff8SPoul-Henning Kamp 
516b4a4f93cSPoul-Henning Kamp static int
517b4a4f93cSPoul-Henning Kamp mdstart_vnode(struct md_s *sc, struct bio *bp)
5188f8def9eSPoul-Henning Kamp {
5195050aa86SKonstantin Belousov 	int error;
5208f8def9eSPoul-Henning Kamp 	struct uio auio;
5218f8def9eSPoul-Henning Kamp 	struct iovec aiov;
5228f8def9eSPoul-Henning Kamp 	struct mount *mp;
5235541f25eSPawel Jakub Dawidek 	struct vnode *vp;
5245541f25eSPawel Jakub Dawidek 	struct thread *td;
5250abd21bdSDag-Erling Smørgrav 	off_t end, zerosize;
5265541f25eSPawel Jakub Dawidek 
5275541f25eSPawel Jakub Dawidek 	switch (bp->bio_cmd) {
5285541f25eSPawel Jakub Dawidek 	case BIO_READ:
5295541f25eSPawel Jakub Dawidek 	case BIO_WRITE:
5300abd21bdSDag-Erling Smørgrav 	case BIO_DELETE:
5315541f25eSPawel Jakub Dawidek 	case BIO_FLUSH:
5325541f25eSPawel Jakub Dawidek 		break;
5335541f25eSPawel Jakub Dawidek 	default:
5345541f25eSPawel Jakub Dawidek 		return (EOPNOTSUPP);
5355541f25eSPawel Jakub Dawidek 	}
5365541f25eSPawel Jakub Dawidek 
5375541f25eSPawel Jakub Dawidek 	td = curthread;
5385541f25eSPawel Jakub Dawidek 	vp = sc->vnode;
5398f8def9eSPoul-Henning Kamp 
5408f8def9eSPoul-Henning Kamp 	/*
5418f8def9eSPoul-Henning Kamp 	 * VNODE I/O
5428f8def9eSPoul-Henning Kamp 	 *
5438f8def9eSPoul-Henning Kamp 	 * If an error occurs, we set BIO_ERROR but we do not set
5448f8def9eSPoul-Henning Kamp 	 * B_INVAL because (for a write anyway), the buffer is
5458f8def9eSPoul-Henning Kamp 	 * still valid.
5468f8def9eSPoul-Henning Kamp 	 */
5478f8def9eSPoul-Henning Kamp 
5485541f25eSPawel Jakub Dawidek 	if (bp->bio_cmd == BIO_FLUSH) {
5495541f25eSPawel Jakub Dawidek 		(void) vn_start_write(vp, &mp, V_WAIT);
550cb05b60aSAttilio Rao 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
5515541f25eSPawel Jakub Dawidek 		error = VOP_FSYNC(vp, MNT_WAIT, td);
55222db15c0SAttilio Rao 		VOP_UNLOCK(vp, 0);
5535541f25eSPawel Jakub Dawidek 		vn_finished_write(mp);
5545541f25eSPawel Jakub Dawidek 		return (error);
5555541f25eSPawel Jakub Dawidek 	}
5565541f25eSPawel Jakub Dawidek 
5578f8def9eSPoul-Henning Kamp 	bzero(&auio, sizeof(auio));
5588f8def9eSPoul-Henning Kamp 
5590abd21bdSDag-Erling Smørgrav 	/*
5600abd21bdSDag-Erling Smørgrav 	 * Special case for BIO_DELETE.  On the surface, this is very
5610abd21bdSDag-Erling Smørgrav 	 * similar to BIO_WRITE, except that we write from our own
5620abd21bdSDag-Erling Smørgrav 	 * fixed-length buffer, so we have to loop.  The net result is
5630abd21bdSDag-Erling Smørgrav 	 * that the two cases end up having very little in common.
5640abd21bdSDag-Erling Smørgrav 	 */
5650abd21bdSDag-Erling Smørgrav 	if (bp->bio_cmd == BIO_DELETE) {
56689cb2a19SMatthew D Fleming 		zerosize = ZERO_REGION_SIZE -
56789cb2a19SMatthew D Fleming 		    (ZERO_REGION_SIZE % sc->sectorsize);
5680abd21bdSDag-Erling Smørgrav 		auio.uio_iov = &aiov;
5690abd21bdSDag-Erling Smørgrav 		auio.uio_iovcnt = 1;
5700abd21bdSDag-Erling Smørgrav 		auio.uio_offset = (vm_ooffset_t)bp->bio_offset;
5710abd21bdSDag-Erling Smørgrav 		auio.uio_segflg = UIO_SYSSPACE;
5720abd21bdSDag-Erling Smørgrav 		auio.uio_rw = UIO_WRITE;
5730abd21bdSDag-Erling Smørgrav 		auio.uio_td = td;
5740abd21bdSDag-Erling Smørgrav 		end = bp->bio_offset + bp->bio_length;
5750abd21bdSDag-Erling Smørgrav 		(void) vn_start_write(vp, &mp, V_WAIT);
5760abd21bdSDag-Erling Smørgrav 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
5770abd21bdSDag-Erling Smørgrav 		error = 0;
5780abd21bdSDag-Erling Smørgrav 		while (auio.uio_offset < end) {
57989cb2a19SMatthew D Fleming 			aiov.iov_base = __DECONST(void *, zero_region);
5800abd21bdSDag-Erling Smørgrav 			aiov.iov_len = end - auio.uio_offset;
5810abd21bdSDag-Erling Smørgrav 			if (aiov.iov_len > zerosize)
5820abd21bdSDag-Erling Smørgrav 				aiov.iov_len = zerosize;
5830abd21bdSDag-Erling Smørgrav 			auio.uio_resid = aiov.iov_len;
5840abd21bdSDag-Erling Smørgrav 			error = VOP_WRITE(vp, &auio,
5850abd21bdSDag-Erling Smørgrav 			    sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred);
5860abd21bdSDag-Erling Smørgrav 			if (error != 0)
5870abd21bdSDag-Erling Smørgrav 				break;
5880abd21bdSDag-Erling Smørgrav 		}
5890abd21bdSDag-Erling Smørgrav 		VOP_UNLOCK(vp, 0);
5900abd21bdSDag-Erling Smørgrav 		vn_finished_write(mp);
5910abd21bdSDag-Erling Smørgrav 		bp->bio_resid = end - auio.uio_offset;
5920abd21bdSDag-Erling Smørgrav 		return (error);
5930abd21bdSDag-Erling Smørgrav 	}
5940abd21bdSDag-Erling Smørgrav 
5958f8def9eSPoul-Henning Kamp 	aiov.iov_base = bp->bio_data;
596a8a58d03SPawel Jakub Dawidek 	aiov.iov_len = bp->bio_length;
5978f8def9eSPoul-Henning Kamp 	auio.uio_iov = &aiov;
5988f8def9eSPoul-Henning Kamp 	auio.uio_iovcnt = 1;
599a8a58d03SPawel Jakub Dawidek 	auio.uio_offset = (vm_ooffset_t)bp->bio_offset;
6008f8def9eSPoul-Henning Kamp 	auio.uio_segflg = UIO_SYSSPACE;
6018f8def9eSPoul-Henning Kamp 	if (bp->bio_cmd == BIO_READ)
6028f8def9eSPoul-Henning Kamp 		auio.uio_rw = UIO_READ;
6038e28326aSPoul-Henning Kamp 	else if (bp->bio_cmd == BIO_WRITE)
6048f8def9eSPoul-Henning Kamp 		auio.uio_rw = UIO_WRITE;
6058e28326aSPoul-Henning Kamp 	else
6068e28326aSPoul-Henning Kamp 		panic("wrong BIO_OP in mdstart_vnode");
607a8a58d03SPawel Jakub Dawidek 	auio.uio_resid = bp->bio_length;
6085541f25eSPawel Jakub Dawidek 	auio.uio_td = td;
6097e76bb56SMatthew Dillon 	/*
6107e76bb56SMatthew Dillon 	 * When reading set IO_DIRECT to try to avoid double-caching
61117a13919SPoul-Henning Kamp 	 * the data.  When writing IO_DIRECT is not optimal.
6127e76bb56SMatthew Dillon 	 */
6138f8def9eSPoul-Henning Kamp 	if (bp->bio_cmd == BIO_READ) {
614cb05b60aSAttilio Rao 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
6155541f25eSPawel Jakub Dawidek 		error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred);
61622db15c0SAttilio Rao 		VOP_UNLOCK(vp, 0);
6178f8def9eSPoul-Henning Kamp 	} else {
6185541f25eSPawel Jakub Dawidek 		(void) vn_start_write(vp, &mp, V_WAIT);
619cb05b60aSAttilio Rao 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
6205541f25eSPawel Jakub Dawidek 		error = VOP_WRITE(vp, &auio, sc->flags & MD_ASYNC ? 0 : IO_SYNC,
6215541f25eSPawel Jakub Dawidek 		    sc->cred);
62222db15c0SAttilio Rao 		VOP_UNLOCK(vp, 0);
6238f8def9eSPoul-Henning Kamp 		vn_finished_write(mp);
6248f8def9eSPoul-Henning Kamp 	}
6258f8def9eSPoul-Henning Kamp 	bp->bio_resid = auio.uio_resid;
626b4a4f93cSPoul-Henning Kamp 	return (error);
6278f8def9eSPoul-Henning Kamp }
6288f8def9eSPoul-Henning Kamp 
629b4a4f93cSPoul-Henning Kamp static int
630b4a4f93cSPoul-Henning Kamp mdstart_swap(struct md_s *sc, struct bio *bp)
6318f8def9eSPoul-Henning Kamp {
6327cd53fddSAlan Cox 	struct sf_buf *sf;
6336ab0a0aeSPawel Jakub Dawidek 	int rv, offs, len, lastend;
6346ab0a0aeSPawel Jakub Dawidek 	vm_pindex_t i, lastp;
6358e28326aSPoul-Henning Kamp 	vm_page_t m;
6368e28326aSPoul-Henning Kamp 	u_char *p;
6378f8def9eSPoul-Henning Kamp 
6385541f25eSPawel Jakub Dawidek 	switch (bp->bio_cmd) {
6395541f25eSPawel Jakub Dawidek 	case BIO_READ:
6405541f25eSPawel Jakub Dawidek 	case BIO_WRITE:
6415541f25eSPawel Jakub Dawidek 	case BIO_DELETE:
6425541f25eSPawel Jakub Dawidek 		break;
6435541f25eSPawel Jakub Dawidek 	default:
6445541f25eSPawel Jakub Dawidek 		return (EOPNOTSUPP);
6455541f25eSPawel Jakub Dawidek 	}
6465541f25eSPawel Jakub Dawidek 
6478e28326aSPoul-Henning Kamp 	p = bp->bio_data;
648e07113d6SColin Percival 
649e07113d6SColin Percival 	/*
6506c3cd0e2SMaxim Konovalov 	 * offs is the offset at which to start operating on the
651e07113d6SColin Percival 	 * next (ie, first) page.  lastp is the last page on
652e07113d6SColin Percival 	 * which we're going to operate.  lastend is the ending
653e07113d6SColin Percival 	 * position within that last page (ie, PAGE_SIZE if
654e07113d6SColin Percival 	 * we're operating on complete aligned pages).
655e07113d6SColin Percival 	 */
656e07113d6SColin Percival 	offs = bp->bio_offset % PAGE_SIZE;
657e07113d6SColin Percival 	lastp = (bp->bio_offset + bp->bio_length - 1) / PAGE_SIZE;
658e07113d6SColin Percival 	lastend = (bp->bio_offset + bp->bio_length - 1) % PAGE_SIZE + 1;
659e07113d6SColin Percival 
660812851b6SBrian Feldman 	rv = VM_PAGER_OK;
6618e28326aSPoul-Henning Kamp 	VM_OBJECT_LOCK(sc->object);
6628e28326aSPoul-Henning Kamp 	vm_object_pip_add(sc->object, 1);
663e07113d6SColin Percival 	for (i = bp->bio_offset / PAGE_SIZE; i <= lastp; i++) {
664e07113d6SColin Percival 		len = ((i == lastp) ? lastend : PAGE_SIZE) - offs;
665e07113d6SColin Percival 
666e07113d6SColin Percival 		m = vm_page_grab(sc->object, i,
6678e28326aSPoul-Henning Kamp 		    VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
6687cd53fddSAlan Cox 		VM_OBJECT_UNLOCK(sc->object);
669e340fc60SAlan Cox 		sched_pin();
670e340fc60SAlan Cox 		sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
6717cd53fddSAlan Cox 		VM_OBJECT_LOCK(sc->object);
6728e28326aSPoul-Henning Kamp 		if (bp->bio_cmd == BIO_READ) {
67307be617fSAlan Cox 			if (m->valid != VM_PAGE_BITS_ALL)
67407be617fSAlan Cox 				rv = vm_pager_get_pages(sc->object, &m, 1, 0);
675812851b6SBrian Feldman 			if (rv == VM_PAGER_ERROR) {
676812851b6SBrian Feldman 				sf_buf_free(sf);
677e340fc60SAlan Cox 				sched_unpin();
678812851b6SBrian Feldman 				vm_page_wakeup(m);
679812851b6SBrian Feldman 				break;
680e9f581baSKonstantin Belousov 			} else if (rv == VM_PAGER_FAIL) {
681e9f581baSKonstantin Belousov 				/*
682e9f581baSKonstantin Belousov 				 * Pager does not have the page.  Zero
683e9f581baSKonstantin Belousov 				 * the allocated page, and mark it as
684e9f581baSKonstantin Belousov 				 * valid. Do not set dirty, the page
685e9f581baSKonstantin Belousov 				 * can be recreated if thrown out.
686e9f581baSKonstantin Belousov 				 */
687e9f581baSKonstantin Belousov 				bzero((void *)sf_buf_kva(sf), PAGE_SIZE);
688e9f581baSKonstantin Belousov 				m->valid = VM_PAGE_BITS_ALL;
689812851b6SBrian Feldman 			}
6907cd53fddSAlan Cox 			bcopy((void *)(sf_buf_kva(sf) + offs), p, len);
691dbb95048SMarcel Moolenaar 			cpu_flush_dcache(p, len);
6928e28326aSPoul-Henning Kamp 		} else if (bp->bio_cmd == BIO_WRITE) {
69307be617fSAlan Cox 			if (len != PAGE_SIZE && m->valid != VM_PAGE_BITS_ALL)
69407be617fSAlan Cox 				rv = vm_pager_get_pages(sc->object, &m, 1, 0);
695812851b6SBrian Feldman 			if (rv == VM_PAGER_ERROR) {
696812851b6SBrian Feldman 				sf_buf_free(sf);
697e340fc60SAlan Cox 				sched_unpin();
698812851b6SBrian Feldman 				vm_page_wakeup(m);
699812851b6SBrian Feldman 				break;
700812851b6SBrian Feldman 			}
7017cd53fddSAlan Cox 			bcopy(p, (void *)(sf_buf_kva(sf) + offs), len);
7028e28326aSPoul-Henning Kamp 			m->valid = VM_PAGE_BITS_ALL;
7038e28326aSPoul-Henning Kamp 		} else if (bp->bio_cmd == BIO_DELETE) {
70407be617fSAlan Cox 			if (len != PAGE_SIZE && m->valid != VM_PAGE_BITS_ALL)
70507be617fSAlan Cox 				rv = vm_pager_get_pages(sc->object, &m, 1, 0);
706812851b6SBrian Feldman 			if (rv == VM_PAGER_ERROR) {
707812851b6SBrian Feldman 				sf_buf_free(sf);
708e340fc60SAlan Cox 				sched_unpin();
709812851b6SBrian Feldman 				vm_page_wakeup(m);
710812851b6SBrian Feldman 				break;
711812851b6SBrian Feldman 			}
7124a13a769SKonstantin Belousov 			if (len != PAGE_SIZE) {
7137cd53fddSAlan Cox 				bzero((void *)(sf_buf_kva(sf) + offs), len);
7144a13a769SKonstantin Belousov 				vm_page_clear_dirty(m, offs, len);
7158e28326aSPoul-Henning Kamp 				m->valid = VM_PAGE_BITS_ALL;
7164a13a769SKonstantin Belousov 			} else
7174a13a769SKonstantin Belousov 				vm_pager_page_unswapped(m);
7188e28326aSPoul-Henning Kamp 		}
7197cd53fddSAlan Cox 		sf_buf_free(sf);
720e340fc60SAlan Cox 		sched_unpin();
7218e28326aSPoul-Henning Kamp 		vm_page_wakeup(m);
722fc0c3802SKonstantin Belousov 		vm_page_lock(m);
7234a13a769SKonstantin Belousov 		if (bp->bio_cmd == BIO_DELETE && len == PAGE_SIZE)
7244a13a769SKonstantin Belousov 			vm_page_free(m);
7254a13a769SKonstantin Belousov 		else
7268e28326aSPoul-Henning Kamp 			vm_page_activate(m);
727ecd5dd95SAlan Cox 		vm_page_unlock(m);
72807be617fSAlan Cox 		if (bp->bio_cmd == BIO_WRITE)
7298e28326aSPoul-Henning Kamp 			vm_page_dirty(m);
730e07113d6SColin Percival 
731e07113d6SColin Percival 		/* Actions on further pages start at offset 0 */
732e07113d6SColin Percival 		p += PAGE_SIZE - offs;
733e07113d6SColin Percival 		offs = 0;
7348e28326aSPoul-Henning Kamp 	}
7358e28326aSPoul-Henning Kamp 	vm_object_pip_subtract(sc->object, 1);
7368e28326aSPoul-Henning Kamp 	VM_OBJECT_UNLOCK(sc->object);
737812851b6SBrian Feldman 	return (rv != VM_PAGER_ERROR ? 0 : ENOSPC);
7388e28326aSPoul-Henning Kamp }
7398f8def9eSPoul-Henning Kamp 
7408f8def9eSPoul-Henning Kamp static void
7415c97ca54SIan Dowse md_kthread(void *arg)
7425c97ca54SIan Dowse {
7435c97ca54SIan Dowse 	struct md_s *sc;
7445c97ca54SIan Dowse 	struct bio *bp;
745a08d2e7fSJohn Baldwin 	int error;
7465c97ca54SIan Dowse 
7475c97ca54SIan Dowse 	sc = arg;
748982d11f8SJeff Roberson 	thread_lock(curthread);
74963710c4dSJohn Baldwin 	sched_prio(curthread, PRIBIO);
750982d11f8SJeff Roberson 	thread_unlock(curthread);
7513b7b5496SKonstantin Belousov 	if (sc->type == MD_VNODE)
7523b7b5496SKonstantin Belousov 		curthread->td_pflags |= TDP_NORUNNINGBUF;
7535c97ca54SIan Dowse 
754b4a4f93cSPoul-Henning Kamp 	for (;;) {
755a08d2e7fSJohn Baldwin 		mtx_lock(&sc->queue_mtx);
7565c97ca54SIan Dowse 		if (sc->flags & MD_SHUTDOWN) {
757a08d2e7fSJohn Baldwin 			sc->flags |= MD_EXITING;
758a08d2e7fSJohn Baldwin 			mtx_unlock(&sc->queue_mtx);
7593745c395SJulian Elischer 			kproc_exit(0);
7605c97ca54SIan Dowse 		}
7619b00ca19SPoul-Henning Kamp 		bp = bioq_takefirst(&sc->bio_queue);
7629b00ca19SPoul-Henning Kamp 		if (!bp) {
7630f8500a5SPoul-Henning Kamp 			msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0);
7645c97ca54SIan Dowse 			continue;
7655c97ca54SIan Dowse 		}
7660f8500a5SPoul-Henning Kamp 		mtx_unlock(&sc->queue_mtx);
7674e8bfe14SPoul-Henning Kamp 		if (bp->bio_cmd == BIO_GETATTR) {
768d91e813cSKonstantin Belousov 			if ((sc->fwsectors && sc->fwheads &&
7694e8bfe14SPoul-Henning Kamp 			    (g_handleattr_int(bp, "GEOM::fwsectors",
7704e8bfe14SPoul-Henning Kamp 			    sc->fwsectors) ||
7714e8bfe14SPoul-Henning Kamp 			    g_handleattr_int(bp, "GEOM::fwheads",
772d91e813cSKonstantin Belousov 			    sc->fwheads))) ||
773d91e813cSKonstantin Belousov 			    g_handleattr_int(bp, "GEOM::candelete", 1))
7744e8bfe14SPoul-Henning Kamp 				error = -1;
7754e8bfe14SPoul-Henning Kamp 			else
7764e8bfe14SPoul-Henning Kamp 				error = EOPNOTSUPP;
7774e8bfe14SPoul-Henning Kamp 		} else {
7789b00ca19SPoul-Henning Kamp 			error = sc->start(sc, bp);
7794e8bfe14SPoul-Henning Kamp 		}
780b4a4f93cSPoul-Henning Kamp 
7816f4f00f1SPoul-Henning Kamp 		if (error != -1) {
7826f4f00f1SPoul-Henning Kamp 			bp->bio_completed = bp->bio_length;
783a03be42dSMaxim Sobolev 			if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE))
784a03be42dSMaxim Sobolev 				devstat_end_transaction_bio(sc->devstat, bp);
78596410b95SKonstantin Belousov 			g_io_deliver(bp, error);
786b4a4f93cSPoul-Henning Kamp 		}
7878f8def9eSPoul-Henning Kamp 	}
78826d48b40SPoul-Henning Kamp }
7898f8def9eSPoul-Henning Kamp 
7908f8def9eSPoul-Henning Kamp static struct md_s *
7918f8def9eSPoul-Henning Kamp mdfind(int unit)
7928f8def9eSPoul-Henning Kamp {
7938f8def9eSPoul-Henning Kamp 	struct md_s *sc;
7948f8def9eSPoul-Henning Kamp 
7953f54a085SPoul-Henning Kamp 	LIST_FOREACH(sc, &md_softc_list, list) {
7963f54a085SPoul-Henning Kamp 		if (sc->unit == unit)
7978f8def9eSPoul-Henning Kamp 			break;
7988f8def9eSPoul-Henning Kamp 	}
7998f8def9eSPoul-Henning Kamp 	return (sc);
8008f8def9eSPoul-Henning Kamp }
8018f8def9eSPoul-Henning Kamp 
8028f8def9eSPoul-Henning Kamp static struct md_s *
803947fc8deSPoul-Henning Kamp mdnew(int unit, int *errp, enum md_types type)
8048f8def9eSPoul-Henning Kamp {
805f4e7c5a8SJaakko Heinonen 	struct md_s *sc;
806f4e7c5a8SJaakko Heinonen 	int error;
8078f8def9eSPoul-Henning Kamp 
8089b00ca19SPoul-Henning Kamp 	*errp = 0;
809f4e7c5a8SJaakko Heinonen 	if (unit == -1)
810f4e7c5a8SJaakko Heinonen 		unit = alloc_unr(md_uh);
811f4e7c5a8SJaakko Heinonen 	else
812f4e7c5a8SJaakko Heinonen 		unit = alloc_unr_specific(md_uh, unit);
813f4e7c5a8SJaakko Heinonen 
814f4e7c5a8SJaakko Heinonen 	if (unit == -1) {
8157ee3c044SPawel Jakub Dawidek 		*errp = EBUSY;
8163f54a085SPoul-Henning Kamp 		return (NULL);
8173f54a085SPoul-Henning Kamp 	}
818f4e7c5a8SJaakko Heinonen 
8199b00ca19SPoul-Henning Kamp 	sc = (struct md_s *)malloc(sizeof *sc, M_MD, M_WAITOK | M_ZERO);
820947fc8deSPoul-Henning Kamp 	sc->type = type;
8219b00ca19SPoul-Henning Kamp 	bioq_init(&sc->bio_queue);
8229b00ca19SPoul-Henning Kamp 	mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF);
8233f54a085SPoul-Henning Kamp 	sc->unit = unit;
824f43b2bacSPoul-Henning Kamp 	sprintf(sc->name, "md%d", unit);
8257ee3c044SPawel Jakub Dawidek 	LIST_INSERT_HEAD(&md_softc_list, sc, list);
8263745c395SJulian Elischer 	error = kproc_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name);
8279b00ca19SPoul-Henning Kamp 	if (error == 0)
8289b00ca19SPoul-Henning Kamp 		return (sc);
8297ee3c044SPawel Jakub Dawidek 	LIST_REMOVE(sc, list);
8307ee3c044SPawel Jakub Dawidek 	mtx_destroy(&sc->queue_mtx);
831f4e7c5a8SJaakko Heinonen 	free_unr(md_uh, sc->unit);
8325c97ca54SIan Dowse 	free(sc, M_MD);
8337ee3c044SPawel Jakub Dawidek 	*errp = error;
8345c97ca54SIan Dowse 	return (NULL);
8355c97ca54SIan Dowse }
8368f8def9eSPoul-Henning Kamp 
8378f8def9eSPoul-Henning Kamp static void
8388f8def9eSPoul-Henning Kamp mdinit(struct md_s *sc)
8398f8def9eSPoul-Henning Kamp {
8406f4f00f1SPoul-Henning Kamp 	struct g_geom *gp;
8416f4f00f1SPoul-Henning Kamp 	struct g_provider *pp;
8426f4f00f1SPoul-Henning Kamp 
8436f4f00f1SPoul-Henning Kamp 	g_topology_lock();
8446f4f00f1SPoul-Henning Kamp 	gp = g_new_geomf(&g_md_class, "md%d", sc->unit);
8456f4f00f1SPoul-Henning Kamp 	gp->softc = sc;
8466f4f00f1SPoul-Henning Kamp 	pp = g_new_providerf(gp, "md%d", sc->unit);
847b830359bSPawel Jakub Dawidek 	pp->mediasize = sc->mediasize;
848b830359bSPawel Jakub Dawidek 	pp->sectorsize = sc->sectorsize;
8496f4f00f1SPoul-Henning Kamp 	sc->gp = gp;
8506f4f00f1SPoul-Henning Kamp 	sc->pp = pp;
8516f4f00f1SPoul-Henning Kamp 	g_error_provider(pp, 0);
8526f4f00f1SPoul-Henning Kamp 	g_topology_unlock();
853a03be42dSMaxim Sobolev 	sc->devstat = devstat_new_entry("md", sc->unit, sc->sectorsize,
854a03be42dSMaxim Sobolev 	    DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX);
8556f4f00f1SPoul-Henning Kamp }
85671e4fff8SPoul-Henning Kamp 
8578f8def9eSPoul-Henning Kamp static int
858b830359bSPawel Jakub Dawidek mdcreate_malloc(struct md_s *sc, struct md_ioctl *mdio)
85995f1a897SPoul-Henning Kamp {
860c6517568SPoul-Henning Kamp 	uintptr_t sp;
861c6517568SPoul-Henning Kamp 	int error;
862b830359bSPawel Jakub Dawidek 	off_t u;
86395f1a897SPoul-Henning Kamp 
864c6517568SPoul-Henning Kamp 	error = 0;
8658f8def9eSPoul-Henning Kamp 	if (mdio->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE))
8668f8def9eSPoul-Henning Kamp 		return (EINVAL);
867b830359bSPawel Jakub Dawidek 	if (mdio->md_sectorsize != 0 && !powerof2(mdio->md_sectorsize))
868ebe789d6SPoul-Henning Kamp 		return (EINVAL);
8698f8def9eSPoul-Henning Kamp 	/* Compression doesn't make sense if we have reserved space */
8708f8def9eSPoul-Henning Kamp 	if (mdio->md_options & MD_RESERVE)
8718f8def9eSPoul-Henning Kamp 		mdio->md_options &= ~MD_COMPRESS;
8724e8bfe14SPoul-Henning Kamp 	if (mdio->md_fwsectors != 0)
8734e8bfe14SPoul-Henning Kamp 		sc->fwsectors = mdio->md_fwsectors;
8744e8bfe14SPoul-Henning Kamp 	if (mdio->md_fwheads != 0)
8754e8bfe14SPoul-Henning Kamp 		sc->fwheads = mdio->md_fwheads;
87626a0ee75SDima Dorfman 	sc->flags = mdio->md_options & (MD_COMPRESS | MD_FORCE);
877b830359bSPawel Jakub Dawidek 	sc->indir = dimension(sc->mediasize / sc->sectorsize);
878b830359bSPawel Jakub Dawidek 	sc->uma = uma_zcreate(sc->name, sc->sectorsize, NULL, NULL, NULL, NULL,
879b830359bSPawel Jakub Dawidek 	    0x1ff, 0);
88096b6a55fSPoul-Henning Kamp 	if (mdio->md_options & MD_RESERVE) {
881b830359bSPawel Jakub Dawidek 		off_t nsectors;
882b830359bSPawel Jakub Dawidek 
883b830359bSPawel Jakub Dawidek 		nsectors = sc->mediasize / sc->sectorsize;
884b830359bSPawel Jakub Dawidek 		for (u = 0; u < nsectors; u++) {
885007777f1SKonstantin Belousov 			sp = (uintptr_t)uma_zalloc(sc->uma, (md_malloc_wait ?
886007777f1SKonstantin Belousov 			    M_WAITOK : M_NOWAIT) | M_ZERO);
887c6517568SPoul-Henning Kamp 			if (sp != 0)
888fde2a2e4SPoul-Henning Kamp 				error = s_write(sc->indir, u, sp);
889c6517568SPoul-Henning Kamp 			else
890c6517568SPoul-Henning Kamp 				error = ENOMEM;
891b830359bSPawel Jakub Dawidek 			if (error != 0)
892c6517568SPoul-Henning Kamp 				break;
8938f8def9eSPoul-Henning Kamp 		}
894c6517568SPoul-Henning Kamp 	}
895c6517568SPoul-Henning Kamp 	return (error);
89600a6a3c6SPoul-Henning Kamp }
89700a6a3c6SPoul-Henning Kamp 
8983f54a085SPoul-Henning Kamp 
8998f8def9eSPoul-Henning Kamp static int
9008f8def9eSPoul-Henning Kamp mdsetcred(struct md_s *sc, struct ucred *cred)
9018f8def9eSPoul-Henning Kamp {
9028f8def9eSPoul-Henning Kamp 	char *tmpbuf;
9038f8def9eSPoul-Henning Kamp 	int error = 0;
9048f8def9eSPoul-Henning Kamp 
9053f54a085SPoul-Henning Kamp 	/*
9068f8def9eSPoul-Henning Kamp 	 * Set credits in our softc
9073f54a085SPoul-Henning Kamp 	 */
9088f8def9eSPoul-Henning Kamp 
9098f8def9eSPoul-Henning Kamp 	if (sc->cred)
9108f8def9eSPoul-Henning Kamp 		crfree(sc->cred);
911bd78ceceSJohn Baldwin 	sc->cred = crhold(cred);
9128f8def9eSPoul-Henning Kamp 
9138f8def9eSPoul-Henning Kamp 	/*
9148f8def9eSPoul-Henning Kamp 	 * Horrible kludge to establish credentials for NFS  XXX.
9158f8def9eSPoul-Henning Kamp 	 */
9168f8def9eSPoul-Henning Kamp 
9178f8def9eSPoul-Henning Kamp 	if (sc->vnode) {
9188f8def9eSPoul-Henning Kamp 		struct uio auio;
9198f8def9eSPoul-Henning Kamp 		struct iovec aiov;
9208f8def9eSPoul-Henning Kamp 
921b830359bSPawel Jakub Dawidek 		tmpbuf = malloc(sc->sectorsize, M_TEMP, M_WAITOK);
9228f8def9eSPoul-Henning Kamp 		bzero(&auio, sizeof(auio));
9238f8def9eSPoul-Henning Kamp 
9248f8def9eSPoul-Henning Kamp 		aiov.iov_base = tmpbuf;
925b830359bSPawel Jakub Dawidek 		aiov.iov_len = sc->sectorsize;
9268f8def9eSPoul-Henning Kamp 		auio.uio_iov = &aiov;
9278f8def9eSPoul-Henning Kamp 		auio.uio_iovcnt = 1;
9288f8def9eSPoul-Henning Kamp 		auio.uio_offset = 0;
9298f8def9eSPoul-Henning Kamp 		auio.uio_rw = UIO_READ;
9308f8def9eSPoul-Henning Kamp 		auio.uio_segflg = UIO_SYSSPACE;
9318f8def9eSPoul-Henning Kamp 		auio.uio_resid = aiov.iov_len;
932cb05b60aSAttilio Rao 		vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY);
9338f8def9eSPoul-Henning Kamp 		error = VOP_READ(sc->vnode, &auio, 0, sc->cred);
93422db15c0SAttilio Rao 		VOP_UNLOCK(sc->vnode, 0);
9358f8def9eSPoul-Henning Kamp 		free(tmpbuf, M_TEMP);
9368f8def9eSPoul-Henning Kamp 	}
9378f8def9eSPoul-Henning Kamp 	return (error);
9388f8def9eSPoul-Henning Kamp }
9398f8def9eSPoul-Henning Kamp 
9408f8def9eSPoul-Henning Kamp static int
941b830359bSPawel Jakub Dawidek mdcreate_vnode(struct md_s *sc, struct md_ioctl *mdio, struct thread *td)
9428f8def9eSPoul-Henning Kamp {
9438f8def9eSPoul-Henning Kamp 	struct vattr vattr;
9448f8def9eSPoul-Henning Kamp 	struct nameidata nd;
9453d5c947dSMarcel Moolenaar 	char *fname;
9465050aa86SKonstantin Belousov 	int error, flags;
9478f8def9eSPoul-Henning Kamp 
9483d5c947dSMarcel Moolenaar 	/*
9493d5c947dSMarcel Moolenaar 	 * Kernel-originated requests must have the filename appended
9503d5c947dSMarcel Moolenaar 	 * to the mdio structure to protect against malicious software.
9513d5c947dSMarcel Moolenaar 	 */
9523d5c947dSMarcel Moolenaar 	fname = mdio->md_file;
9533d5c947dSMarcel Moolenaar 	if ((void *)fname != (void *)(mdio + 1)) {
9543d5c947dSMarcel Moolenaar 		error = copyinstr(fname, sc->file, sizeof(sc->file), NULL);
95588b5b78dSPawel Jakub Dawidek 		if (error != 0)
95688b5b78dSPawel Jakub Dawidek 			return (error);
9573d5c947dSMarcel Moolenaar 	} else
9583d5c947dSMarcel Moolenaar 		strlcpy(sc->file, fname, sizeof(sc->file));
9593d5c947dSMarcel Moolenaar 
96086776891SChristian S.J. Peron 	/*
9613d5c947dSMarcel Moolenaar 	 * If the user specified that this is a read only device, don't
9623d5c947dSMarcel Moolenaar 	 * set the FWRITE mask before trying to open the backing store.
96386776891SChristian S.J. Peron 	 */
9643d5c947dSMarcel Moolenaar 	flags = FREAD | ((mdio->md_options & MD_READONLY) ? 0 : FWRITE);
9655050aa86SKonstantin Belousov 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->file, td);
9669e223287SKonstantin Belousov 	error = vn_open(&nd, &flags, 0, NULL);
967e3ed29a7SPawel Jakub Dawidek 	if (error != 0)
96852c6716fSPawel Jakub Dawidek 		return (error);
969b322d85dSPawel Jakub Dawidek 	NDFREE(&nd, NDF_ONLY_PNBUF);
97033fc3625SJohn Baldwin 	if (nd.ni_vp->v_type != VREG) {
97133fc3625SJohn Baldwin 		error = EINVAL;
97233fc3625SJohn Baldwin 		goto bad;
97333fc3625SJohn Baldwin 	}
97433fc3625SJohn Baldwin 	error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred);
97533fc3625SJohn Baldwin 	if (error != 0)
97633fc3625SJohn Baldwin 		goto bad;
97733fc3625SJohn Baldwin 	if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) {
97833fc3625SJohn Baldwin 		vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY);
97933fc3625SJohn Baldwin 		if (nd.ni_vp->v_iflag & VI_DOOMED) {
98033fc3625SJohn Baldwin 			/* Forced unmount. */
98133fc3625SJohn Baldwin 			error = EBADF;
98233fc3625SJohn Baldwin 			goto bad;
98333fc3625SJohn Baldwin 		}
9848f8def9eSPoul-Henning Kamp 	}
9853b7b5496SKonstantin Belousov 	nd.ni_vp->v_vflag |= VV_MD;
98622db15c0SAttilio Rao 	VOP_UNLOCK(nd.ni_vp, 0);
9879589c256SPoul-Henning Kamp 
988d5a929dcSPoul-Henning Kamp 	if (mdio->md_fwsectors != 0)
989d5a929dcSPoul-Henning Kamp 		sc->fwsectors = mdio->md_fwsectors;
990d5a929dcSPoul-Henning Kamp 	if (mdio->md_fwheads != 0)
991d5a929dcSPoul-Henning Kamp 		sc->fwheads = mdio->md_fwheads;
9927a6b2b64SPoul-Henning Kamp 	sc->flags = mdio->md_options & (MD_FORCE | MD_ASYNC);
9939589c256SPoul-Henning Kamp 	if (!(flags & FWRITE))
9949589c256SPoul-Henning Kamp 		sc->flags |= MD_READONLY;
9958f8def9eSPoul-Henning Kamp 	sc->vnode = nd.ni_vp;
9968f8def9eSPoul-Henning Kamp 
997a854ed98SJohn Baldwin 	error = mdsetcred(sc, td->td_ucred);
998b830359bSPawel Jakub Dawidek 	if (error != 0) {
9993cf74e53SPhilip Paeps 		sc->vnode = NULL;
1000cb05b60aSAttilio Rao 		vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
10013b7b5496SKonstantin Belousov 		nd.ni_vp->v_vflag &= ~VV_MD;
100233fc3625SJohn Baldwin 		goto bad;
100333fc3625SJohn Baldwin 	}
100433fc3625SJohn Baldwin 	return (0);
100533fc3625SJohn Baldwin bad:
100622db15c0SAttilio Rao 	VOP_UNLOCK(nd.ni_vp, 0);
100752c6716fSPawel Jakub Dawidek 	(void)vn_close(nd.ni_vp, flags, td->td_ucred, td);
10088f8def9eSPoul-Henning Kamp 	return (error);
10098f8def9eSPoul-Henning Kamp }
10108f8def9eSPoul-Henning Kamp 
10118f8def9eSPoul-Henning Kamp static int
1012b40ce416SJulian Elischer mddestroy(struct md_s *sc, struct thread *td)
10138f8def9eSPoul-Henning Kamp {
10140cddd8f0SMatthew Dillon 
10156f4f00f1SPoul-Henning Kamp 	if (sc->gp) {
10166f4f00f1SPoul-Henning Kamp 		sc->gp->softc = NULL;
10179b00ca19SPoul-Henning Kamp 		g_topology_lock();
10189b00ca19SPoul-Henning Kamp 		g_wither_geom(sc->gp, ENXIO);
10199b00ca19SPoul-Henning Kamp 		g_topology_unlock();
10206b60a2cdSPoul-Henning Kamp 		sc->gp = NULL;
10216b60a2cdSPoul-Henning Kamp 		sc->pp = NULL;
10221f4ee1aaSPoul-Henning Kamp 	}
1023a03be42dSMaxim Sobolev 	if (sc->devstat) {
1024a03be42dSMaxim Sobolev 		devstat_remove_entry(sc->devstat);
1025a03be42dSMaxim Sobolev 		sc->devstat = NULL;
1026a03be42dSMaxim Sobolev 	}
1027a08d2e7fSJohn Baldwin 	mtx_lock(&sc->queue_mtx);
10285c97ca54SIan Dowse 	sc->flags |= MD_SHUTDOWN;
10295c97ca54SIan Dowse 	wakeup(sc);
1030a08d2e7fSJohn Baldwin 	while (!(sc->flags & MD_EXITING))
1031a08d2e7fSJohn Baldwin 		msleep(sc->procp, &sc->queue_mtx, PRIBIO, "mddestroy", hz / 10);
1032a08d2e7fSJohn Baldwin 	mtx_unlock(&sc->queue_mtx);
10339fbea3e3SPoul-Henning Kamp 	mtx_destroy(&sc->queue_mtx);
10349b00ca19SPoul-Henning Kamp 	if (sc->vnode != NULL) {
1035cb05b60aSAttilio Rao 		vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY);
10363b7b5496SKonstantin Belousov 		sc->vnode->v_vflag &= ~VV_MD;
103722db15c0SAttilio Rao 		VOP_UNLOCK(sc->vnode, 0);
10389d4b5945SMaxim Sobolev 		(void)vn_close(sc->vnode, sc->flags & MD_READONLY ?
1039b40ce416SJulian Elischer 		    FREAD : (FREAD|FWRITE), sc->cred, td);
10409b00ca19SPoul-Henning Kamp 	}
10418f8def9eSPoul-Henning Kamp 	if (sc->cred != NULL)
10428f8def9eSPoul-Henning Kamp 		crfree(sc->cred);
10431db17c6dSPawel Jakub Dawidek 	if (sc->object != NULL)
1044f820bc50SAlan Cox 		vm_object_deallocate(sc->object);
1045f43b2bacSPoul-Henning Kamp 	if (sc->indir)
1046f43b2bacSPoul-Henning Kamp 		destroy_indir(sc, sc->indir);
1047f43b2bacSPoul-Henning Kamp 	if (sc->uma)
1048f43b2bacSPoul-Henning Kamp 		uma_zdestroy(sc->uma);
10491f4ee1aaSPoul-Henning Kamp 
10501f4ee1aaSPoul-Henning Kamp 	LIST_REMOVE(sc, list);
1051f4e7c5a8SJaakko Heinonen 	free_unr(md_uh, sc->unit);
1052c6517568SPoul-Henning Kamp 	free(sc, M_MD);
10538f8def9eSPoul-Henning Kamp 	return (0);
10548f8def9eSPoul-Henning Kamp }
10558f8def9eSPoul-Henning Kamp 
10568f8def9eSPoul-Henning Kamp static int
1057dc604f0cSEdward Tomasz Napierala mdresize(struct md_s *sc, struct md_ioctl *mdio)
1058dc604f0cSEdward Tomasz Napierala {
1059dc604f0cSEdward Tomasz Napierala 	int error, res;
1060dc604f0cSEdward Tomasz Napierala 	vm_pindex_t oldpages, newpages;
1061dc604f0cSEdward Tomasz Napierala 
1062dc604f0cSEdward Tomasz Napierala 	switch (sc->type) {
1063dc604f0cSEdward Tomasz Napierala 	case MD_VNODE:
1064dc604f0cSEdward Tomasz Napierala 		break;
1065dc604f0cSEdward Tomasz Napierala 	case MD_SWAP:
10668cb51643SJaakko Heinonen 		if (mdio->md_mediasize <= 0 ||
1067dc604f0cSEdward Tomasz Napierala 		    (mdio->md_mediasize % PAGE_SIZE) != 0)
1068dc604f0cSEdward Tomasz Napierala 			return (EDOM);
1069dc604f0cSEdward Tomasz Napierala 		oldpages = OFF_TO_IDX(round_page(sc->mediasize));
1070dc604f0cSEdward Tomasz Napierala 		newpages = OFF_TO_IDX(round_page(mdio->md_mediasize));
1071dc604f0cSEdward Tomasz Napierala 		if (newpages < oldpages) {
1072dc604f0cSEdward Tomasz Napierala 			VM_OBJECT_LOCK(sc->object);
1073dc604f0cSEdward Tomasz Napierala 			vm_object_page_remove(sc->object, newpages, 0, 0);
1074dc604f0cSEdward Tomasz Napierala 			swap_pager_freespace(sc->object, newpages,
1075dc604f0cSEdward Tomasz Napierala 			    oldpages - newpages);
1076dc604f0cSEdward Tomasz Napierala 			swap_release_by_cred(IDX_TO_OFF(oldpages -
1077dc604f0cSEdward Tomasz Napierala 			    newpages), sc->cred);
1078dc604f0cSEdward Tomasz Napierala 			sc->object->charge = IDX_TO_OFF(newpages);
1079dc604f0cSEdward Tomasz Napierala 			sc->object->size = newpages;
1080dc604f0cSEdward Tomasz Napierala 			VM_OBJECT_UNLOCK(sc->object);
1081dc604f0cSEdward Tomasz Napierala 		} else if (newpages > oldpages) {
1082dc604f0cSEdward Tomasz Napierala 			res = swap_reserve_by_cred(IDX_TO_OFF(newpages -
1083dc604f0cSEdward Tomasz Napierala 			    oldpages), sc->cred);
1084dc604f0cSEdward Tomasz Napierala 			if (!res)
1085dc604f0cSEdward Tomasz Napierala 				return (ENOMEM);
1086dc604f0cSEdward Tomasz Napierala 			if ((mdio->md_options & MD_RESERVE) ||
1087dc604f0cSEdward Tomasz Napierala 			    (sc->flags & MD_RESERVE)) {
1088dc604f0cSEdward Tomasz Napierala 				error = swap_pager_reserve(sc->object,
1089dc604f0cSEdward Tomasz Napierala 				    oldpages, newpages - oldpages);
1090dc604f0cSEdward Tomasz Napierala 				if (error < 0) {
1091dc604f0cSEdward Tomasz Napierala 					swap_release_by_cred(
1092dc604f0cSEdward Tomasz Napierala 					    IDX_TO_OFF(newpages - oldpages),
1093dc604f0cSEdward Tomasz Napierala 					    sc->cred);
1094dc604f0cSEdward Tomasz Napierala 					return (EDOM);
1095dc604f0cSEdward Tomasz Napierala 				}
1096dc604f0cSEdward Tomasz Napierala 			}
1097dc604f0cSEdward Tomasz Napierala 			VM_OBJECT_LOCK(sc->object);
1098dc604f0cSEdward Tomasz Napierala 			sc->object->charge = IDX_TO_OFF(newpages);
1099dc604f0cSEdward Tomasz Napierala 			sc->object->size = newpages;
1100dc604f0cSEdward Tomasz Napierala 			VM_OBJECT_UNLOCK(sc->object);
1101dc604f0cSEdward Tomasz Napierala 		}
1102dc604f0cSEdward Tomasz Napierala 		break;
1103dc604f0cSEdward Tomasz Napierala 	default:
1104dc604f0cSEdward Tomasz Napierala 		return (EOPNOTSUPP);
1105dc604f0cSEdward Tomasz Napierala 	}
1106dc604f0cSEdward Tomasz Napierala 
1107dc604f0cSEdward Tomasz Napierala 	sc->mediasize = mdio->md_mediasize;
1108dc604f0cSEdward Tomasz Napierala 	g_topology_lock();
1109dc604f0cSEdward Tomasz Napierala 	g_resize_provider(sc->pp, sc->mediasize);
1110dc604f0cSEdward Tomasz Napierala 	g_topology_unlock();
1111dc604f0cSEdward Tomasz Napierala 	return (0);
1112dc604f0cSEdward Tomasz Napierala }
1113dc604f0cSEdward Tomasz Napierala 
1114dc604f0cSEdward Tomasz Napierala static int
1115b830359bSPawel Jakub Dawidek mdcreate_swap(struct md_s *sc, struct md_ioctl *mdio, struct thread *td)
11168f8def9eSPoul-Henning Kamp {
1117fcd57fbeSPawel Jakub Dawidek 	vm_ooffset_t npage;
1118fcd57fbeSPawel Jakub Dawidek 	int error;
11198f8def9eSPoul-Henning Kamp 
11208f8def9eSPoul-Henning Kamp 	/*
11218f8def9eSPoul-Henning Kamp 	 * Range check.  Disallow negative sizes or any size less then the
11228f8def9eSPoul-Henning Kamp 	 * size of a page.  Then round to a page.
11238f8def9eSPoul-Henning Kamp 	 */
11248cb51643SJaakko Heinonen 	if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0)
11258f8def9eSPoul-Henning Kamp 		return (EDOM);
11268f8def9eSPoul-Henning Kamp 
11278f8def9eSPoul-Henning Kamp 	/*
11288f8def9eSPoul-Henning Kamp 	 * Allocate an OBJT_SWAP object.
11298f8def9eSPoul-Henning Kamp 	 *
11308f8def9eSPoul-Henning Kamp 	 * Note the truncation.
11318f8def9eSPoul-Henning Kamp 	 */
11328f8def9eSPoul-Henning Kamp 
1133b830359bSPawel Jakub Dawidek 	npage = mdio->md_mediasize / PAGE_SIZE;
11349ed40643SPoul-Henning Kamp 	if (mdio->md_fwsectors != 0)
11359ed40643SPoul-Henning Kamp 		sc->fwsectors = mdio->md_fwsectors;
11369ed40643SPoul-Henning Kamp 	if (mdio->md_fwheads != 0)
11379ed40643SPoul-Henning Kamp 		sc->fwheads = mdio->md_fwheads;
1138fcd57fbeSPawel Jakub Dawidek 	sc->object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * npage,
11393364c323SKonstantin Belousov 	    VM_PROT_DEFAULT, 0, td->td_ucred);
1140812851b6SBrian Feldman 	if (sc->object == NULL)
1141812851b6SBrian Feldman 		return (ENOMEM);
1142dc604f0cSEdward Tomasz Napierala 	sc->flags = mdio->md_options & (MD_FORCE | MD_RESERVE);
11438f8def9eSPoul-Henning Kamp 	if (mdio->md_options & MD_RESERVE) {
1144fcd57fbeSPawel Jakub Dawidek 		if (swap_pager_reserve(sc->object, 0, npage) < 0) {
11453364c323SKonstantin Belousov 			error = EDOM;
11463364c323SKonstantin Belousov 			goto finish;
11478f8def9eSPoul-Henning Kamp 		}
11488f8def9eSPoul-Henning Kamp 	}
1149a854ed98SJohn Baldwin 	error = mdsetcred(sc, td->td_ucred);
11503364c323SKonstantin Belousov  finish:
1151e3ed29a7SPawel Jakub Dawidek 	if (error != 0) {
11522eafd8b1SPawel Jakub Dawidek 		vm_object_deallocate(sc->object);
11532eafd8b1SPawel Jakub Dawidek 		sc->object = NULL;
11548f8def9eSPoul-Henning Kamp 	}
1155b830359bSPawel Jakub Dawidek 	return (error);
1156b3b3d1b7SPoul-Henning Kamp }
11578f8def9eSPoul-Henning Kamp 
11589d4b5945SMaxim Sobolev 
11599d4b5945SMaxim Sobolev static int
11609b00ca19SPoul-Henning Kamp xmdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td)
11618f8def9eSPoul-Henning Kamp {
11628f8def9eSPoul-Henning Kamp 	struct md_ioctl *mdio;
11638f8def9eSPoul-Henning Kamp 	struct md_s *sc;
1164b830359bSPawel Jakub Dawidek 	int error, i;
11658cb51643SJaakko Heinonen 	unsigned sectsize;
11668f8def9eSPoul-Henning Kamp 
11678f8def9eSPoul-Henning Kamp 	if (md_debug)
11688f8def9eSPoul-Henning Kamp 		printf("mdctlioctl(%s %lx %p %x %p)\n",
1169b40ce416SJulian Elischer 			devtoname(dev), cmd, addr, flags, td);
11708f8def9eSPoul-Henning Kamp 
11719b00ca19SPoul-Henning Kamp 	mdio = (struct md_ioctl *)addr;
11729b00ca19SPoul-Henning Kamp 	if (mdio->md_version != MDIOVERSION)
11739b00ca19SPoul-Henning Kamp 		return (EINVAL);
11749b00ca19SPoul-Henning Kamp 
117553d745bcSDima Dorfman 	/*
117653d745bcSDima Dorfman 	 * We assert the version number in the individual ioctl
117753d745bcSDima Dorfman 	 * handlers instead of out here because (a) it is possible we
117853d745bcSDima Dorfman 	 * may add another ioctl in the future which doesn't read an
117953d745bcSDima Dorfman 	 * mdio, and (b) the correct return value for an unknown ioctl
118053d745bcSDima Dorfman 	 * is ENOIOCTL, not EINVAL.
118153d745bcSDima Dorfman 	 */
11829b00ca19SPoul-Henning Kamp 	error = 0;
11838f8def9eSPoul-Henning Kamp 	switch (cmd) {
11848f8def9eSPoul-Henning Kamp 	case MDIOCATTACH:
11858f8def9eSPoul-Henning Kamp 		switch (mdio->md_type) {
11868f8def9eSPoul-Henning Kamp 		case MD_MALLOC:
11878f8def9eSPoul-Henning Kamp 		case MD_PRELOAD:
11888f8def9eSPoul-Henning Kamp 		case MD_VNODE:
11898f8def9eSPoul-Henning Kamp 		case MD_SWAP:
1190b830359bSPawel Jakub Dawidek 			break;
11918f8def9eSPoul-Henning Kamp 		default:
11928f8def9eSPoul-Henning Kamp 			return (EINVAL);
11938f8def9eSPoul-Henning Kamp 		}
11948cb51643SJaakko Heinonen 		if (mdio->md_sectorsize == 0)
11958cb51643SJaakko Heinonen 			sectsize = DEV_BSIZE;
11968cb51643SJaakko Heinonen 		else
11978cb51643SJaakko Heinonen 			sectsize = mdio->md_sectorsize;
11988cb51643SJaakko Heinonen 		if (sectsize > MAXPHYS || mdio->md_mediasize < sectsize)
11998cb51643SJaakko Heinonen 			return (EINVAL);
12007ee3c044SPawel Jakub Dawidek 		if (mdio->md_options & MD_AUTOUNIT)
1201947fc8deSPoul-Henning Kamp 			sc = mdnew(-1, &error, mdio->md_type);
1202f4e7c5a8SJaakko Heinonen 		else {
1203f4e7c5a8SJaakko Heinonen 			if (mdio->md_unit > INT_MAX)
1204f4e7c5a8SJaakko Heinonen 				return (EINVAL);
1205947fc8deSPoul-Henning Kamp 			sc = mdnew(mdio->md_unit, &error, mdio->md_type);
1206f4e7c5a8SJaakko Heinonen 		}
1207b830359bSPawel Jakub Dawidek 		if (sc == NULL)
12087ee3c044SPawel Jakub Dawidek 			return (error);
12097ee3c044SPawel Jakub Dawidek 		if (mdio->md_options & MD_AUTOUNIT)
12107ee3c044SPawel Jakub Dawidek 			mdio->md_unit = sc->unit;
1211b830359bSPawel Jakub Dawidek 		sc->mediasize = mdio->md_mediasize;
12128cb51643SJaakko Heinonen 		sc->sectorsize = sectsize;
1213b830359bSPawel Jakub Dawidek 		error = EDOOFUS;
1214b830359bSPawel Jakub Dawidek 		switch (sc->type) {
1215b830359bSPawel Jakub Dawidek 		case MD_MALLOC:
12169b00ca19SPoul-Henning Kamp 			sc->start = mdstart_malloc;
1217b830359bSPawel Jakub Dawidek 			error = mdcreate_malloc(sc, mdio);
1218b830359bSPawel Jakub Dawidek 			break;
1219b830359bSPawel Jakub Dawidek 		case MD_PRELOAD:
1220*734e78dfSJaakko Heinonen 			/*
1221*734e78dfSJaakko Heinonen 			 * We disallow attaching preloaded memory disks via
1222*734e78dfSJaakko Heinonen 			 * ioctl. Preloaded memory disks are automatically
1223*734e78dfSJaakko Heinonen 			 * attached in g_md_init().
1224*734e78dfSJaakko Heinonen 			 */
1225*734e78dfSJaakko Heinonen 			error = EOPNOTSUPP;
1226b830359bSPawel Jakub Dawidek 			break;
1227b830359bSPawel Jakub Dawidek 		case MD_VNODE:
12289b00ca19SPoul-Henning Kamp 			sc->start = mdstart_vnode;
1229b830359bSPawel Jakub Dawidek 			error = mdcreate_vnode(sc, mdio, td);
1230b830359bSPawel Jakub Dawidek 			break;
1231b830359bSPawel Jakub Dawidek 		case MD_SWAP:
12329b00ca19SPoul-Henning Kamp 			sc->start = mdstart_swap;
1233b830359bSPawel Jakub Dawidek 			error = mdcreate_swap(sc, mdio, td);
1234b830359bSPawel Jakub Dawidek 			break;
1235b830359bSPawel Jakub Dawidek 		}
1236b830359bSPawel Jakub Dawidek 		if (error != 0) {
1237b830359bSPawel Jakub Dawidek 			mddestroy(sc, td);
1238b830359bSPawel Jakub Dawidek 			return (error);
1239b830359bSPawel Jakub Dawidek 		}
12409b00ca19SPoul-Henning Kamp 
12419b00ca19SPoul-Henning Kamp 		/* Prune off any residual fractional sector */
12429b00ca19SPoul-Henning Kamp 		i = sc->mediasize % sc->sectorsize;
12439b00ca19SPoul-Henning Kamp 		sc->mediasize -= i;
12449b00ca19SPoul-Henning Kamp 
1245b830359bSPawel Jakub Dawidek 		mdinit(sc);
1246b830359bSPawel Jakub Dawidek 		return (0);
12478f8def9eSPoul-Henning Kamp 	case MDIOCDETACH:
1248a9ebb311SEdward Tomasz Napierala 		if (mdio->md_mediasize != 0 ||
1249a9ebb311SEdward Tomasz Napierala 		    (mdio->md_options & ~MD_FORCE) != 0)
12508f8def9eSPoul-Henning Kamp 			return (EINVAL);
12519b00ca19SPoul-Henning Kamp 
12529b00ca19SPoul-Henning Kamp 		sc = mdfind(mdio->md_unit);
12539b00ca19SPoul-Henning Kamp 		if (sc == NULL)
12549b00ca19SPoul-Henning Kamp 			return (ENOENT);
1255a9ebb311SEdward Tomasz Napierala 		if (sc->opencount != 0 && !(sc->flags & MD_FORCE) &&
1256a9ebb311SEdward Tomasz Napierala 		    !(mdio->md_options & MD_FORCE))
12579b00ca19SPoul-Henning Kamp 			return (EBUSY);
12589b00ca19SPoul-Henning Kamp 		return (mddestroy(sc, td));
1259dc604f0cSEdward Tomasz Napierala 	case MDIOCRESIZE:
1260dc604f0cSEdward Tomasz Napierala 		if ((mdio->md_options & ~(MD_FORCE | MD_RESERVE)) != 0)
1261dc604f0cSEdward Tomasz Napierala 			return (EINVAL);
1262dc604f0cSEdward Tomasz Napierala 
1263dc604f0cSEdward Tomasz Napierala 		sc = mdfind(mdio->md_unit);
1264dc604f0cSEdward Tomasz Napierala 		if (sc == NULL)
1265dc604f0cSEdward Tomasz Napierala 			return (ENOENT);
12668cb51643SJaakko Heinonen 		if (mdio->md_mediasize < sc->sectorsize)
12678cb51643SJaakko Heinonen 			return (EINVAL);
1268dc604f0cSEdward Tomasz Napierala 		if (mdio->md_mediasize < sc->mediasize &&
1269dc604f0cSEdward Tomasz Napierala 		    !(sc->flags & MD_FORCE) &&
1270dc604f0cSEdward Tomasz Napierala 		    !(mdio->md_options & MD_FORCE))
1271dc604f0cSEdward Tomasz Napierala 			return (EBUSY);
1272dc604f0cSEdward Tomasz Napierala 		return (mdresize(sc, mdio));
1273174b5e9aSPoul-Henning Kamp 	case MDIOCQUERY:
1274174b5e9aSPoul-Henning Kamp 		sc = mdfind(mdio->md_unit);
1275174b5e9aSPoul-Henning Kamp 		if (sc == NULL)
1276174b5e9aSPoul-Henning Kamp 			return (ENOENT);
1277174b5e9aSPoul-Henning Kamp 		mdio->md_type = sc->type;
1278174b5e9aSPoul-Henning Kamp 		mdio->md_options = sc->flags;
1279b830359bSPawel Jakub Dawidek 		mdio->md_mediasize = sc->mediasize;
1280b830359bSPawel Jakub Dawidek 		mdio->md_sectorsize = sc->sectorsize;
12819b00ca19SPoul-Henning Kamp 		if (sc->type == MD_VNODE)
128288b5b78dSPawel Jakub Dawidek 			error = copyout(sc->file, mdio->md_file,
128388b5b78dSPawel Jakub Dawidek 			    strlen(sc->file) + 1);
128488b5b78dSPawel Jakub Dawidek 		return (error);
128516bcbe8cSPoul-Henning Kamp 	case MDIOCLIST:
128616bcbe8cSPoul-Henning Kamp 		i = 1;
128716bcbe8cSPoul-Henning Kamp 		LIST_FOREACH(sc, &md_softc_list, list) {
128816bcbe8cSPoul-Henning Kamp 			if (i == MDNPAD - 1)
128916bcbe8cSPoul-Henning Kamp 				mdio->md_pad[i] = -1;
129016bcbe8cSPoul-Henning Kamp 			else
129116bcbe8cSPoul-Henning Kamp 				mdio->md_pad[i++] = sc->unit;
129216bcbe8cSPoul-Henning Kamp 		}
129316bcbe8cSPoul-Henning Kamp 		mdio->md_pad[0] = i - 1;
129416bcbe8cSPoul-Henning Kamp 		return (0);
12958f8def9eSPoul-Henning Kamp 	default:
12968f8def9eSPoul-Henning Kamp 		return (ENOIOCTL);
12978f8def9eSPoul-Henning Kamp 	};
12989b00ca19SPoul-Henning Kamp }
12999b00ca19SPoul-Henning Kamp 
13009b00ca19SPoul-Henning Kamp static int
13019b00ca19SPoul-Henning Kamp mdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td)
13029b00ca19SPoul-Henning Kamp {
13039b00ca19SPoul-Henning Kamp 	int error;
13049b00ca19SPoul-Henning Kamp 
13059b00ca19SPoul-Henning Kamp 	sx_xlock(&md_sx);
13069b00ca19SPoul-Henning Kamp 	error = xmdctlioctl(dev, cmd, addr, flags, td);
13079b00ca19SPoul-Henning Kamp 	sx_xunlock(&md_sx);
13089b00ca19SPoul-Henning Kamp 	return (error);
13093f54a085SPoul-Henning Kamp }
13103f54a085SPoul-Henning Kamp 
131100a6a3c6SPoul-Henning Kamp static void
1312b830359bSPawel Jakub Dawidek md_preloaded(u_char *image, size_t length)
1313637f671aSPoul-Henning Kamp {
1314637f671aSPoul-Henning Kamp 	struct md_s *sc;
13159b00ca19SPoul-Henning Kamp 	int error;
1316637f671aSPoul-Henning Kamp 
1317947fc8deSPoul-Henning Kamp 	sc = mdnew(-1, &error, MD_PRELOAD);
1318637f671aSPoul-Henning Kamp 	if (sc == NULL)
1319637f671aSPoul-Henning Kamp 		return;
1320b830359bSPawel Jakub Dawidek 	sc->mediasize = length;
1321b830359bSPawel Jakub Dawidek 	sc->sectorsize = DEV_BSIZE;
1322637f671aSPoul-Henning Kamp 	sc->pl_ptr = image;
1323637f671aSPoul-Henning Kamp 	sc->pl_len = length;
13249b00ca19SPoul-Henning Kamp 	sc->start = mdstart_preload;
13255d4ca75eSLuigi Rizzo #ifdef MD_ROOT
1326637f671aSPoul-Henning Kamp 	if (sc->unit == 0)
132722ff74b2SMarcel Moolenaar 		rootdevnames[0] = MD_ROOT_FSTYPE ":/dev/md0";
13285d4ca75eSLuigi Rizzo #endif
1329637f671aSPoul-Henning Kamp 	mdinit(sc);
1330637f671aSPoul-Henning Kamp }
1331637f671aSPoul-Henning Kamp 
1332637f671aSPoul-Henning Kamp static void
133319945697SPoul-Henning Kamp g_md_init(struct g_class *mp __unused)
133400a6a3c6SPoul-Henning Kamp {
133595f1a897SPoul-Henning Kamp 	caddr_t mod;
133695f1a897SPoul-Henning Kamp 	u_char *ptr, *name, *type;
133795f1a897SPoul-Henning Kamp 	unsigned len;
1338d12fc952SKonstantin Belousov 	int i;
1339d12fc952SKonstantin Belousov 
1340d12fc952SKonstantin Belousov 	/* figure out log2(NINDIR) */
1341d12fc952SKonstantin Belousov 	for (i = NINDIR, nshift = -1; i; nshift++)
1342d12fc952SKonstantin Belousov 		i >>= 1;
134395f1a897SPoul-Henning Kamp 
13440a937206SPoul-Henning Kamp 	mod = NULL;
13459b00ca19SPoul-Henning Kamp 	sx_init(&md_sx, "MD config lock");
13460a937206SPoul-Henning Kamp 	g_topology_unlock();
1347f4e7c5a8SJaakko Heinonen 	md_uh = new_unrhdr(0, INT_MAX, NULL);
134871e4fff8SPoul-Henning Kamp #ifdef MD_ROOT_SIZE
13499b00ca19SPoul-Henning Kamp 	sx_xlock(&md_sx);
1350de64f22aSLuigi Rizzo 	md_preloaded(mfs_root.start, sizeof(mfs_root.start));
13519b00ca19SPoul-Henning Kamp 	sx_xunlock(&md_sx);
135271e4fff8SPoul-Henning Kamp #endif
13539b00ca19SPoul-Henning Kamp 	/* XXX: are preload_* static or do they need Giant ? */
135495f1a897SPoul-Henning Kamp 	while ((mod = preload_search_next_name(mod)) != NULL) {
135595f1a897SPoul-Henning Kamp 		name = (char *)preload_search_info(mod, MODINFO_NAME);
135695f1a897SPoul-Henning Kamp 		if (name == NULL)
135795f1a897SPoul-Henning Kamp 			continue;
13589b00ca19SPoul-Henning Kamp 		type = (char *)preload_search_info(mod, MODINFO_TYPE);
135995f1a897SPoul-Henning Kamp 		if (type == NULL)
136095f1a897SPoul-Henning Kamp 			continue;
136171e4fff8SPoul-Henning Kamp 		if (strcmp(type, "md_image") && strcmp(type, "mfs_root"))
136295f1a897SPoul-Henning Kamp 			continue;
13638d5ac6c3SMarcel Moolenaar 		ptr = preload_fetch_addr(mod);
13648d5ac6c3SMarcel Moolenaar 		len = preload_fetch_size(mod);
13658d5ac6c3SMarcel Moolenaar 		if (ptr != NULL && len != 0) {
1366fe603109SMaxim Sobolev 			printf("%s%d: Preloaded image <%s> %d bytes at %p\n",
1367fe603109SMaxim Sobolev 			    MD_NAME, mdunits, name, len, ptr);
13689b00ca19SPoul-Henning Kamp 			sx_xlock(&md_sx);
1369637f671aSPoul-Henning Kamp 			md_preloaded(ptr, len);
13709b00ca19SPoul-Henning Kamp 			sx_xunlock(&md_sx);
137195f1a897SPoul-Henning Kamp 		}
13728d5ac6c3SMarcel Moolenaar 	}
137306d425f9SEd Schouten 	status_dev = make_dev(&mdctl_cdevsw, INT_MAX, UID_ROOT, GID_WHEEL,
137410b0e058SDima Dorfman 	    0600, MDCTL_NAME);
13750eb14309SPoul-Henning Kamp 	g_topology_lock();
137600a6a3c6SPoul-Henning Kamp }
137700a6a3c6SPoul-Henning Kamp 
137819945697SPoul-Henning Kamp static void
1379c27a8954SWojciech A. Koszek g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1380c27a8954SWojciech A. Koszek     struct g_consumer *cp __unused, struct g_provider *pp)
1381c27a8954SWojciech A. Koszek {
1382c27a8954SWojciech A. Koszek 	struct md_s *mp;
1383c27a8954SWojciech A. Koszek 	char *type;
1384c27a8954SWojciech A. Koszek 
1385c27a8954SWojciech A. Koszek 	mp = gp->softc;
1386c27a8954SWojciech A. Koszek 	if (mp == NULL)
1387c27a8954SWojciech A. Koszek 		return;
1388c27a8954SWojciech A. Koszek 
1389c27a8954SWojciech A. Koszek 	switch (mp->type) {
1390c27a8954SWojciech A. Koszek 	case MD_MALLOC:
1391c27a8954SWojciech A. Koszek 		type = "malloc";
1392c27a8954SWojciech A. Koszek 		break;
1393c27a8954SWojciech A. Koszek 	case MD_PRELOAD:
1394c27a8954SWojciech A. Koszek 		type = "preload";
1395c27a8954SWojciech A. Koszek 		break;
1396c27a8954SWojciech A. Koszek 	case MD_VNODE:
1397c27a8954SWojciech A. Koszek 		type = "vnode";
1398c27a8954SWojciech A. Koszek 		break;
1399c27a8954SWojciech A. Koszek 	case MD_SWAP:
1400c27a8954SWojciech A. Koszek 		type = "swap";
1401c27a8954SWojciech A. Koszek 		break;
1402c27a8954SWojciech A. Koszek 	default:
1403c27a8954SWojciech A. Koszek 		type = "unknown";
1404c27a8954SWojciech A. Koszek 		break;
1405c27a8954SWojciech A. Koszek 	}
1406c27a8954SWojciech A. Koszek 
1407c27a8954SWojciech A. Koszek 	if (pp != NULL) {
1408c27a8954SWojciech A. Koszek 		if (indent == NULL) {
1409c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " u %d", mp->unit);
1410c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " s %ju", (uintmax_t) mp->sectorsize);
1411c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " f %ju", (uintmax_t) mp->fwheads);
1412c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " fs %ju", (uintmax_t) mp->fwsectors);
1413c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " l %ju", (uintmax_t) mp->mediasize);
1414c27a8954SWojciech A. Koszek 			sbuf_printf(sb, " t %s", type);
1415c27a8954SWojciech A. Koszek 			if (mp->type == MD_VNODE && mp->vnode != NULL)
1416c27a8954SWojciech A. Koszek 				sbuf_printf(sb, " file %s", mp->file);
1417c27a8954SWojciech A. Koszek 		} else {
1418c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<unit>%d</unit>\n", indent,
1419c27a8954SWojciech A. Koszek 			    mp->unit);
1420c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<sectorsize>%ju</sectorsize>\n",
1421c27a8954SWojciech A. Koszek 			    indent, (uintmax_t) mp->sectorsize);
1422c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<fwheads>%ju</fwheads>\n",
1423c27a8954SWojciech A. Koszek 			    indent, (uintmax_t) mp->fwheads);
1424c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<fwsectors>%ju</fwsectors>\n",
1425c27a8954SWojciech A. Koszek 			    indent, (uintmax_t) mp->fwsectors);
1426c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<length>%ju</length>\n",
1427c27a8954SWojciech A. Koszek 			    indent, (uintmax_t) mp->mediasize);
14281f192809SAndrey V. Elsukov 			sbuf_printf(sb, "%s<compression>%s</compression>\n", indent,
14291f192809SAndrey V. Elsukov 			    (mp->flags & MD_COMPRESS) == 0 ? "off": "on");
14301f192809SAndrey V. Elsukov 			sbuf_printf(sb, "%s<access>%s</access>\n", indent,
14311f192809SAndrey V. Elsukov 			    (mp->flags & MD_READONLY) == 0 ? "read-write":
14321f192809SAndrey V. Elsukov 			    "read-only");
1433c27a8954SWojciech A. Koszek 			sbuf_printf(sb, "%s<type>%s</type>\n", indent,
1434c27a8954SWojciech A. Koszek 			    type);
1435c27a8954SWojciech A. Koszek 			if (mp->type == MD_VNODE && mp->vnode != NULL)
1436c27a8954SWojciech A. Koszek 				sbuf_printf(sb, "%s<file>%s</file>\n",
1437c27a8954SWojciech A. Koszek 				    indent, mp->file);
1438c27a8954SWojciech A. Koszek 		}
1439c27a8954SWojciech A. Koszek 	}
1440c27a8954SWojciech A. Koszek }
1441c27a8954SWojciech A. Koszek 
1442c27a8954SWojciech A. Koszek static void
144319945697SPoul-Henning Kamp g_md_fini(struct g_class *mp __unused)
144457e9624eSPoul-Henning Kamp {
14459d4b5945SMaxim Sobolev 
14469b00ca19SPoul-Henning Kamp 	sx_destroy(&md_sx);
144719945697SPoul-Henning Kamp 	if (status_dev != NULL)
144857e9624eSPoul-Henning Kamp 		destroy_dev(status_dev);
1449f4e7c5a8SJaakko Heinonen 	delete_unrhdr(md_uh);
145057e9624eSPoul-Henning Kamp }
1451