xref: /illumos-gate/usr/src/uts/common/io/mem.c (revision 99ea293e719ac006d413e4fde6ac0d5cd4dd6c59)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright (c) 2016 by Delphix. All rights reserved.
25  */
26 
27 /*
28  * Copyright 2017 Joyent, Inc.
29  * Copyright 2017 James S Blachly, MD <james.blachly@gmail.com>
30  */
31 
32 /*
33  * Memory special file
34  */
35 
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/user.h>
39 #include <sys/buf.h>
40 #include <sys/systm.h>
41 #include <sys/cred.h>
42 #include <sys/vm.h>
43 #include <sys/uio.h>
44 #include <sys/mman.h>
45 #include <sys/kmem.h>
46 #include <vm/seg.h>
47 #include <vm/page.h>
48 #include <sys/stat.h>
49 #include <sys/vmem.h>
50 #include <sys/memlist.h>
51 #include <sys/bootconf.h>
52 
53 #include <vm/seg_vn.h>
54 #include <vm/seg_dev.h>
55 #include <vm/seg_kmem.h>
56 #include <vm/seg_kp.h>
57 #include <vm/seg_kpm.h>
58 #include <vm/hat.h>
59 
60 #include <sys/conf.h>
61 #include <sys/mem.h>
62 #include <sys/types.h>
63 #include <sys/conf.h>
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/errno.h>
67 #include <sys/modctl.h>
68 #include <sys/memlist.h>
69 #include <sys/ddi.h>
70 #include <sys/sunddi.h>
71 #include <sys/debug.h>
72 #include <sys/fm/protocol.h>
73 
74 #if defined(__sparc)
75 extern int cpu_get_mem_name(uint64_t, uint64_t *, uint64_t, char *, int, int *);
76 extern int cpu_get_mem_info(uint64_t, uint64_t, uint64_t *, uint64_t *,
77     uint64_t *, int *, int *, int *);
78 extern size_t cpu_get_name_bufsize(void);
79 extern int cpu_get_mem_sid(char *, char *, int, int *);
80 extern int cpu_get_mem_addr(char *, char *, uint64_t, uint64_t *);
81 #elif defined(__x86)
82 #include <sys/cpu_module.h>
83 #endif	/* __sparc */
84 
85 /*
86  * Turn a byte length into a pagecount.  The DDI btop takes a
87  * 32-bit size on 32-bit machines, this handles 64-bit sizes for
88  * large physical-memory 32-bit machines.
89  */
90 #define	BTOP(x)	((pgcnt_t)((x) >> _pageshift))
91 
92 static kmutex_t mm_lock;
93 static caddr_t mm_map;
94 
95 static dev_info_t *mm_dip;	/* private copy of devinfo pointer */
96 
97 static int mm_kmem_io_access;
98 
99 static int mm_kstat_update(kstat_t *ksp, int rw);
100 static int mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw);
101 
102 static int mm_read_mem_name(intptr_t data, mem_name_t *mem_name);
103 
104 #define	MM_KMEMLOG_NENTRIES	64
105 
106 static int mm_kmemlogent;
107 static mm_logentry_t mm_kmemlog[MM_KMEMLOG_NENTRIES];
108 
109 /*
110  * On kmem/allmem writes, we log information that might be useful in the event
111  * that a write is errant (that is, due to operator error) and induces a later
112  * problem.  Note that (in particular) in the event of such operator-induced
113  * corruption, a search over the kernel address space for the corrupted
114  * address will yield the ring buffer entry that recorded the write.  And
115  * should it seem baroque or otherwise unnecessary, yes, we need this kind of
116  * auditing facility and yes, we learned that the hard way: disturbingly,
117  * there exist recommendations for "tuning" the system that involve writing to
118  * kernel memory addresses via the kernel debugger, and -- as we discovered --
119  * these can easily be applied incorrectly or unsafely, yielding an entirely
120  * undebuggable "can't happen" kind of panic.
121  */
122 static void
123 mm_logkmem(struct uio *uio)
124 {
125 	mm_logentry_t *ent;
126 	proc_t *p = curthread->t_procp;
127 
128 	mutex_enter(&mm_lock);
129 
130 	ent = &mm_kmemlog[mm_kmemlogent++];
131 
132 	if (mm_kmemlogent == MM_KMEMLOG_NENTRIES)
133 		mm_kmemlogent = 0;
134 
135 	ent->mle_vaddr = (uintptr_t)uio->uio_loffset;
136 	ent->mle_len = uio->uio_resid;
137 	gethrestime(&ent->mle_hrestime);
138 	ent->mle_hrtime = gethrtime();
139 	ent->mle_pid = p->p_pidp->pid_id;
140 
141 	(void) strncpy(ent->mle_psargs,
142 	    p->p_user.u_psargs, sizeof (ent->mle_psargs));
143 
144 	mutex_exit(&mm_lock);
145 }
146 
147 /*ARGSUSED1*/
148 static int
149 mm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
150 {
151 	int i;
152 	struct mem_minor {
153 		char *name;
154 		minor_t minor;
155 		int privonly;
156 		const char *rdpriv;
157 		const char *wrpriv;
158 		mode_t priv_mode;
159 	} mm[] = {
160 		{ "mem",	M_MEM,		0,	NULL,	"all",	0640 },
161 		{ "kmem",	M_KMEM,		0,	NULL,	"all",	0640 },
162 		{ "allkmem",	M_ALLKMEM,	0,	"all",	"all",	0600 },
163 		{ "null",	M_NULL,	PRIVONLY_DEV,	NULL,	NULL,	0666 },
164 		{ "zero",	M_ZERO, PRIVONLY_DEV,	NULL,	NULL,	0666 },
165 		{ "full",	M_FULL, PRIVONLY_DEV,	NULL,	NULL,	0666 },
166 	};
167 	kstat_t *ksp;
168 
169 	mutex_init(&mm_lock, NULL, MUTEX_DEFAULT, NULL);
170 	mm_map = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
171 
172 	for (i = 0; i < (sizeof (mm) / sizeof (mm[0])); i++) {
173 		if (ddi_create_priv_minor_node(devi, mm[i].name, S_IFCHR,
174 		    mm[i].minor, DDI_PSEUDO, mm[i].privonly,
175 		    mm[i].rdpriv, mm[i].wrpriv, mm[i].priv_mode) ==
176 		    DDI_FAILURE) {
177 			ddi_remove_minor_node(devi, NULL);
178 			return (DDI_FAILURE);
179 		}
180 	}
181 
182 	mm_dip = devi;
183 
184 	ksp = kstat_create("mm", 0, "phys_installed", "misc",
185 	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL);
186 	if (ksp != NULL) {
187 		ksp->ks_update = mm_kstat_update;
188 		ksp->ks_snapshot = mm_kstat_snapshot;
189 		ksp->ks_lock = &mm_lock; /* XXX - not really needed */
190 		kstat_install(ksp);
191 	}
192 
193 	mm_kmem_io_access = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
194 	    "kmem_io_access", 0);
195 
196 	return (DDI_SUCCESS);
197 }
198 
199 /*ARGSUSED*/
200 static int
201 mm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
202 {
203 	register int error;
204 
205 	switch (infocmd) {
206 	case DDI_INFO_DEVT2DEVINFO:
207 		*result = (void *)mm_dip;
208 		error = DDI_SUCCESS;
209 		break;
210 	case DDI_INFO_DEVT2INSTANCE:
211 		*result = (void *)0;
212 		error = DDI_SUCCESS;
213 		break;
214 	default:
215 		error = DDI_FAILURE;
216 	}
217 	return (error);
218 }
219 
220 /*ARGSUSED1*/
221 static int
222 mmopen(dev_t *devp, int flag, int typ, struct cred *cred)
223 {
224 	switch (getminor(*devp)) {
225 	case M_NULL:
226 	case M_ZERO:
227 	case M_FULL:
228 	case M_MEM:
229 	case M_KMEM:
230 	case M_ALLKMEM:
231 		/* standard devices */
232 		break;
233 
234 	default:
235 		/* Unsupported or unknown type */
236 		return (EINVAL);
237 	}
238 	/* must be character device */
239 	if (typ != OTYP_CHR)
240 		return (EINVAL);
241 	return (0);
242 }
243 
244 struct pollhead	mm_pollhd;
245 
246 /*ARGSUSED*/
247 static int
248 mmchpoll(dev_t dev, short events, int anyyet, short *reventsp,
249     struct pollhead **phpp)
250 {
251 	switch (getminor(dev)) {
252 	case M_NULL:
253 	case M_ZERO:
254 	case M_FULL:
255 	case M_MEM:
256 	case M_KMEM:
257 	case M_ALLKMEM:
258 		*reventsp = events & (POLLIN | POLLOUT | POLLPRI | POLLRDNORM |
259 		    POLLWRNORM | POLLRDBAND | POLLWRBAND);
260 		/*
261 		 * A non NULL pollhead pointer should be returned in case
262 		 * user polls for 0 events or is doing an edge-triggerd poll.
263 		 */
264 		if ((!*reventsp && !anyyet) || (events & POLLET)) {
265 			*phpp = &mm_pollhd;
266 		}
267 		return (0);
268 	default:
269 		/* no other devices currently support polling */
270 		return (ENXIO);
271 	}
272 }
273 
274 static int
275 mmpropop(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags,
276     char *name, caddr_t valuep, int *lengthp)
277 {
278 	/*
279 	 * implement zero size to reduce overhead (avoid two failing
280 	 * property lookups per stat).
281 	 */
282 	return (ddi_prop_op_size(dev, dip, prop_op,
283 	    flags, name, valuep, lengthp, 0));
284 }
285 
286 static int
287 mmio(struct uio *uio, enum uio_rw rw, pfn_t pfn, off_t pageoff, int allowio,
288     page_t *pp)
289 {
290 	int error = 0;
291 	int devload = 0;
292 	int is_memory = pf_is_memory(pfn);
293 	size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
294 	    (size_t)uio->uio_iov->iov_len);
295 	caddr_t va = NULL;
296 
297 	mutex_enter(&mm_lock);
298 
299 	if (is_memory && kpm_enable) {
300 		if (pp)
301 			va = hat_kpm_mapin(pp, NULL);
302 		else
303 			va = hat_kpm_mapin_pfn(pfn);
304 	}
305 
306 	if (va == NULL) {
307 		hat_devload(kas.a_hat, mm_map, PAGESIZE, pfn,
308 		    (uint_t)(rw == UIO_READ ? PROT_READ : PROT_READ|PROT_WRITE),
309 		    HAT_LOAD_NOCONSIST|HAT_LOAD_LOCK);
310 		va = mm_map;
311 		devload = 1;
312 	}
313 
314 	if (!is_memory) {
315 		if (allowio) {
316 			size_t c = uio->uio_iov->iov_len;
317 
318 			if (ddi_peekpokeio(NULL, uio, rw,
319 			    (caddr_t)(uintptr_t)uio->uio_loffset, c,
320 			    sizeof (int32_t)) != DDI_SUCCESS)
321 				error = EFAULT;
322 		} else
323 			error = EIO;
324 	} else
325 		error = uiomove(va + pageoff, nbytes, rw, uio);
326 
327 	if (devload)
328 		hat_unload(kas.a_hat, mm_map, PAGESIZE, HAT_UNLOAD_UNLOCK);
329 	else if (pp)
330 		hat_kpm_mapout(pp, NULL, va);
331 	else
332 		hat_kpm_mapout_pfn(pfn);
333 
334 	mutex_exit(&mm_lock);
335 	return (error);
336 }
337 
338 static int
339 mmpagelock(struct as *as, caddr_t va)
340 {
341 	struct seg *seg;
342 	int i;
343 
344 	AS_LOCK_ENTER(as, RW_READER);
345 	seg = as_segat(as, va);
346 	i = (seg != NULL)? SEGOP_CAPABLE(seg, S_CAPABILITY_NOMINFLT) : 0;
347 	AS_LOCK_EXIT(as);
348 
349 	return (i);
350 }
351 
352 #ifdef	__sparc
353 
354 #define	NEED_LOCK_KVADDR(kva)	mmpagelock(&kas, kva)
355 
356 #else	/* __i386, __amd64 */
357 
358 #define	NEED_LOCK_KVADDR(va)	0
359 
360 #endif	/* __sparc */
361 
362 /*ARGSUSED3*/
363 static int
364 mmrw(dev_t dev, struct uio *uio, enum uio_rw rw, cred_t *cred)
365 {
366 	pfn_t v;
367 	struct iovec *iov;
368 	int error = 0;
369 	size_t c;
370 	ssize_t oresid = uio->uio_resid;
371 	minor_t minor = getminor(dev);
372 
373 	while (uio->uio_resid > 0 && error == 0) {
374 		iov = uio->uio_iov;
375 		if (iov->iov_len == 0) {
376 			uio->uio_iov++;
377 			uio->uio_iovcnt--;
378 			if (uio->uio_iovcnt < 0)
379 				panic("mmrw");
380 			continue;
381 		}
382 		switch (minor) {
383 
384 		case M_MEM:
385 			memlist_read_lock();
386 			if (!address_in_memlist(phys_install,
387 			    (uint64_t)uio->uio_loffset, 1)) {
388 				memlist_read_unlock();
389 				error = EFAULT;
390 				break;
391 			}
392 			memlist_read_unlock();
393 
394 			v = BTOP((u_offset_t)uio->uio_loffset);
395 			error = mmio(uio, rw, v,
396 			    uio->uio_loffset & PAGEOFFSET, 0, NULL);
397 			break;
398 
399 		case M_KMEM:
400 		case M_ALLKMEM:
401 			{
402 			page_t **ppp = NULL;
403 			caddr_t vaddr = (caddr_t)uio->uio_offset;
404 			int try_lock = NEED_LOCK_KVADDR(vaddr);
405 			int locked = 0;
406 
407 			if ((error = plat_mem_do_mmio(uio, rw)) != ENOTSUP)
408 				break;
409 
410 			if (rw == UIO_WRITE)
411 				mm_logkmem(uio);
412 
413 			/*
414 			 * If vaddr does not map a valid page, as_pagelock()
415 			 * will return failure. Hence we can't check the
416 			 * return value and return EFAULT here as we'd like.
417 			 * seg_kp and seg_kpm do not properly support
418 			 * as_pagelock() for this context so we avoid it
419 			 * using the try_lock set check above.  Some day when
420 			 * the kernel page locking gets redesigned all this
421 			 * muck can be cleaned up.
422 			 */
423 			if (try_lock)
424 				locked = (as_pagelock(&kas, &ppp, vaddr,
425 				    PAGESIZE, S_WRITE) == 0);
426 
427 			v = hat_getpfnum(kas.a_hat,
428 			    (caddr_t)(uintptr_t)uio->uio_loffset);
429 			if (v == PFN_INVALID) {
430 				if (locked)
431 					as_pageunlock(&kas, ppp, vaddr,
432 					    PAGESIZE, S_WRITE);
433 				error = EFAULT;
434 				break;
435 			}
436 
437 			error = mmio(uio, rw, v, uio->uio_loffset & PAGEOFFSET,
438 			    minor == M_ALLKMEM || mm_kmem_io_access,
439 			    (locked && ppp) ? *ppp : NULL);
440 			if (locked)
441 				as_pageunlock(&kas, ppp, vaddr, PAGESIZE,
442 				    S_WRITE);
443 			}
444 
445 			break;
446 
447 		case M_FULL:
448 			if (rw == UIO_WRITE) {
449 				error = ENOSPC;
450 				break;
451 			}
452 			/* else it's a read, fall through to zero case */
453 			/*FALLTHROUGH*/
454 
455 		case M_ZERO:
456 			if (rw == UIO_READ) {
457 				label_t ljb;
458 
459 				if (on_fault(&ljb)) {
460 					no_fault();
461 					error = EFAULT;
462 					break;
463 				}
464 				uzero(iov->iov_base, iov->iov_len);
465 				no_fault();
466 				uio->uio_resid -= iov->iov_len;
467 				uio->uio_loffset += iov->iov_len;
468 				break;
469 			}
470 			/* else it's a write, fall through to NULL case */
471 			/*FALLTHROUGH*/
472 
473 		case M_NULL:
474 			if (rw == UIO_READ)
475 				return (0);
476 			c = iov->iov_len;
477 			iov->iov_base += c;
478 			iov->iov_len -= c;
479 			uio->uio_loffset += c;
480 			uio->uio_resid -= c;
481 			break;
482 
483 		}
484 	}
485 	return (uio->uio_resid == oresid ? error : 0);
486 }
487 
488 static int
489 mmread(dev_t dev, struct uio *uio, cred_t *cred)
490 {
491 	return (mmrw(dev, uio, UIO_READ, cred));
492 }
493 
494 static int
495 mmwrite(dev_t dev, struct uio *uio, cred_t *cred)
496 {
497 	return (mmrw(dev, uio, UIO_WRITE, cred));
498 }
499 
500 /*
501  * Private ioctl for libkvm to support kvm_physaddr().
502  * Given an address space and a VA, compute the PA.
503  */
504 static int
505 mmioctl_vtop(intptr_t data)
506 {
507 #ifdef _SYSCALL32
508 	mem_vtop32_t vtop32;
509 #endif
510 	mem_vtop_t mem_vtop;
511 	proc_t *p;
512 	pfn_t pfn = (pfn_t)PFN_INVALID;
513 	pid_t pid = 0;
514 	struct as *as;
515 	struct seg *seg;
516 
517 	if (get_udatamodel() == DATAMODEL_NATIVE) {
518 		if (copyin((void *)data, &mem_vtop, sizeof (mem_vtop_t)))
519 			return (EFAULT);
520 	}
521 #ifdef _SYSCALL32
522 	else {
523 		if (copyin((void *)data, &vtop32, sizeof (mem_vtop32_t)))
524 			return (EFAULT);
525 		mem_vtop.m_as = (struct as *)(uintptr_t)vtop32.m_as;
526 		mem_vtop.m_va = (void *)(uintptr_t)vtop32.m_va;
527 
528 		if (mem_vtop.m_as != NULL)
529 			return (EINVAL);
530 	}
531 #endif
532 
533 	if (mem_vtop.m_as == &kas) {
534 		pfn = hat_getpfnum(kas.a_hat, mem_vtop.m_va);
535 	} else {
536 		if (mem_vtop.m_as == NULL) {
537 			/*
538 			 * Assume the calling process's address space if the
539 			 * caller didn't specify one.
540 			 */
541 			p = curthread->t_procp;
542 			if (p == NULL)
543 				return (EIO);
544 			mem_vtop.m_as = p->p_as;
545 		}
546 
547 		mutex_enter(&pidlock);
548 		for (p = practive; p != NULL; p = p->p_next) {
549 			if (p->p_as == mem_vtop.m_as) {
550 				pid = p->p_pid;
551 				break;
552 			}
553 		}
554 		mutex_exit(&pidlock);
555 		if (p == NULL)
556 			return (EIO);
557 		p = sprlock(pid);
558 		if (p == NULL)
559 			return (EIO);
560 		as = p->p_as;
561 		if (as == mem_vtop.m_as) {
562 			mutex_exit(&p->p_lock);
563 			AS_LOCK_ENTER(as, RW_READER);
564 			for (seg = AS_SEGFIRST(as); seg != NULL;
565 			    seg = AS_SEGNEXT(as, seg))
566 				if ((uintptr_t)mem_vtop.m_va -
567 				    (uintptr_t)seg->s_base < seg->s_size)
568 					break;
569 			if (seg != NULL)
570 				pfn = hat_getpfnum(as->a_hat, mem_vtop.m_va);
571 			AS_LOCK_EXIT(as);
572 			mutex_enter(&p->p_lock);
573 		}
574 		sprunlock(p);
575 	}
576 	mem_vtop.m_pfn = pfn;
577 	if (pfn == PFN_INVALID)
578 		return (EIO);
579 
580 	if (get_udatamodel() == DATAMODEL_NATIVE) {
581 		if (copyout(&mem_vtop, (void *)data, sizeof (mem_vtop_t)))
582 			return (EFAULT);
583 	}
584 #ifdef _SYSCALL32
585 	else {
586 		vtop32.m_pfn = mem_vtop.m_pfn;
587 		if (copyout(&vtop32, (void *)data, sizeof (mem_vtop32_t)))
588 			return (EFAULT);
589 	}
590 #endif
591 
592 	return (0);
593 }
594 
595 /*
596  * Given a PA, execute the given page retire command on it.
597  */
598 static int
599 mmioctl_page_retire(int cmd, intptr_t data)
600 {
601 	extern int page_retire_test(void);
602 	uint64_t pa;
603 
604 	if (copyin((void *)data, &pa, sizeof (uint64_t))) {
605 		return (EFAULT);
606 	}
607 
608 	switch (cmd) {
609 	case MEM_PAGE_ISRETIRED:
610 		return (page_retire_check(pa, NULL));
611 
612 	case MEM_PAGE_UNRETIRE:
613 		return (page_unretire(pa));
614 
615 	case MEM_PAGE_RETIRE:
616 		return (page_retire(pa, PR_FMA));
617 
618 	case MEM_PAGE_RETIRE_MCE:
619 		return (page_retire(pa, PR_MCE));
620 
621 	case MEM_PAGE_RETIRE_UE:
622 		return (page_retire(pa, PR_UE));
623 
624 	case MEM_PAGE_GETERRORS:
625 		{
626 			uint64_t page_errors;
627 			int rc = page_retire_check(pa, &page_errors);
628 			if (copyout(&page_errors, (void *)data,
629 			    sizeof (uint64_t))) {
630 				return (EFAULT);
631 			}
632 			return (rc);
633 		}
634 
635 	case MEM_PAGE_RETIRE_TEST:
636 		return (page_retire_test());
637 
638 	}
639 
640 	return (EINVAL);
641 }
642 
643 #ifdef __sparc
644 /*
645  * Given a syndrome, syndrome type, and address return the
646  * associated memory name in the provided data buffer.
647  */
648 static int
649 mmioctl_get_mem_name(intptr_t data)
650 {
651 	mem_name_t mem_name;
652 	void *buf;
653 	size_t bufsize;
654 	int len, err;
655 
656 	if ((bufsize = cpu_get_name_bufsize()) == 0)
657 		return (ENOTSUP);
658 
659 	if ((err = mm_read_mem_name(data, &mem_name)) < 0)
660 		return (err);
661 
662 	buf = kmem_alloc(bufsize, KM_SLEEP);
663 
664 	/*
665 	 * Call into cpu specific code to do the lookup.
666 	 */
667 	if ((err = cpu_get_mem_name(mem_name.m_synd, mem_name.m_type,
668 	    mem_name.m_addr, buf, bufsize, &len)) != 0) {
669 		kmem_free(buf, bufsize);
670 		return (err);
671 	}
672 
673 	if (len >= mem_name.m_namelen) {
674 		kmem_free(buf, bufsize);
675 		return (ENOSPC);
676 	}
677 
678 	if (copyoutstr(buf, (char *)mem_name.m_name,
679 	    mem_name.m_namelen, NULL) != 0) {
680 		kmem_free(buf, bufsize);
681 		return (EFAULT);
682 	}
683 
684 	kmem_free(buf, bufsize);
685 	return (0);
686 }
687 
688 /*
689  * Given a syndrome and address return information about the associated memory.
690  */
691 static int
692 mmioctl_get_mem_info(intptr_t data)
693 {
694 	mem_info_t mem_info;
695 	int err;
696 
697 	if (copyin((void *)data, &mem_info, sizeof (mem_info_t)))
698 		return (EFAULT);
699 
700 	if ((err = cpu_get_mem_info(mem_info.m_synd, mem_info.m_addr,
701 	    &mem_info.m_mem_size, &mem_info.m_seg_size, &mem_info.m_bank_size,
702 	    &mem_info.m_segments, &mem_info.m_banks, &mem_info.m_mcid)) != 0)
703 		return (err);
704 
705 	if (copyout(&mem_info, (void *)data, sizeof (mem_info_t)) != 0)
706 		return (EFAULT);
707 
708 	return (0);
709 }
710 
711 /*
712  * Given a memory name, return its associated serial id
713  */
714 static int
715 mmioctl_get_mem_sid(intptr_t data)
716 {
717 	mem_name_t mem_name;
718 	void *buf;
719 	void *name;
720 	size_t	name_len;
721 	size_t bufsize;
722 	int len, err;
723 
724 	if ((bufsize = cpu_get_name_bufsize()) == 0)
725 		return (ENOTSUP);
726 
727 	if ((err = mm_read_mem_name(data, &mem_name)) < 0)
728 		return (err);
729 
730 	buf = kmem_alloc(bufsize, KM_SLEEP);
731 
732 	if (mem_name.m_namelen > 1024)
733 		mem_name.m_namelen = 1024; /* cap at 1024 bytes */
734 
735 	name = kmem_alloc(mem_name.m_namelen, KM_SLEEP);
736 
737 	if ((err = copyinstr((char *)mem_name.m_name, (char *)name,
738 	    mem_name.m_namelen, &name_len)) != 0) {
739 		kmem_free(buf, bufsize);
740 		kmem_free(name, mem_name.m_namelen);
741 		return (err);
742 	}
743 
744 	/*
745 	 * Call into cpu specific code to do the lookup.
746 	 */
747 	if ((err = cpu_get_mem_sid(name, buf, bufsize, &len)) != 0) {
748 		kmem_free(buf, bufsize);
749 		kmem_free(name, mem_name.m_namelen);
750 		return (err);
751 	}
752 
753 	if (len > mem_name.m_sidlen) {
754 		kmem_free(buf, bufsize);
755 		kmem_free(name, mem_name.m_namelen);
756 		return (ENAMETOOLONG);
757 	}
758 
759 	if (copyoutstr(buf, (char *)mem_name.m_sid,
760 	    mem_name.m_sidlen, NULL) != 0) {
761 		kmem_free(buf, bufsize);
762 		kmem_free(name, mem_name.m_namelen);
763 		return (EFAULT);
764 	}
765 
766 	kmem_free(buf, bufsize);
767 	kmem_free(name, mem_name.m_namelen);
768 	return (0);
769 }
770 #endif	/* __sparc */
771 
772 /*
773  * Private ioctls for
774  *	libkvm to support kvm_physaddr().
775  *	FMA support for page_retire() and memory attribute information.
776  */
777 /*ARGSUSED*/
778 static int
779 mmioctl(dev_t dev, int cmd, intptr_t data, int flag, cred_t *cred, int *rvalp)
780 {
781 	if ((cmd == MEM_VTOP && getminor(dev) != M_KMEM) ||
782 	    (cmd != MEM_VTOP && getminor(dev) != M_MEM))
783 		return (ENXIO);
784 
785 	switch (cmd) {
786 	case MEM_VTOP:
787 		return (mmioctl_vtop(data));
788 
789 	case MEM_PAGE_RETIRE:
790 	case MEM_PAGE_ISRETIRED:
791 	case MEM_PAGE_UNRETIRE:
792 	case MEM_PAGE_RETIRE_MCE:
793 	case MEM_PAGE_RETIRE_UE:
794 	case MEM_PAGE_GETERRORS:
795 	case MEM_PAGE_RETIRE_TEST:
796 		return (mmioctl_page_retire(cmd, data));
797 
798 #ifdef __sparc
799 	case MEM_NAME:
800 		return (mmioctl_get_mem_name(data));
801 
802 	case MEM_INFO:
803 		return (mmioctl_get_mem_info(data));
804 
805 	case MEM_SID:
806 		return (mmioctl_get_mem_sid(data));
807 #else
808 	case MEM_NAME:
809 	case MEM_INFO:
810 	case MEM_SID:
811 		return (ENOTSUP);
812 #endif	/* __sparc */
813 	}
814 	return (ENXIO);
815 }
816 
817 /*ARGSUSED2*/
818 static int
819 mmmmap(dev_t dev, off_t off, int prot)
820 {
821 	pfn_t pf;
822 	struct memlist *pmem;
823 	minor_t minor = getminor(dev);
824 
825 	switch (minor) {
826 	case M_MEM:
827 		pf = btop(off);
828 		memlist_read_lock();
829 		for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
830 			if (pf >= BTOP(pmem->ml_address) &&
831 			    pf < BTOP(pmem->ml_address + pmem->ml_size)) {
832 				memlist_read_unlock();
833 				return (impl_obmem_pfnum(pf));
834 			}
835 		}
836 		memlist_read_unlock();
837 		break;
838 
839 	case M_KMEM:
840 	case M_ALLKMEM:
841 		/* no longer supported with KPR */
842 		return (-1);
843 
844 	case M_FULL:
845 	case M_ZERO:
846 		/*
847 		 * We shouldn't be mmap'ing to /dev/zero here as
848 		 * mmsegmap() should have already converted
849 		 * a mapping request for this device to a mapping
850 		 * using seg_vn for anonymous memory.
851 		 */
852 		break;
853 
854 	}
855 	return (-1);
856 }
857 
858 /*
859  * This function is called when a memory device is mmap'ed.
860  * Set up the mapping to the correct device driver.
861  */
862 static int
863 mmsegmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len,
864     uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
865 {
866 	struct segvn_crargs vn_a;
867 	struct segdev_crargs dev_a;
868 	int error;
869 	minor_t minor;
870 	off_t i;
871 
872 	minor = getminor(dev);
873 
874 	as_rangelock(as);
875 	/*
876 	 * No need to worry about vac alignment on /dev/zero
877 	 * since this is a "clone" object that doesn't yet exist.
878 	 */
879 	error = choose_addr(as, addrp, len, off,
880 	    (minor == M_MEM) || (minor == M_KMEM), flags);
881 	if (error != 0) {
882 		as_rangeunlock(as);
883 		return (error);
884 	}
885 
886 	switch (minor) {
887 	case M_MEM:
888 		/* /dev/mem cannot be mmap'ed with MAP_PRIVATE */
889 		if ((flags & MAP_TYPE) != MAP_SHARED) {
890 			as_rangeunlock(as);
891 			return (EINVAL);
892 		}
893 
894 		/*
895 		 * Check to ensure that the entire range is
896 		 * legal and we are not trying to map in
897 		 * more than the device will let us.
898 		 */
899 		for (i = 0; i < len; i += PAGESIZE) {
900 			if (mmmmap(dev, off + i, maxprot) == -1) {
901 				as_rangeunlock(as);
902 				return (ENXIO);
903 			}
904 		}
905 
906 		/*
907 		 * Use seg_dev segment driver for /dev/mem mapping.
908 		 */
909 		dev_a.mapfunc = mmmmap;
910 		dev_a.dev = dev;
911 		dev_a.offset = off;
912 		dev_a.type = (flags & MAP_TYPE);
913 		dev_a.prot = (uchar_t)prot;
914 		dev_a.maxprot = (uchar_t)maxprot;
915 		dev_a.hat_attr = 0;
916 
917 		/*
918 		 * Make /dev/mem mappings non-consistent since we can't
919 		 * alias pages that don't have page structs behind them,
920 		 * such as kernel stack pages. If someone mmap()s a kernel
921 		 * stack page and if we give them a tte with cv, a line from
922 		 * that page can get into both pages of the spitfire d$.
923 		 * But snoop from another processor will only invalidate
924 		 * the first page. This later caused kernel (xc_attention)
925 		 * to go into an infinite loop at pil 13 and no interrupts
926 		 * could come in. See 1203630.
927 		 *
928 		 */
929 		dev_a.hat_flags = HAT_LOAD_NOCONSIST;
930 		dev_a.devmap_data = NULL;
931 
932 		error = as_map(as, *addrp, len, segdev_create, &dev_a);
933 		break;
934 
935 	case M_ZERO:
936 		/*
937 		 * Use seg_vn segment driver for /dev/zero mapping.
938 		 * Passing in a NULL amp gives us the "cloning" effect.
939 		 */
940 		vn_a.vp = NULL;
941 		vn_a.offset = 0;
942 		vn_a.type = (flags & MAP_TYPE);
943 		vn_a.prot = prot;
944 		vn_a.maxprot = maxprot;
945 		vn_a.flags = flags & ~MAP_TYPE;
946 		vn_a.cred = cred;
947 		vn_a.amp = NULL;
948 		vn_a.szc = 0;
949 		vn_a.lgrp_mem_policy_flags = 0;
950 		error = as_map(as, *addrp, len, segvn_create, &vn_a);
951 		break;
952 
953 	case M_KMEM:
954 	case M_ALLKMEM:
955 		/* No longer supported with KPR. */
956 		error = ENXIO;
957 		break;
958 
959 	case M_NULL:
960 		/*
961 		 * Use seg_dev segment driver for /dev/null mapping.
962 		 */
963 		dev_a.mapfunc = mmmmap;
964 		dev_a.dev = dev;
965 		dev_a.offset = off;
966 		dev_a.type = 0;		/* neither PRIVATE nor SHARED */
967 		dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
968 		dev_a.hat_attr = 0;
969 		dev_a.hat_flags = 0;
970 		error = as_map(as, *addrp, len, segdev_create, &dev_a);
971 		break;
972 
973 	default:
974 		error = ENXIO;
975 	}
976 
977 	as_rangeunlock(as);
978 	return (error);
979 }
980 
981 static struct cb_ops mm_cb_ops = {
982 	mmopen,			/* open */
983 	nulldev,		/* close */
984 	nodev,			/* strategy */
985 	nodev,			/* print */
986 	nodev,			/* dump */
987 	mmread,			/* read */
988 	mmwrite,		/* write */
989 	mmioctl,		/* ioctl */
990 	nodev,			/* devmap */
991 	mmmmap,			/* mmap */
992 	mmsegmap,		/* segmap */
993 	mmchpoll,		/* poll */
994 	mmpropop,		/* prop_op */
995 	0,			/* streamtab  */
996 	D_NEW | D_MP | D_64BIT | D_U64BIT
997 };
998 
999 static struct dev_ops mm_ops = {
1000 	DEVO_REV,		/* devo_rev, */
1001 	0,			/* refcnt  */
1002 	mm_info,		/* get_dev_info */
1003 	nulldev,		/* identify */
1004 	nulldev,		/* probe */
1005 	mm_attach,		/* attach */
1006 	nodev,			/* detach */
1007 	nodev,			/* reset */
1008 	&mm_cb_ops,		/* driver operations */
1009 	(struct bus_ops *)0,	/* bus operations */
1010 	NULL,			/* power */
1011 	ddi_quiesce_not_needed,		/* quiesce */
1012 };
1013 
1014 static struct modldrv modldrv = {
1015 	&mod_driverops, "memory driver", &mm_ops,
1016 };
1017 
1018 static struct modlinkage modlinkage = {
1019 	MODREV_1, &modldrv, NULL
1020 };
1021 
1022 int
1023 _init(void)
1024 {
1025 	return (mod_install(&modlinkage));
1026 }
1027 
1028 int
1029 _info(struct modinfo *modinfop)
1030 {
1031 	return (mod_info(&modlinkage, modinfop));
1032 }
1033 
1034 int
1035 _fini(void)
1036 {
1037 	return (mod_remove(&modlinkage));
1038 }
1039 
1040 static int
1041 mm_kstat_update(kstat_t *ksp, int rw)
1042 {
1043 	struct memlist *pmem;
1044 	uint_t count;
1045 
1046 	if (rw == KSTAT_WRITE)
1047 		return (EACCES);
1048 
1049 	count = 0;
1050 	memlist_read_lock();
1051 	for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
1052 		count++;
1053 	}
1054 	memlist_read_unlock();
1055 
1056 	ksp->ks_ndata = count;
1057 	ksp->ks_data_size = count * 2 * sizeof (uint64_t);
1058 
1059 	return (0);
1060 }
1061 
1062 static int
1063 mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw)
1064 {
1065 	struct memlist *pmem;
1066 	struct memunit {
1067 		uint64_t address;
1068 		uint64_t size;
1069 	} *kspmem;
1070 
1071 	if (rw == KSTAT_WRITE)
1072 		return (EACCES);
1073 
1074 	ksp->ks_snaptime = gethrtime();
1075 
1076 	kspmem = (struct memunit *)buf;
1077 	memlist_read_lock();
1078 	for (pmem = phys_install; pmem != NULL;
1079 	    pmem = pmem->ml_next, kspmem++) {
1080 		if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size)
1081 			break;
1082 		kspmem->address = pmem->ml_address;
1083 		kspmem->size = pmem->ml_size;
1084 	}
1085 	memlist_read_unlock();
1086 
1087 	return (0);
1088 }
1089 
1090 /*
1091  * Read a mem_name_t from user-space and store it in the mem_name_t
1092  * pointed to by the mem_name argument.
1093  */
1094 static int
1095 mm_read_mem_name(intptr_t data, mem_name_t *mem_name)
1096 {
1097 	if (get_udatamodel() == DATAMODEL_NATIVE) {
1098 		if (copyin((void *)data, mem_name, sizeof (mem_name_t)))
1099 			return (EFAULT);
1100 	}
1101 #ifdef	_SYSCALL32
1102 	else {
1103 		mem_name32_t mem_name32;
1104 
1105 		if (copyin((void *)data, &mem_name32, sizeof (mem_name32_t)))
1106 			return (EFAULT);
1107 		mem_name->m_addr = mem_name32.m_addr;
1108 		mem_name->m_synd = mem_name32.m_synd;
1109 		mem_name->m_type[0] = mem_name32.m_type[0];
1110 		mem_name->m_type[1] = mem_name32.m_type[1];
1111 		mem_name->m_name = (caddr_t)(uintptr_t)mem_name32.m_name;
1112 		mem_name->m_namelen = (size_t)mem_name32.m_namelen;
1113 		mem_name->m_sid = (caddr_t)(uintptr_t)mem_name32.m_sid;
1114 		mem_name->m_sidlen = (size_t)mem_name32.m_sidlen;
1115 	}
1116 #endif	/* _SYSCALL32 */
1117 
1118 	return (0);
1119 }
1120