1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #include <sys/types.h>
29 #include <sys/sysmacros.h>
30 #include <sys/systm.h>
31 #include <sys/mman.h>
32 #include <sys/buf.h>
33 #include <sys/vmem.h>
34 #include <sys/cmn_err.h>
35 #include <sys/debug.h>
36 #include <sys/machparam.h>
37 #include <vm/page.h>
38 #include <vm/seg_kmem.h>
39 #include <vm/seg_kpm.h>
40
41 #ifdef __sparc
42 #include <sys/cpu_module.h>
43 #define BP_FLUSH(addr, size) flush_instr_mem((void *)addr, size);
44 #else
45 #define BP_FLUSH(addr, size)
46 #endif
47
48 int bp_force_copy = 0;
49 typedef enum {
50 BP_COPYIN = 0,
51 BP_COPYOUT = 1
52 } bp_copydir_t;
53 static int bp_copy_common(bp_copydir_t dir, struct buf *bp, void *driverbuf,
54 offset_t offset, size_t size);
55
56 static vmem_t *bp_map_arena;
57 static size_t bp_align;
58 static uint_t bp_devload_flags = PROT_READ | PROT_WRITE | HAT_NOSYNC;
59 int bp_max_cache = 1 << 17; /* 128K default; tunable */
60 int bp_mapin_kpm_enable = 1; /* enable default; tunable */
61
62 static void *
bp_vmem_alloc(vmem_t * vmp,size_t size,int vmflag)63 bp_vmem_alloc(vmem_t *vmp, size_t size, int vmflag)
64 {
65 return (vmem_xalloc(vmp, size, bp_align, 0, 0, NULL, NULL, vmflag));
66 }
67
68 void
bp_init(size_t align,uint_t devload_flags)69 bp_init(size_t align, uint_t devload_flags)
70 {
71 bp_align = MAX(align, PAGESIZE);
72 bp_devload_flags |= devload_flags;
73
74 if (bp_align <= bp_max_cache)
75 bp_map_arena = vmem_create("bp_map", NULL, 0, bp_align,
76 bp_vmem_alloc, vmem_free, heap_arena,
77 MIN(8 * bp_align, bp_max_cache), VM_SLEEP);
78 }
79
80 /*
81 * common routine so can be called with/without VM_SLEEP
82 */
83 void *
bp_mapin_common(struct buf * bp,int flag)84 bp_mapin_common(struct buf *bp, int flag)
85 {
86 struct as *as;
87 pfn_t pfnum;
88 page_t *pp;
89 page_t **pplist;
90 caddr_t kaddr;
91 caddr_t addr;
92 uintptr_t off;
93 size_t size;
94 pgcnt_t npages;
95 int color;
96
97 /* return if already mapped in, no pageio/physio, or physio to kas */
98 if ((bp->b_flags & B_REMAPPED) ||
99 !(bp->b_flags & (B_PAGEIO | B_PHYS)) ||
100 (((bp->b_flags & (B_PAGEIO | B_PHYS)) == B_PHYS) &&
101 ((bp->b_proc == NULL) || (bp->b_proc->p_as == &kas))))
102 return (bp->b_un.b_addr);
103
104 ASSERT((bp->b_flags & (B_PAGEIO | B_PHYS)) != (B_PAGEIO | B_PHYS));
105
106 addr = (caddr_t)bp->b_un.b_addr;
107 off = (uintptr_t)addr & PAGEOFFSET;
108 size = P2ROUNDUP(bp->b_bcount + off, PAGESIZE);
109 npages = btop(size);
110
111 /* Fastpath single page IO to locked memory by using kpm. */
112 if ((bp->b_flags & (B_SHADOW | B_PAGEIO)) && (npages == 1) &&
113 kpm_enable && bp_mapin_kpm_enable) {
114 if (bp->b_flags & B_SHADOW)
115 pp = *bp->b_shadow;
116 else
117 pp = bp->b_pages;
118 kaddr = hat_kpm_mapin(pp, NULL);
119 bp->b_un.b_addr = kaddr + off;
120 bp->b_flags |= B_REMAPPED;
121 return (bp->b_un.b_addr);
122 }
123
124 /*
125 * Allocate kernel virtual space for remapping.
126 */
127 color = bp_color(bp);
128 ASSERT(color < bp_align);
129
130 if (bp_map_arena != NULL) {
131 kaddr = (caddr_t)vmem_alloc(bp_map_arena,
132 P2ROUNDUP(color + size, bp_align), flag);
133 if (kaddr == NULL)
134 return (NULL);
135 kaddr += color;
136 } else {
137 kaddr = vmem_xalloc(heap_arena, size, bp_align, color,
138 0, NULL, NULL, flag);
139 if (kaddr == NULL)
140 return (NULL);
141 }
142
143 ASSERT(P2PHASE((uintptr_t)kaddr, bp_align) == color);
144
145 /*
146 * Map bp into the virtual space we just allocated.
147 */
148 if (bp->b_flags & B_PAGEIO) {
149 pp = bp->b_pages;
150 pplist = NULL;
151 } else if (bp->b_flags & B_SHADOW) {
152 pp = NULL;
153 pplist = bp->b_shadow;
154 } else {
155 pp = NULL;
156 pplist = NULL;
157 if (bp->b_proc == NULL || (as = bp->b_proc->p_as) == NULL)
158 as = &kas;
159 }
160
161 bp->b_flags |= B_REMAPPED;
162 bp->b_un.b_addr = kaddr + off;
163
164 while (npages-- != 0) {
165 if (pp) {
166 pfnum = pp->p_pagenum;
167 pp = pp->p_next;
168 } else if (pplist == NULL) {
169 pfnum = hat_getpfnum(as->a_hat,
170 (caddr_t)((uintptr_t)addr & MMU_PAGEMASK));
171 if (pfnum == PFN_INVALID)
172 panic("bp_mapin_common: hat_getpfnum for"
173 " addr %p failed\n", (void *)addr);
174 addr += PAGESIZE;
175 } else {
176 pfnum = (*pplist)->p_pagenum;
177 pplist++;
178 }
179
180 hat_devload(kas.a_hat, kaddr, PAGESIZE, pfnum,
181 bp_devload_flags, HAT_LOAD_LOCK);
182
183 kaddr += PAGESIZE;
184 }
185 return (bp->b_un.b_addr);
186 }
187
188 /*
189 * Convert bp for pageio/physio to a kernel addressable location.
190 */
191 void
bp_mapin(struct buf * bp)192 bp_mapin(struct buf *bp)
193 {
194 (void) bp_mapin_common(bp, VM_SLEEP);
195 }
196
197 /*
198 * Release all the resources associated with a previous bp_mapin() call.
199 */
200 void
bp_mapout(struct buf * bp)201 bp_mapout(struct buf *bp)
202 {
203 caddr_t addr;
204 uintptr_t off;
205 uintptr_t base;
206 uintptr_t color;
207 size_t size;
208 pgcnt_t npages;
209 page_t *pp;
210
211 if ((bp->b_flags & B_REMAPPED) == 0)
212 return;
213
214 addr = bp->b_un.b_addr;
215 off = (uintptr_t)addr & PAGEOFFSET;
216 size = P2ROUNDUP(bp->b_bcount + off, PAGESIZE);
217 npages = btop(size);
218
219 bp->b_un.b_addr = (caddr_t)off; /* debugging aid */
220
221 if ((bp->b_flags & (B_SHADOW | B_PAGEIO)) && (npages == 1) &&
222 kpm_enable && bp_mapin_kpm_enable) {
223 if (bp->b_flags & B_SHADOW)
224 pp = *bp->b_shadow;
225 else
226 pp = bp->b_pages;
227 addr = (caddr_t)((uintptr_t)addr & MMU_PAGEMASK);
228 hat_kpm_mapout(pp, NULL, addr);
229 bp->b_flags &= ~B_REMAPPED;
230 return;
231 }
232
233 base = (uintptr_t)addr & MMU_PAGEMASK;
234 BP_FLUSH(base, size);
235 hat_unload(kas.a_hat, (void *)base, size,
236 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
237 if (bp_map_arena != NULL) {
238 color = P2PHASE(base, bp_align);
239 vmem_free(bp_map_arena, (void *)(base - color),
240 P2ROUNDUP(color + size, bp_align));
241 } else
242 vmem_free(heap_arena, (void *)base, size);
243 bp->b_flags &= ~B_REMAPPED;
244 }
245
246 /*
247 * copy data from a KVA into a buf_t which may not be mapped in. offset
248 * is relative to the buf_t only.
249 */
250 int
bp_copyout(void * driverbuf,struct buf * bp,offset_t offset,size_t size)251 bp_copyout(void *driverbuf, struct buf *bp, offset_t offset, size_t size)
252 {
253 return (bp_copy_common(BP_COPYOUT, bp, driverbuf, offset, size));
254 }
255
256 /*
257 * copy data from a buf_t which may not be mapped in, into a KVA.. offset
258 * is relative to the buf_t only.
259 */
260 int
bp_copyin(struct buf * bp,void * driverbuf,offset_t offset,size_t size)261 bp_copyin(struct buf *bp, void *driverbuf, offset_t offset, size_t size)
262 {
263 return (bp_copy_common(BP_COPYIN, bp, driverbuf, offset, size));
264 }
265
266
267 #define BP_COPY(dir, driverbuf, baddr, sz) \
268 (dir == BP_COPYIN) ? \
269 bcopy(baddr, driverbuf, sz) : bcopy(driverbuf, baddr, sz)
270
271 static int
bp_copy_common(bp_copydir_t dir,struct buf * bp,void * driverbuf,offset_t offset,size_t size)272 bp_copy_common(bp_copydir_t dir, struct buf *bp, void *driverbuf,
273 offset_t offset, size_t size)
274 {
275 page_t **pplist;
276 uintptr_t poff;
277 uintptr_t voff;
278 struct as *as;
279 caddr_t kaddr;
280 caddr_t addr;
281 page_t *page;
282 size_t psize;
283 page_t *pp;
284 pfn_t pfn;
285
286
287 ASSERT((offset + size) <= bp->b_bcount);
288
289 /* if the buf_t already has a KVA, just do a bcopy */
290 if (!(bp->b_flags & (B_PHYS | B_PAGEIO))) {
291 BP_COPY(dir, driverbuf, bp->b_un.b_addr + offset, size);
292 return (0);
293 }
294
295 /* if we don't have kpm enabled, we need to do the slow path */
296 if (!kpm_enable || bp_force_copy) {
297 bp_mapin(bp);
298 BP_COPY(dir, driverbuf, bp->b_un.b_addr + offset, size);
299 bp_mapout(bp);
300 return (0);
301 }
302
303 /*
304 * kpm is enabled, and we need to map in the buf_t for the copy
305 */
306
307 /* setup pp, plist, and make sure 'as' is right */
308 if (bp->b_flags & B_PAGEIO) {
309 pp = bp->b_pages;
310 pplist = NULL;
311 } else if (bp->b_flags & B_SHADOW) {
312 pp = NULL;
313 pplist = bp->b_shadow;
314 } else {
315 pp = NULL;
316 pplist = NULL;
317 if (bp->b_proc == NULL || (as = bp->b_proc->p_as) == NULL) {
318 as = &kas;
319 }
320 }
321
322 /*
323 * locals for the address, the offset into the first page, and the
324 * size of the first page we are going to copy.
325 */
326 addr = (caddr_t)bp->b_un.b_addr;
327 poff = (uintptr_t)addr & PAGEOFFSET;
328 psize = MIN(PAGESIZE - poff, size);
329
330 /*
331 * we always start with a 0 offset into the driverbuf provided. The
332 * offset passed in only applies to the buf_t.
333 */
334 voff = 0;
335
336 /* Loop until we've copied al the data */
337 while (size > 0) {
338
339 /*
340 * for a pp or pplist, get the pfn, then go to the next page_t
341 * for the next time around the loop.
342 */
343 if (pp) {
344 page = pp;
345 pp = pp->p_next;
346 } else if (pplist != NULL) {
347 page = (*pplist);
348 pplist++;
349
350 /*
351 * We have a user VA. If we are going to copy this page, (e.g.
352 * the offset into the buf_t where we start to copy is
353 * within this page), get the pfn. Don't waste the cycles
354 * getting the pfn if we're not copying this page.
355 */
356 } else if (offset < psize) {
357 pfn = hat_getpfnum(as->a_hat,
358 (caddr_t)((uintptr_t)addr & PAGEMASK));
359 if (pfn == PFN_INVALID) {
360 return (-1);
361 }
362 page = page_numtopp_nolock(pfn);
363 addr += psize - offset;
364 } else {
365 addr += psize;
366 }
367
368 /*
369 * if we have an initial offset into the buf_t passed in,
370 * and it falls within the current page, account for it in
371 * the page size (how much we will copy) and the offset into the
372 * page (where we'll start copying from).
373 */
374 if ((offset > 0) && (offset < psize)) {
375 psize -= offset;
376 poff += offset;
377 offset = 0;
378
379 /*
380 * if we have an initial offset into the buf_t passed in,
381 * and it's not within the current page, skip this page.
382 * We don't have to worry about the first page offset and size
383 * anymore. psize will normally be PAGESIZE now unless we are
384 * on the last page.
385 */
386 } else if (offset >= psize) {
387 offset -= psize;
388 psize = MIN(PAGESIZE, size);
389 poff = 0;
390 continue;
391 }
392
393 /*
394 * get a kpm mapping to the page, them copy in/out of the
395 * page. update size left and offset into the driverbuf passed
396 * in for the next time around the loop.
397 */
398 kaddr = hat_kpm_mapin(page, NULL) + poff;
399 BP_COPY(dir, (void *)((uintptr_t)driverbuf + voff), kaddr,
400 psize);
401 hat_kpm_mapout(page, NULL, kaddr - poff);
402
403 size -= psize;
404 voff += psize;
405
406 poff = 0;
407 psize = MIN(PAGESIZE, size);
408 }
409
410 return (0);
411 }
412