1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 #include <sys/types.h>
27 #include <sys/systm.h>
28 #include <sys/archsystm.h>
29 #include <sys/machparam.h>
30 #include <sys/machsystm.h>
31 #include <sys/cpu.h>
32 #include <sys/elf_SPARC.h>
33 #include <vm/hat_sfmmu.h>
34 #include <vm/page.h>
35 #include <vm/vm_dep.h>
36 #include <sys/cpuvar.h>
37 #include <sys/async.h>
38 #include <sys/cmn_err.h>
39 #include <sys/debug.h>
40 #include <sys/dditypes.h>
41 #include <sys/sunddi.h>
42 #include <sys/cpu_module.h>
43 #include <sys/prom_debug.h>
44 #include <sys/vmsystm.h>
45 #include <sys/prom_plat.h>
46 #include <sys/sysmacros.h>
47 #include <sys/intreg.h>
48 #include <sys/machtrap.h>
49 #include <sys/ontrap.h>
50 #include <sys/ivintr.h>
51 #include <sys/atomic.h>
52 #include <sys/panic.h>
53 #include <sys/dtrace.h>
54 #include <sys/simulate.h>
55 #include <sys/fault.h>
56 #include <sys/niagara2regs.h>
57 #include <sys/hsvc.h>
58 #include <sys/trapstat.h>
59 #include <sys/mutex_impl.h>
60
61 uint_t root_phys_addr_lo_mask = 0xffffffffU;
62 #if defined(NIAGARA2_IMPL)
63 char cpu_module_name[] = "SUNW,UltraSPARC-T2";
64 #elif defined(VFALLS_IMPL)
65 char cpu_module_name[] = "SUNW,UltraSPARC-T2+";
66 #elif defined(KT_IMPL)
67 char cpu_module_name[] = "SPARC-T3";
68 #endif
69
70 /*
71 * Hypervisor services information for the NIAGARA2 and Victoria Falls
72 * CPU module
73 */
74 static boolean_t cpu_hsvc_available = B_TRUE;
75 static uint64_t cpu_sup_minor; /* Supported minor number */
76 #if defined(NIAGARA2_IMPL)
77 static hsvc_info_t cpu_hsvc = {
78 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR,
79 NIAGARA2_HSVC_MINOR, cpu_module_name
80 };
81 #elif defined(VFALLS_IMPL)
82 static hsvc_info_t cpu_hsvc = {
83 HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR,
84 VFALLS_HSVC_MINOR, cpu_module_name
85 };
86 #elif defined(KT_IMPL)
87 static hsvc_info_t cpu_hsvc = {
88 HSVC_REV_1, NULL, HSVC_GROUP_KT_CPU, KT_HSVC_MAJOR,
89 KT_HSVC_MINOR, cpu_module_name
90 };
91 #endif
92
93 void
cpu_setup(void)94 cpu_setup(void)
95 {
96 extern int mmu_exported_pagesize_mask;
97 extern int cpc_has_overflow_intr;
98 extern size_t contig_mem_prealloc_base_size;
99 int status;
100
101 /*
102 * Negotiate the API version for Niagara2 specific hypervisor
103 * services.
104 */
105 status = hsvc_register(&cpu_hsvc, &cpu_sup_minor);
106 if (status != 0) {
107 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services "
108 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d",
109 cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group,
110 cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status);
111 cpu_hsvc_available = B_FALSE;
112 }
113
114 /*
115 * The setup common to all CPU modules is done in cpu_setup_common
116 * routine.
117 */
118 cpu_setup_common(NULL);
119
120 /*
121 * Initialize the cpu_hwcap_flags for N2 and VF if it is not already
122 * set in cpu_setup_common() by the hwcap MD info. Note that this MD
123 * info may not be available for N2/VF.
124 */
125 if (cpu_hwcap_flags == 0) {
126 #ifdef KT_IMPL
127 /*
128 * This should not happen since hwcap MD info is always
129 * available for KT platforms.
130 */
131 ASSERT(cpu_hwcap_flags != 0); /* panic in DEBUG mode */
132 cpu_hwcap_flags |= AV_SPARC_VIS3 | AV_SPARC_HPC | AV_SPARC_FMAF;
133 #endif /* KT_IMPL */
134 cpu_hwcap_flags |= AV_SPARC_VIS | AV_SPARC_VIS2 |
135 AV_SPARC_ASI_BLK_INIT | AV_SPARC_POPC;
136 }
137
138 cache |= (CACHE_PTAG | CACHE_IOCOHERENT);
139
140 if ((mmu_exported_pagesize_mask &
141 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) !=
142 DEFAULT_SUN4V_MMU_PAGESIZE_MASK)
143 cmn_err(CE_PANIC, "machine description"
144 " does not have required sun4v page sizes"
145 " 8K, 64K and 4M: MD mask is 0x%x",
146 mmu_exported_pagesize_mask);
147
148 /*
149 * Niagara2 supports a 48-bit subset of the full 64-bit virtual
150 * address space. Virtual addresses between 0x0000800000000000
151 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole"
152 * and must never be mapped. In addition, software must not use
153 * pages within 4GB of the VA hole as instruction pages to
154 * avoid problems with prefetching into the VA hole.
155 */
156 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32));
157 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32));
158
159 /*
160 * Niagara2 has a performance counter overflow interrupt
161 */
162 cpc_has_overflow_intr = 1;
163
164 /*
165 * Enable 4M pages for OOB.
166 */
167 max_uheap_lpsize = MMU_PAGESIZE4M;
168 max_ustack_lpsize = MMU_PAGESIZE4M;
169 max_privmap_lpsize = MMU_PAGESIZE4M;
170
171 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB
172 /*
173 * Use CPU Makefile specific compile time define (if exists)
174 * to add to the contig preallocation size.
175 */
176 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB);
177 #endif
178 }
179
180 /*
181 * Set the magic constants of the implementation.
182 */
183 void
cpu_fiximp(struct cpu_node * cpunode)184 cpu_fiximp(struct cpu_node *cpunode)
185 {
186 /*
187 * The Cache node is optional in MD. Therefore in case "Cache"
188 * node does not exists in MD, set the default L2 cache associativity,
189 * size, linesize.
190 */
191 if (cpunode->ecache_size == 0)
192 cpunode->ecache_size = L2CACHE_SIZE;
193 if (cpunode->ecache_linesize == 0)
194 cpunode->ecache_linesize = L2CACHE_LINESIZE;
195 if (cpunode->ecache_associativity == 0)
196 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY;
197 }
198
199 void
cpu_map_exec_units(struct cpu * cp)200 cpu_map_exec_units(struct cpu *cp)
201 {
202 ASSERT(MUTEX_HELD(&cpu_lock));
203
204 /*
205 * The cpu_ipipe and cpu_fpu fields are initialized based on
206 * the execution unit sharing information from the MD. They
207 * default to the CPU id in the absence of such information.
208 */
209 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping;
210 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND)
211 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id);
212
213 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping;
214 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND)
215 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id);
216
217 /*
218 * Niagara 2 defines the core to be at the FPU level
219 */
220 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu;
221
222 /*
223 * The cpu_chip field is initialized based on the information
224 * in the MD and assume that all cpus within a chip
225 * share the same L2 cache. If no such info is available, we
226 * set the cpu to belong to the defacto chip 0.
227 */
228 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping;
229 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND)
230 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID;
231
232 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping;
233 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND)
234 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID;
235 }
236
237 static int cpucnt;
238
239 void
cpu_init_private(struct cpu * cp)240 cpu_init_private(struct cpu *cp)
241 {
242 extern void niagara_kstat_init(void);
243
244 ASSERT(MUTEX_HELD(&cpu_lock));
245
246 cpu_map_exec_units(cp);
247
248 if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE))
249 (void) niagara_kstat_init();
250
251 mutex_delay = rdccr_delay;
252 }
253
254 /*ARGSUSED*/
255 void
cpu_uninit_private(struct cpu * cp)256 cpu_uninit_private(struct cpu *cp)
257 {
258 extern void niagara_kstat_fini(void);
259
260 ASSERT(MUTEX_HELD(&cpu_lock));
261 if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE))
262 (void) niagara_kstat_fini();
263 }
264
265 /*
266 * On Niagara2, any flush will cause all preceding stores to be
267 * synchronized wrt the i$, regardless of address or ASI. In fact,
268 * the address is ignored, so we always flush address 0.
269 */
270 /*ARGSUSED*/
271 void
dtrace_flush_sec(uintptr_t addr)272 dtrace_flush_sec(uintptr_t addr)
273 {
274 doflush(0);
275 }
276
277 /*
278 * Trapstat support for Niagara2 processor
279 * The Niagara2 provides HWTW support for TSB lookup and with HWTW
280 * enabled no TSB hit information will be available. Therefore setting
281 * the time spent in TLB miss handler for TSB hits to 0.
282 */
283 int
cpu_trapstat_conf(int cmd)284 cpu_trapstat_conf(int cmd)
285 {
286 int status = 0;
287
288 switch (cmd) {
289 case CPU_TSTATCONF_INIT:
290 case CPU_TSTATCONF_FINI:
291 case CPU_TSTATCONF_ENABLE:
292 case CPU_TSTATCONF_DISABLE:
293 break;
294 default:
295 status = EINVAL;
296 break;
297 }
298 return (status);
299 }
300
301 void
cpu_trapstat_data(void * buf,uint_t tstat_pgszs)302 cpu_trapstat_data(void *buf, uint_t tstat_pgszs)
303 {
304 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf;
305 int i;
306
307 for (i = 0; i < tstat_pgszs; i++, tstatp++) {
308 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0;
309 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0;
310 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0;
311 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0;
312 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
313 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
314 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
315 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
316 }
317 }
318
319 /*
320 * Page coloring support for hashed cache index mode
321 */
322
323 /*
324 * Node id bits from machine description (MD). Node id distinguishes
325 * local versus remote memory. Because of MPO, page allocation does
326 * not cross node boundaries. Therefore, remove the node id bits from
327 * the color, since they are fixed. Either bit 30, or 31:30 in
328 * Victoria Falls processors.
329 * The number of node id bits is always 0 in Niagara2.
330 */
331 typedef struct n2color {
332 uchar_t nnbits; /* number of node id bits */
333 uchar_t nnmask; /* mask for node id bits */
334 uchar_t lomask; /* mask for bits below node id */
335 uchar_t lobits; /* number of bits below node id */
336 } n2color_t;
337
338 n2color_t n2color[MMU_PAGE_SIZES];
339 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5};
340
341 /*
342 * Remove node id bits from color bits 32:28.
343 * This will reduce the number of colors.
344 * No change if number of node bits is zero.
345 */
346 static inline uint_t
n2_hash2color(uint_t color,uchar_t szc)347 n2_hash2color(uint_t color, uchar_t szc)
348 {
349 n2color_t m = n2color[szc];
350
351 if (m.nnbits > 0) {
352 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask);
353 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0);
354 }
355
356 return (color);
357 }
358
359 /*
360 * Restore node id bits into page color.
361 * This will increase the number of colors to match N2.
362 * No change if number of node bits is zero.
363 */
364 static inline uint_t
n2_color2hash(uint_t color,uchar_t szc,uint_t node)365 n2_color2hash(uint_t color, uchar_t szc, uint_t node)
366 {
367 n2color_t m = n2color[szc];
368
369 if (m.nnbits > 0) {
370 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask);
371 color |= (node & m.nnmask) << m.lobits;
372 }
373
374 return (color);
375 }
376
377 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */
378
379 /*
380 * iterator NULL means pfn is VA, do not adjust ra_to_pa
381 * iterator (-1) means pfn is RA, need to convert to PA
382 * iterator non-null means pfn is RA, use ra_to_pa
383 */
384 uint_t
page_pfn_2_color_cpu(pfn_t pfn,uchar_t szc,void * cookie)385 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie)
386 {
387 mem_node_iterator_t *it = cookie;
388 uint_t color;
389
390 ASSERT(szc <= TTE256M);
391
392 if (it == ((mem_node_iterator_t *)(-1))) {
393 pfn = plat_rapfn_to_papfn(pfn);
394 } else if (it != NULL) {
395 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end);
396 pfn = pfn + it->mi_ra_to_pa;
397 }
398 pfn = PFN_BASE(pfn, szc);
399 color = ((pfn >> 15) ^ pfn) & 0x1f;
400 if (szc < TTE4M) {
401 /* 19:18 */
402 color = (color << 2) | ((pfn >> 5) & 0x3);
403 if (szc > TTE64K)
404 color >>= 1; /* 19 */
405 }
406 return (n2_hash2color(color, szc));
407 }
408
409 static uint_t
page_papfn_2_color_cpu(pfn_t papfn,uchar_t szc)410 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc)
411 {
412 uint_t color;
413
414 ASSERT(szc <= TTE256M);
415
416 papfn = PFN_BASE(papfn, szc);
417 color = ((papfn >> 15) ^ papfn) & 0x1f;
418 if (szc < TTE4M) {
419 /* 19:18 */
420 color = (color << 2) | ((papfn >> 5) & 0x3);
421 if (szc > TTE64K)
422 color >>= 1; /* 19 */
423 }
424 return (color);
425 }
426
427 #if TTE256M != 5
428 #error TTE256M is not 5
429 #endif
430
431 uint_t
page_get_nsz_color_mask_cpu(uchar_t szc,uint_t mask)432 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask)
433 {
434 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f};
435 ASSERT(szc < TTE256M);
436 mask = n2_color2hash(mask, szc, 0);
437 mask &= ni2_color_masks[szc];
438 if (szc == TTE64K || szc == TTE512K)
439 mask >>= 1;
440 return (n2_hash2color(mask, szc + 1));
441 }
442
443 uint_t
page_get_nsz_color_cpu(uchar_t szc,uint_t color)444 page_get_nsz_color_cpu(uchar_t szc, uint_t color)
445 {
446 ASSERT(szc < TTE256M);
447 color = n2_color2hash(color, szc, 0);
448 if (szc == TTE64K || szc == TTE512K)
449 color >>= 1;
450 return (n2_hash2color(color, szc + 1));
451 }
452
453 uint_t
page_get_color_shift_cpu(uchar_t szc,uchar_t nszc)454 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc)
455 {
456 uint_t s;
457 ASSERT(nszc >= szc);
458 ASSERT(nszc <= TTE256M);
459
460 s = nhbits[szc] - n2color[szc].nnbits;
461 s -= nhbits[nszc] - n2color[nszc].nnbits;
462
463 return (s);
464 }
465
466 uint_t
page_convert_color_cpu(uint_t ncolor,uchar_t szc,uchar_t nszc)467 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc)
468 {
469 uint_t color;
470
471 ASSERT(nszc > szc);
472 ASSERT(nszc <= TTE256M);
473 ncolor = n2_color2hash(ncolor, nszc, 0);
474 color = ncolor << (nhbits[szc] - nhbits[nszc]);
475 color = n2_hash2color(color, szc);
476 return (color);
477 }
478
479 #define PAPFN_2_MNODE(pfn) \
480 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift)
481
482 /*ARGSUSED*/
483 pfn_t
page_next_pfn_for_color_cpu(pfn_t pfn,uchar_t szc,uint_t color,uint_t ceq_mask,uint_t color_mask,void * cookie)484 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color,
485 uint_t ceq_mask, uint_t color_mask, void *cookie)
486 {
487 mem_node_iterator_t *it = cookie;
488 pfn_t pstep = PNUM_SIZE(szc);
489 pfn_t npfn, pfn_ceq_mask, pfn_color;
490 pfn_t tmpmask, mask = (pfn_t)-1;
491 uint_t pfnmn;
492
493 ASSERT((color & ~ceq_mask) == 0);
494 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end);
495
496 /* convert RA to PA for accurate color calculation */
497 if (it->mi_init) {
498 /* first call after it, so cache these values */
499 it->mi_hash_ceq_mask =
500 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask);
501 it->mi_hash_color =
502 n2_color2hash(color, szc, it->mi_mnode);
503 it->mi_init = 0;
504 } else {
505 ASSERT(it->mi_hash_ceq_mask ==
506 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask));
507 ASSERT(it->mi_hash_color ==
508 n2_color2hash(color, szc, it->mi_mnode));
509 }
510 ceq_mask = it->mi_hash_ceq_mask;
511 color = it->mi_hash_color;
512 pfn += it->mi_ra_to_pa;
513
514 /* restart here when we switch memblocks */
515 next_mem_block:
516 pfnmn = PAPFN_2_MNODE(pfn);
517 if ((((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) &&
518 (pfnmn == it->mi_mnode)) {
519
520 /* we start from the page with correct color and mnode */
521 if (szc >= TTE512K) {
522 if (szc >= TTE4M) {
523 /* page color is PA[32:28] */
524 pfn_ceq_mask = ceq_mask << 15;
525 } else {
526 /* page color is PA[32:28].PA[19:19] */
527 pfn_ceq_mask = ((ceq_mask & 1) << 6) |
528 ((ceq_mask >> 1) << 15);
529 }
530 /*
531 * Preserve mnode bits in case they are not part of the
532 * color mask (eg., 8GB interleave, mnode bits 34:33).
533 */
534 pfn_ceq_mask |= it->mi_mnode_pfn_mask;
535 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
536 goto done;
537 } else {
538 /*
539 * We deal 64K or 8K page. Check if we could the
540 * satisfy the request without changing PA[32:28]
541 */
542 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
543 pfn_ceq_mask |= it->mi_mnode_pfn_mask;
544 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
545
546 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
547 goto done;
548
549 /*
550 * for next pfn we have to change bits PA[32:28]
551 * set PA[63:28] and PA[19:18] of the next pfn
552 */
553 npfn = (pfn >> 15) << 15;
554 npfn |= (ceq_mask & color & 3) << 5;
555 pfn_ceq_mask = (szc == TTE8K) ? 0 :
556 (ceq_mask & 0x1c) << 13;
557 pfn_ceq_mask |= it->mi_mnode_pfn_mask;
558 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask);
559
560 /*
561 * set bits PA[17:13] to match the color
562 */
563 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2);
564 goto done;
565 }
566 }
567
568 /*
569 * we start from the page with incorrect color - rare case
570 */
571 if (szc >= TTE512K) {
572 if (szc >= TTE4M) {
573 /* page color is in bits PA[32:28] */
574 npfn = ((pfn >> 20) << 20) | (color << 15);
575 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff;
576 } else {
577 /* try get the right color by changing bit PA[19:19] */
578 npfn = pfn + pstep;
579 pfnmn = PAPFN_2_MNODE(npfn);
580 if ((((page_papfn_2_color_cpu(npfn, szc) ^ color) &
581 ceq_mask) == 0) && (pfnmn == it->mi_mnode))
582 goto done;
583
584 /* page color is PA[32:28].PA[19:19] */
585 pfn_ceq_mask = ((ceq_mask & 1) << 6) |
586 ((ceq_mask >> 1) << 15) | (0xff << 7);
587 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15);
588 npfn = ((pfn >> 20) << 20) | pfn_color;
589 }
590
591 /* Fix mnode if necessary */
592 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode)
593 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) <<
594 it->mi_mnode_pfn_shift;
595
596 /*
597 * Preserve mnode bits in case they are not part of the color
598 * mask eg 8GB interleave, mnode bits 34:33).
599 */
600 pfn_ceq_mask |= it->mi_mnode_pfn_mask;
601 while (npfn <= pfn) {
602 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask);
603 }
604 goto done;
605 }
606
607 /*
608 * We deal 64K or 8K page of incorrect color.
609 * Try correcting color without changing PA[32:28]
610 */
611 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
612 pfn_color = ((color & 3) << 5) | (color >> 2);
613 if (pfnmn == it->mi_mnode) {
614 npfn = (pfn & ~(pfn_t)0x7f);
615 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
616 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
617
618 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) &
619 ceq_mask) == 0) {
620 /* the color is fixed - find the next page */
621 pfn_ceq_mask |= it->mi_mnode_pfn_mask;
622 while (npfn <= pfn) {
623 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask,
624 mask);
625 }
626 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
627 goto done;
628 }
629 }
630
631 /* to fix the color need to touch PA[32:28] */
632 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) :
633 (((pfn >> 18) << 18) | ((color & 0x1c) << 13));
634
635 /* fix mnode if input pfn is in the wrong mnode. */
636 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) {
637 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) <<
638 it->mi_mnode_pfn_shift;
639 }
640
641 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13;
642 tmpmask |= it->mi_mnode_pfn_mask;
643
644 while (npfn <= pfn) {
645 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask);
646 }
647
648 /* set bits PA[19:13] to match the color */
649 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
650 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
651
652 done:
653 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0);
654 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode);
655
656 /* PA to RA */
657 npfn -= it->mi_ra_to_pa;
658
659 /* check for possible memblock switch */
660 if (npfn > it->mi_mblock_end) {
661 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, szc, it,
662 0);
663 if (pfn == (pfn_t)-1)
664 return (pfn);
665 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end);
666 pfn += it->mi_ra_to_pa;
667 goto next_mem_block;
668 }
669
670 return (npfn);
671 }
672
673 /*
674 * init page coloring
675 * VF encodes node_id for an L-group in either bit 30 or 31:30,
676 * which effectively reduces the number of colors available per mnode.
677 */
678 void
page_coloring_init_cpu()679 page_coloring_init_cpu()
680 {
681 int i;
682 uchar_t id;
683 uchar_t lo;
684 uchar_t hi;
685 n2color_t m;
686 mem_node_iterator_t it;
687 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f};
688
689 for (i = 0; i < max_mem_nodes; i++) {
690 memset(&it, 0, sizeof (it));
691 if (plat_mem_node_iterator_init(0, i, 0, &it, 1) != (pfn_t)-1)
692 break;
693 }
694 ASSERT(i < max_mem_nodes);
695 for (i = 0; i < mmu_page_sizes; i++) {
696 (void) memset(&m, 0, sizeof (m));
697 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */
698 id &= idmask[i];
699 lo = lowbit(id);
700 if (lo > 0) {
701 hi = highbit(id);
702 m.nnbits = hi - lo + 1;
703 m.nnmask = (1 << m.nnbits) - 1;
704 lo += nhbits[i] - 5;
705 m.lomask = (1 << (lo - 1)) - 1;
706 m.lobits = lo - 1;
707 }
708 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits);
709 n2color[i] = m;
710 }
711 }
712
713 /*
714 * group colorequiv colors on N2 by low order bits of the color first
715 */
716 void
page_set_colorequiv_arr_cpu(void)717 page_set_colorequiv_arr_cpu(void)
718 {
719 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0};
720
721 nequiv_shades_log2[1] -= n2color[1].nnbits;
722 if (colorequiv > 1) {
723 int i;
724 uint_t sv_a = lowbit(colorequiv) - 1;
725
726 if (sv_a > 15)
727 sv_a = 15;
728
729 for (i = 0; i < MMU_PAGE_SIZES; i++) {
730 uint_t colors;
731 uint_t a = sv_a;
732
733 if ((colors = hw_page_array[i].hp_colors) <= 1)
734 continue;
735 while ((colors >> a) == 0)
736 a--;
737 if (a > (colorequivszc[i] & 0xf) +
738 (colorequivszc[i] >> 4)) {
739 if (a <= nequiv_shades_log2[i]) {
740 colorequivszc[i] = (uchar_t)a;
741 } else {
742 colorequivszc[i] =
743 ((a - nequiv_shades_log2[i]) << 4) |
744 nequiv_shades_log2[i];
745 }
746 }
747 }
748 }
749 }
750