1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright (c) 2014 by Delphix. All rights reserved. 27 */ 28 29 #ifndef _VM_HAT_I86_H 30 #define _VM_HAT_I86_H 31 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 37 /* 38 * VM - Hardware Address Translation management. 39 * 40 * This file describes the contents of the x86_64 HAT data structures. 41 */ 42 #include <sys/types.h> 43 #include <sys/t_lock.h> 44 #include <sys/cpuvar.h> 45 #include <sys/x_call.h> 46 #include <vm/seg.h> 47 #include <vm/page.h> 48 #include <sys/vmparam.h> 49 #include <sys/vm_machparam.h> 50 #include <sys/promif.h> 51 #include <vm/hat_pte.h> 52 #include <vm/htable.h> 53 #include <vm/hment.h> 54 55 /* 56 * The essential data types involved: 57 * 58 * htable_t - There is one of these for each page table and it is used 59 * by the HAT to manage the page table. 60 * 61 * hment_t - Links together multiple PTEs to a single page. 62 */ 63 64 /* 65 * VLP processes have a 32 bit address range, so their top level is 2 and 66 * with only 4 PTEs in that table. 67 */ 68 #define VLP_LEVEL (2) 69 #define VLP_NUM_PTES (4) 70 #define VLP_SIZE (VLP_NUM_PTES * sizeof (x86pte_t)) 71 #define TOP_LEVEL(h) (((h)->hat_flags & HAT_VLP) ? VLP_LEVEL : mmu.max_level) 72 #define VLP_COPY(fromptep, toptep) { \ 73 toptep[0] = fromptep[0]; \ 74 toptep[1] = fromptep[1]; \ 75 toptep[2] = fromptep[2]; \ 76 toptep[3] = fromptep[3]; \ 77 } 78 79 /* 80 * The hat struct exists for each address space. 81 */ 82 struct hat { 83 kmutex_t hat_mutex; 84 struct as *hat_as; 85 uint_t hat_stats; 86 pgcnt_t hat_pages_mapped[MAX_PAGE_LEVEL + 1]; 87 pgcnt_t hat_ism_pgcnt; 88 cpuset_t hat_cpus; 89 uint16_t hat_flags; 90 htable_t *hat_htable; /* top level htable */ 91 struct hat *hat_next; 92 struct hat *hat_prev; 93 uint_t hat_num_hash; /* number of htable hash buckets */ 94 htable_t **hat_ht_hash; /* htable hash buckets */ 95 htable_t *hat_ht_cached; /* cached free htables */ 96 x86pte_t hat_vlp_ptes[VLP_NUM_PTES]; 97 #if defined(__amd64) && defined(__xpv) 98 pfn_t hat_user_ptable; /* alt top ptable for user mode */ 99 #endif 100 }; 101 typedef struct hat hat_t; 102 103 #define PGCNT_INC(hat, level) \ 104 atomic_inc_ulong(&(hat)->hat_pages_mapped[level]); 105 #define PGCNT_DEC(hat, level) \ 106 atomic_dec_ulong(&(hat)->hat_pages_mapped[level]); 107 108 /* 109 * Flags for the hat_flags field 110 * 111 * HAT_FREEING - set when HAT is being destroyed - mostly used to detect that 112 * demap()s can be avoided. 113 * 114 * HAT_VLP - indicates a 32 bit process has a virtual address range less than 115 * the hardware's physical address range. (VLP->Virtual Less-than Physical) 116 * Note - never used on the hypervisor. 117 * 118 * HAT_VICTIM - This is set while a hat is being examined for page table 119 * stealing and prevents it from being freed. 120 * 121 * HAT_SHARED - The hat has exported it's page tables via hat_share() 122 * 123 * HAT_PINNED - On the hypervisor, indicates the top page table has been pinned. 124 */ 125 #define HAT_FREEING (0x0001) 126 #define HAT_VLP (0x0002) 127 #define HAT_VICTIM (0x0004) 128 #define HAT_SHARED (0x0008) 129 #define HAT_PINNED (0x0010) 130 131 /* 132 * Additional platform attribute for hat_devload() to force no caching. 133 */ 134 #define HAT_PLAT_NOCACHE (0x100000) 135 136 /* 137 * Simple statistics for the HAT. These are just counters that are 138 * atomically incremented. They can be reset directly from the kernel 139 * debugger. 140 */ 141 struct hatstats { 142 ulong_t hs_reap_attempts; 143 ulong_t hs_reaped; 144 ulong_t hs_steals; 145 ulong_t hs_ptable_allocs; 146 ulong_t hs_ptable_frees; 147 ulong_t hs_htable_rgets; /* allocs from reserve */ 148 ulong_t hs_htable_rputs; /* putbacks to reserve */ 149 ulong_t hs_htable_shared; /* number of htables shared */ 150 ulong_t hs_htable_unshared; /* number of htables unshared */ 151 ulong_t hs_hm_alloc; 152 ulong_t hs_hm_free; 153 ulong_t hs_hm_put_reserve; 154 ulong_t hs_hm_get_reserve; 155 ulong_t hs_hm_steals; 156 ulong_t hs_hm_steal_exam; 157 ulong_t hs_tlb_inval_delayed; 158 }; 159 extern struct hatstats hatstat; 160 #ifdef DEBUG 161 #define HATSTAT_INC(x) (++hatstat.x) 162 #else 163 #define HATSTAT_INC(x) (0) 164 #endif 165 166 #if defined(_KERNEL) 167 168 /* 169 * Useful macro to align hat_XXX() address arguments to a page boundary 170 */ 171 #define ALIGN2PAGE(a) ((uintptr_t)(a) & MMU_PAGEMASK) 172 #define IS_PAGEALIGNED(a) (((uintptr_t)(a) & MMU_PAGEOFFSET) == 0) 173 174 extern uint_t khat_running; /* set at end of hat_kern_setup() */ 175 extern cpuset_t khat_cpuset; /* cpuset for kernal address demap Xcalls */ 176 extern kmutex_t hat_list_lock; 177 extern kcondvar_t hat_list_cv; 178 179 180 181 /* 182 * Interfaces to setup a cpu private mapping (ie. preemption disabled). 183 * The attr and flags arguments are the same as for hat_devload(). 184 * setup() must be called once, then any number of calls to remap(), 185 * followed by a final call to release() 186 * 187 * Used by ppcopy(), page_zero(), the memscrubber, and the kernel debugger. 188 */ 189 typedef paddr_t hat_mempte_t; /* phys addr of PTE */ 190 extern hat_mempte_t hat_mempte_setup(caddr_t addr); 191 extern void hat_mempte_remap(pfn_t, caddr_t, hat_mempte_t, 192 uint_t attr, uint_t flags); 193 extern void hat_mempte_release(caddr_t addr, hat_mempte_t); 194 195 /* 196 * Interfaces to manage which thread has access to htable and hment reserves. 197 * The USE_HAT_RESERVES macro should always be recomputed in full. Its value 198 * (due to curthread) can change after any call into kmem/vmem. 199 */ 200 extern uint_t can_steal_post_boot; 201 extern uint_t use_boot_reserve; 202 #define USE_HAT_RESERVES() \ 203 (use_boot_reserve || curthread->t_hatdepth > 1 || \ 204 panicstr != NULL || vmem_is_populator()) 205 206 /* 207 * initialization stuff needed by by startup, mp_startup... 208 */ 209 extern void hat_cpu_online(struct cpu *); 210 extern void hat_cpu_offline(struct cpu *); 211 extern void setup_vaddr_for_ppcopy(struct cpu *); 212 extern void teardown_vaddr_for_ppcopy(struct cpu *); 213 extern void clear_boot_mappings(uintptr_t, uintptr_t); 214 215 /* 216 * magic value to indicate that all TLB entries should be demapped. 217 */ 218 #define DEMAP_ALL_ADDR (~(uintptr_t)0) 219 220 /* 221 * not in any include file??? 222 */ 223 extern void halt(char *fmt); 224 225 /* 226 * x86 specific routines for use online in setup or i86pc/vm files 227 */ 228 extern void hat_kern_alloc(caddr_t segmap_base, size_t segmap_size, 229 caddr_t ekernelheap); 230 extern void hat_kern_setup(void); 231 extern void hat_tlb_inval(struct hat *hat, uintptr_t va); 232 extern void hat_pte_unmap(htable_t *ht, uint_t entry, uint_t flags, 233 x86pte_t old_pte, void *pte_ptr, boolean_t tlb); 234 extern void hat_init_finish(void); 235 extern caddr_t hat_kpm_pfn2va(pfn_t pfn); 236 extern pfn_t hat_kpm_va2pfn(caddr_t); 237 extern page_t *hat_kpm_vaddr2page(caddr_t); 238 extern uintptr_t hat_kernelbase(uintptr_t); 239 extern void hat_kmap_init(uintptr_t base, size_t len); 240 241 extern hment_t *hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry); 242 243 #if !defined(__xpv) 244 /* 245 * routines to deal with delayed TLB invalidations for idle CPUs 246 */ 247 extern void tlb_going_idle(void); 248 extern void tlb_service(void); 249 #endif 250 251 /* 252 * Hat switch function invoked to load a new context into %cr3 253 */ 254 extern void hat_switch(struct hat *hat); 255 256 #ifdef __xpv 257 /* 258 * Interfaces to use around code that maps/unmaps grant table references. 259 */ 260 extern void hat_prepare_mapping(hat_t *, caddr_t, uint64_t *); 261 extern void hat_release_mapping(hat_t *, caddr_t); 262 263 #define XPV_DISALLOW_MIGRATE() xen_block_migrate() 264 #define XPV_ALLOW_MIGRATE() xen_allow_migrate() 265 266 #else 267 268 #define XPV_DISALLOW_MIGRATE() /* nothing */ 269 #define XPV_ALLOW_MIGRATE() /* nothing */ 270 271 #define pfn_is_foreign(pfn) __lintzero 272 273 #endif 274 275 276 #endif /* _KERNEL */ 277 278 #ifdef __cplusplus 279 } 280 #endif 281 282 #endif /* _VM_HAT_I86_H */ 283