17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ae115bc7Smrj * Common Development and Distribution License (the "License"). 6ae115bc7Smrj * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 227eea693dSMark Johnson * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 25a6a74e0eSMatthew Ahrens /* 26a6a74e0eSMatthew Ahrens * Copyright (c) 2014 by Delphix. All rights reserved. 27*74ecdb51SJohn Levon * Copyright 2018 Joyent, Inc. 28a6a74e0eSMatthew Ahrens */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #ifndef _VM_HAT_I86_H 317c478bd9Sstevel@tonic-gate #define _VM_HAT_I86_H 327c478bd9Sstevel@tonic-gate 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #ifdef __cplusplus 357c478bd9Sstevel@tonic-gate extern "C" { 367c478bd9Sstevel@tonic-gate #endif 377c478bd9Sstevel@tonic-gate 387c478bd9Sstevel@tonic-gate /* 397c478bd9Sstevel@tonic-gate * VM - Hardware Address Translation management. 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * This file describes the contents of the x86_64 HAT data structures. 427c478bd9Sstevel@tonic-gate */ 437c478bd9Sstevel@tonic-gate #include <sys/types.h> 447c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 457c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 467c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 477c478bd9Sstevel@tonic-gate #include <vm/seg.h> 487c478bd9Sstevel@tonic-gate #include <vm/page.h> 497c478bd9Sstevel@tonic-gate #include <sys/vmparam.h> 507c478bd9Sstevel@tonic-gate #include <sys/vm_machparam.h> 517c478bd9Sstevel@tonic-gate #include <sys/promif.h> 527c478bd9Sstevel@tonic-gate #include <vm/hat_pte.h> 537c478bd9Sstevel@tonic-gate #include <vm/htable.h> 547c478bd9Sstevel@tonic-gate #include <vm/hment.h> 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate /* 577c478bd9Sstevel@tonic-gate * The essential data types involved: 587c478bd9Sstevel@tonic-gate * 597c478bd9Sstevel@tonic-gate * htable_t - There is one of these for each page table and it is used 607c478bd9Sstevel@tonic-gate * by the HAT to manage the page table. 617c478bd9Sstevel@tonic-gate * 627c478bd9Sstevel@tonic-gate * hment_t - Links together multiple PTEs to a single page. 637c478bd9Sstevel@tonic-gate */ 647c478bd9Sstevel@tonic-gate 657c478bd9Sstevel@tonic-gate /* 66*74ecdb51SJohn Levon * Maximum number of per-CPU pagetable entries that we'll need to cache in the 67*74ecdb51SJohn Levon * HAT. See the big theory statement in uts/i86pc/vm/hat_i86.c for more 68*74ecdb51SJohn Levon * information. 697c478bd9Sstevel@tonic-gate */ 70*74ecdb51SJohn Levon #if defined(__xpv) 71*74ecdb51SJohn Levon /* 72*74ecdb51SJohn Levon * The Xen hypervisor does not use per-CPU pagetables (PCP). Define a single 73*74ecdb51SJohn Levon * struct member for it at least to make life easier and not make the member 74*74ecdb51SJohn Levon * conditional. 75*74ecdb51SJohn Levon */ 76*74ecdb51SJohn Levon #define MAX_COPIED_PTES 1 77*74ecdb51SJohn Levon #else 78*74ecdb51SJohn Levon /* 79*74ecdb51SJohn Levon * The 64-bit kernel may have up to 512 PTEs present in it for a given process. 80*74ecdb51SJohn Levon */ 81*74ecdb51SJohn Levon #define MAX_COPIED_PTES 512 82*74ecdb51SJohn Levon #endif /* __xpv */ 83*74ecdb51SJohn Levon 84*74ecdb51SJohn Levon #define TOP_LEVEL(h) (((h)->hat_max_level)) 857c478bd9Sstevel@tonic-gate 867c478bd9Sstevel@tonic-gate /* 877c478bd9Sstevel@tonic-gate * The hat struct exists for each address space. 887c478bd9Sstevel@tonic-gate */ 897c478bd9Sstevel@tonic-gate struct hat { 907c478bd9Sstevel@tonic-gate kmutex_t hat_mutex; 917c478bd9Sstevel@tonic-gate struct as *hat_as; 927c478bd9Sstevel@tonic-gate uint_t hat_stats; 937c478bd9Sstevel@tonic-gate pgcnt_t hat_pages_mapped[MAX_PAGE_LEVEL + 1]; 94250b7ff9Sjosephb pgcnt_t hat_ism_pgcnt; 957c478bd9Sstevel@tonic-gate cpuset_t hat_cpus; 967c478bd9Sstevel@tonic-gate uint16_t hat_flags; 97*74ecdb51SJohn Levon uint8_t hat_max_level; /* top level of this HAT */ 98*74ecdb51SJohn Levon uint_t hat_num_copied; /* Actual num of hat_copied_ptes[] */ 997c478bd9Sstevel@tonic-gate htable_t *hat_htable; /* top level htable */ 1007c478bd9Sstevel@tonic-gate struct hat *hat_next; 1017c478bd9Sstevel@tonic-gate struct hat *hat_prev; 1027c478bd9Sstevel@tonic-gate uint_t hat_num_hash; /* number of htable hash buckets */ 1037c478bd9Sstevel@tonic-gate htable_t **hat_ht_hash; /* htable hash buckets */ 1047c478bd9Sstevel@tonic-gate htable_t *hat_ht_cached; /* cached free htables */ 105*74ecdb51SJohn Levon x86pte_t hat_copied_ptes[MAX_COPIED_PTES]; 106843e1988Sjohnlev #if defined(__amd64) && defined(__xpv) 107843e1988Sjohnlev pfn_t hat_user_ptable; /* alt top ptable for user mode */ 108843e1988Sjohnlev #endif 1097c478bd9Sstevel@tonic-gate }; 1107c478bd9Sstevel@tonic-gate typedef struct hat hat_t; 1117c478bd9Sstevel@tonic-gate 1127c478bd9Sstevel@tonic-gate #define PGCNT_INC(hat, level) \ 1131a5e258fSJosef 'Jeff' Sipek atomic_inc_ulong(&(hat)->hat_pages_mapped[level]); 1147c478bd9Sstevel@tonic-gate #define PGCNT_DEC(hat, level) \ 1151a5e258fSJosef 'Jeff' Sipek atomic_dec_ulong(&(hat)->hat_pages_mapped[level]); 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate /* 118*74ecdb51SJohn Levon * Flags for the hat_flags field. For more information, please see the big 119*74ecdb51SJohn Levon * theory statement on the HAT design in uts/i86pc/vm/hat_i86.c. 1207c478bd9Sstevel@tonic-gate * 1217c478bd9Sstevel@tonic-gate * HAT_FREEING - set when HAT is being destroyed - mostly used to detect that 1227c478bd9Sstevel@tonic-gate * demap()s can be avoided. 1237c478bd9Sstevel@tonic-gate * 124*74ecdb51SJohn Levon * HAT_COPIED - Indicates this HAT is a source for per-cpu page tables: see the 125*74ecdb51SJohn Levon * big comment in hat_i86.c for a description. 126*74ecdb51SJohn Levon * 127*74ecdb51SJohn Levon * HAT_COPIED_32 - HAT_COPIED, but for an ILP32 process. 1287c478bd9Sstevel@tonic-gate * 1297c478bd9Sstevel@tonic-gate * HAT_VICTIM - This is set while a hat is being examined for page table 1307c478bd9Sstevel@tonic-gate * stealing and prevents it from being freed. 1317c478bd9Sstevel@tonic-gate * 1327c478bd9Sstevel@tonic-gate * HAT_SHARED - The hat has exported it's page tables via hat_share() 133843e1988Sjohnlev * 134843e1988Sjohnlev * HAT_PINNED - On the hypervisor, indicates the top page table has been pinned. 135*74ecdb51SJohn Levon * 136*74ecdb51SJohn Levon * HAT_PCP - Used for the per-cpu user page table (i.e. associated with a CPU, 137*74ecdb51SJohn Levon * not a process). 1387c478bd9Sstevel@tonic-gate */ 1397c478bd9Sstevel@tonic-gate #define HAT_FREEING (0x0001) 140*74ecdb51SJohn Levon #define HAT_VICTIM (0x0002) 141*74ecdb51SJohn Levon #define HAT_SHARED (0x0004) 142*74ecdb51SJohn Levon #define HAT_PINNED (0x0008) 143*74ecdb51SJohn Levon #define HAT_COPIED (0x0010) 144*74ecdb51SJohn Levon #define HAT_COPIED_32 (0x0020) 145*74ecdb51SJohn Levon #define HAT_PCP (0x0040) 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* 1487c478bd9Sstevel@tonic-gate * Additional platform attribute for hat_devload() to force no caching. 1497c478bd9Sstevel@tonic-gate */ 1507c478bd9Sstevel@tonic-gate #define HAT_PLAT_NOCACHE (0x100000) 1517c478bd9Sstevel@tonic-gate 1527c478bd9Sstevel@tonic-gate /* 1537c478bd9Sstevel@tonic-gate * Simple statistics for the HAT. These are just counters that are 1547c478bd9Sstevel@tonic-gate * atomically incremented. They can be reset directly from the kernel 1557c478bd9Sstevel@tonic-gate * debugger. 1567c478bd9Sstevel@tonic-gate */ 1577c478bd9Sstevel@tonic-gate struct hatstats { 15895c0a3c8Sjosephb ulong_t hs_reap_attempts; 15995c0a3c8Sjosephb ulong_t hs_reaped; 16095c0a3c8Sjosephb ulong_t hs_steals; 16195c0a3c8Sjosephb ulong_t hs_ptable_allocs; 16295c0a3c8Sjosephb ulong_t hs_ptable_frees; 16395c0a3c8Sjosephb ulong_t hs_htable_rgets; /* allocs from reserve */ 16495c0a3c8Sjosephb ulong_t hs_htable_rputs; /* putbacks to reserve */ 16595c0a3c8Sjosephb ulong_t hs_htable_shared; /* number of htables shared */ 16695c0a3c8Sjosephb ulong_t hs_htable_unshared; /* number of htables unshared */ 16795c0a3c8Sjosephb ulong_t hs_hm_alloc; 16895c0a3c8Sjosephb ulong_t hs_hm_free; 16995c0a3c8Sjosephb ulong_t hs_hm_put_reserve; 17095c0a3c8Sjosephb ulong_t hs_hm_get_reserve; 17195c0a3c8Sjosephb ulong_t hs_hm_steals; 17295c0a3c8Sjosephb ulong_t hs_hm_steal_exam; 17395c0a3c8Sjosephb ulong_t hs_tlb_inval_delayed; 174*74ecdb51SJohn Levon ulong_t hs_hat_copied64; 175*74ecdb51SJohn Levon ulong_t hs_hat_copied32; 176*74ecdb51SJohn Levon ulong_t hs_hat_normal64; 1777c478bd9Sstevel@tonic-gate }; 1787c478bd9Sstevel@tonic-gate extern struct hatstats hatstat; 17995c0a3c8Sjosephb #ifdef DEBUG 18095c0a3c8Sjosephb #define HATSTAT_INC(x) (++hatstat.x) 18195c0a3c8Sjosephb #else 18295c0a3c8Sjosephb #define HATSTAT_INC(x) (0) 18395c0a3c8Sjosephb #endif 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate #if defined(_KERNEL) 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate /* 1887c478bd9Sstevel@tonic-gate * Useful macro to align hat_XXX() address arguments to a page boundary 1897c478bd9Sstevel@tonic-gate */ 1907c478bd9Sstevel@tonic-gate #define ALIGN2PAGE(a) ((uintptr_t)(a) & MMU_PAGEMASK) 1917c478bd9Sstevel@tonic-gate #define IS_PAGEALIGNED(a) (((uintptr_t)(a) & MMU_PAGEOFFSET) == 0) 1927c478bd9Sstevel@tonic-gate 1937c478bd9Sstevel@tonic-gate extern uint_t khat_running; /* set at end of hat_kern_setup() */ 1947c478bd9Sstevel@tonic-gate extern cpuset_t khat_cpuset; /* cpuset for kernal address demap Xcalls */ 1957c478bd9Sstevel@tonic-gate extern kmutex_t hat_list_lock; 1967c478bd9Sstevel@tonic-gate extern kcondvar_t hat_list_cv; 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * Interfaces to setup a cpu private mapping (ie. preemption disabled). 2027c478bd9Sstevel@tonic-gate * The attr and flags arguments are the same as for hat_devload(). 2037c478bd9Sstevel@tonic-gate * setup() must be called once, then any number of calls to remap(), 2047c478bd9Sstevel@tonic-gate * followed by a final call to release() 2057c478bd9Sstevel@tonic-gate * 2067c478bd9Sstevel@tonic-gate * Used by ppcopy(), page_zero(), the memscrubber, and the kernel debugger. 2077c478bd9Sstevel@tonic-gate */ 208ae115bc7Smrj typedef paddr_t hat_mempte_t; /* phys addr of PTE */ 209ae115bc7Smrj extern hat_mempte_t hat_mempte_setup(caddr_t addr); 210ae115bc7Smrj extern void hat_mempte_remap(pfn_t, caddr_t, hat_mempte_t, 211ae115bc7Smrj uint_t attr, uint_t flags); 212ae115bc7Smrj extern void hat_mempte_release(caddr_t addr, hat_mempte_t); 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate /* 21597704650Sjosephb * Interfaces to manage which thread has access to htable and hment reserves. 21697704650Sjosephb * The USE_HAT_RESERVES macro should always be recomputed in full. Its value 21797704650Sjosephb * (due to curthread) can change after any call into kmem/vmem. 2187c478bd9Sstevel@tonic-gate */ 2197c478bd9Sstevel@tonic-gate extern uint_t can_steal_post_boot; 2207c478bd9Sstevel@tonic-gate extern uint_t use_boot_reserve; 22197704650Sjosephb #define USE_HAT_RESERVES() \ 222aac11643Sjosephb (use_boot_reserve || curthread->t_hatdepth > 1 || \ 22397704650Sjosephb panicstr != NULL || vmem_is_populator()) 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate /* 2267c478bd9Sstevel@tonic-gate * initialization stuff needed by by startup, mp_startup... 2277c478bd9Sstevel@tonic-gate */ 2287c478bd9Sstevel@tonic-gate extern void hat_cpu_online(struct cpu *); 229ae115bc7Smrj extern void hat_cpu_offline(struct cpu *); 2307c478bd9Sstevel@tonic-gate extern void setup_vaddr_for_ppcopy(struct cpu *); 231ae115bc7Smrj extern void teardown_vaddr_for_ppcopy(struct cpu *); 2327c478bd9Sstevel@tonic-gate extern void clear_boot_mappings(uintptr_t, uintptr_t); 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate /* 2357c478bd9Sstevel@tonic-gate * magic value to indicate that all TLB entries should be demapped. 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate #define DEMAP_ALL_ADDR (~(uintptr_t)0) 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * not in any include file??? 2417c478bd9Sstevel@tonic-gate */ 2427c478bd9Sstevel@tonic-gate extern void halt(char *fmt); 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate /* 2457c478bd9Sstevel@tonic-gate * x86 specific routines for use online in setup or i86pc/vm files 2467c478bd9Sstevel@tonic-gate */ 247ae115bc7Smrj extern void hat_kern_alloc(caddr_t segmap_base, size_t segmap_size, 248ae115bc7Smrj caddr_t ekernelheap); 249ae115bc7Smrj extern void hat_kern_setup(void); 2507c478bd9Sstevel@tonic-gate extern void hat_pte_unmap(htable_t *ht, uint_t entry, uint_t flags, 251a6a74e0eSMatthew Ahrens x86pte_t old_pte, void *pte_ptr, boolean_t tlb); 2527c478bd9Sstevel@tonic-gate extern void hat_init_finish(void); 2537c478bd9Sstevel@tonic-gate extern caddr_t hat_kpm_pfn2va(pfn_t pfn); 2547c478bd9Sstevel@tonic-gate extern pfn_t hat_kpm_va2pfn(caddr_t); 2557c478bd9Sstevel@tonic-gate extern page_t *hat_kpm_vaddr2page(caddr_t); 2567c478bd9Sstevel@tonic-gate extern uintptr_t hat_kernelbase(uintptr_t); 257ae115bc7Smrj extern void hat_kmap_init(uintptr_t base, size_t len); 2587c478bd9Sstevel@tonic-gate 2597c478bd9Sstevel@tonic-gate extern hment_t *hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry); 26095c0a3c8Sjosephb 261*74ecdb51SJohn Levon extern void mmu_calc_user_slots(void); 262*74ecdb51SJohn Levon extern void hat_tlb_inval(struct hat *hat, uintptr_t va); 2637c478bd9Sstevel@tonic-gate extern void hat_switch(struct hat *hat); 2647c478bd9Sstevel@tonic-gate 265*74ecdb51SJohn Levon #define TLB_RANGE_LEN(r) ((r)->tr_cnt << LEVEL_SHIFT((r)->tr_level)) 266*74ecdb51SJohn Levon 267*74ecdb51SJohn Levon /* 268*74ecdb51SJohn Levon * A range of virtual pages for purposes of demapping. 269*74ecdb51SJohn Levon */ 270*74ecdb51SJohn Levon typedef struct tlb_range { 271*74ecdb51SJohn Levon uintptr_t tr_va; /* address of page */ 272*74ecdb51SJohn Levon ulong_t tr_cnt; /* number of pages in range */ 273*74ecdb51SJohn Levon int8_t tr_level; /* page table level */ 274*74ecdb51SJohn Levon } tlb_range_t; 275*74ecdb51SJohn Levon 276*74ecdb51SJohn Levon #if defined(__xpv) 277*74ecdb51SJohn Levon 278*74ecdb51SJohn Levon #define XPV_DISALLOW_MIGRATE() xen_block_migrate() 279*74ecdb51SJohn Levon #define XPV_ALLOW_MIGRATE() xen_allow_migrate() 280*74ecdb51SJohn Levon 281*74ecdb51SJohn Levon #define mmu_flush_tlb_page(va) mmu_invlpg((caddr_t)va) 282*74ecdb51SJohn Levon #define mmu_flush_tlb_kpage(va) mmu_invlpg((caddr_t)va) 283*74ecdb51SJohn Levon 284843e1988Sjohnlev /* 285843e1988Sjohnlev * Interfaces to use around code that maps/unmaps grant table references. 286843e1988Sjohnlev */ 2877eea693dSMark Johnson extern void hat_prepare_mapping(hat_t *, caddr_t, uint64_t *); 288843e1988Sjohnlev extern void hat_release_mapping(hat_t *, caddr_t); 289843e1988Sjohnlev 290843e1988Sjohnlev #else 291843e1988Sjohnlev 292843e1988Sjohnlev #define XPV_DISALLOW_MIGRATE() /* nothing */ 293843e1988Sjohnlev #define XPV_ALLOW_MIGRATE() /* nothing */ 294843e1988Sjohnlev 295ae115bc7Smrj #define pfn_is_foreign(pfn) __lintzero 2967c478bd9Sstevel@tonic-gate 297*74ecdb51SJohn Levon typedef enum flush_tlb_type { 298*74ecdb51SJohn Levon FLUSH_TLB_ALL = 1, 299*74ecdb51SJohn Levon FLUSH_TLB_NONGLOBAL = 2, 300*74ecdb51SJohn Levon FLUSH_TLB_RANGE = 3, 301*74ecdb51SJohn Levon } flush_tlb_type_t; 302843e1988Sjohnlev 303*74ecdb51SJohn Levon extern void mmu_flush_tlb(flush_tlb_type_t, tlb_range_t *); 304*74ecdb51SJohn Levon extern void mmu_flush_tlb_kpage(uintptr_t); 305*74ecdb51SJohn Levon extern void mmu_flush_tlb_page(uintptr_t); 306*74ecdb51SJohn Levon 307*74ecdb51SJohn Levon extern void hati_cpu_punchin(cpu_t *cpu, uintptr_t va, uint_t attrs); 308*74ecdb51SJohn Levon 309*74ecdb51SJohn Levon /* 310*74ecdb51SJohn Levon * routines to deal with delayed TLB invalidations for idle CPUs 311*74ecdb51SJohn Levon */ 312*74ecdb51SJohn Levon extern void tlb_going_idle(void); 313*74ecdb51SJohn Levon extern void tlb_service(void); 314*74ecdb51SJohn Levon 315*74ecdb51SJohn Levon #endif /* !__xpv */ 316843e1988Sjohnlev 3177c478bd9Sstevel@tonic-gate #endif /* _KERNEL */ 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate #ifdef __cplusplus 3207c478bd9Sstevel@tonic-gate } 3217c478bd9Sstevel@tonic-gate #endif 3227c478bd9Sstevel@tonic-gate 3237c478bd9Sstevel@tonic-gate #endif /* _VM_HAT_I86_H */ 324