| calling.h (39d64ee59ceee0fb61243eab3c4b7b4492f80df2) | calling.h (ea4654e0885348f0faa47f6d7b44a08d75ad16e9) |
|---|---|
| 1/* SPDX-License-Identifier: GPL-2.0 */ 2#include <linux/jump_label.h> 3#include <asm/unwind_hints.h> 4#include <asm/cpufeatures.h> 5#include <asm/page_types.h> 6#include <asm/percpu.h> 7#include <asm/asm-offsets.h> 8#include <asm/processor-flags.h> --- 128 unchanged lines hidden (view full) --- 137 popq %rcx 138 popq %rdx 139 popq %rsi 140 .if \pop_rdi 141 popq %rdi 142 .endif 143.endm 144 | 1/* SPDX-License-Identifier: GPL-2.0 */ 2#include <linux/jump_label.h> 3#include <asm/unwind_hints.h> 4#include <asm/cpufeatures.h> 5#include <asm/page_types.h> 6#include <asm/percpu.h> 7#include <asm/asm-offsets.h> 8#include <asm/processor-flags.h> --- 128 unchanged lines hidden (view full) --- 137 popq %rcx 138 popq %rdx 139 popq %rsi 140 .if \pop_rdi 141 popq %rdi 142 .endif 143.endm 144 |
| 145#ifdef CONFIG_PAGE_TABLE_ISOLATION | 145#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION |
| 146 147/* | 146 147/* |
| 148 * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two | 148 * MITIGATION_PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two |
| 149 * halves: 150 */ 151#define PTI_USER_PGTABLE_BIT PAGE_SHIFT 152#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) 153#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT 154#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) 155#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) 156 157.macro SET_NOFLUSH_BIT reg:req 158 bts $X86_CR3_PCID_NOFLUSH_BIT, \reg 159.endm 160 161.macro ADJUST_KERNEL_CR3 reg:req 162 ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID | 149 * halves: 150 */ 151#define PTI_USER_PGTABLE_BIT PAGE_SHIFT 152#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) 153#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT 154#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) 155#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) 156 157.macro SET_NOFLUSH_BIT reg:req 158 bts $X86_CR3_PCID_NOFLUSH_BIT, \reg 159.endm 160 161.macro ADJUST_KERNEL_CR3 reg:req 162 ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID |
| 163 /* Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ | 163 /* Clear PCID and "MITIGATION_PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ |
| 164 andq $(~PTI_USER_PGTABLE_AND_PCID_MASK), \reg 165.endm 166 167.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req 168 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 169 mov %cr3, \scratch_reg 170 ADJUST_KERNEL_CR3 \scratch_reg 171 mov \scratch_reg, %cr3 172.Lend_\@: 173.endm 174 175#define THIS_CPU_user_pcid_flush_mask \ | 164 andq $(~PTI_USER_PGTABLE_AND_PCID_MASK), \reg 165.endm 166 167.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req 168 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 169 mov %cr3, \scratch_reg 170 ADJUST_KERNEL_CR3 \scratch_reg 171 mov \scratch_reg, %cr3 172.Lend_\@: 173.endm 174 175#define THIS_CPU_user_pcid_flush_mask \ |
| 176 PER_CPU_VAR(cpu_tlbstate + TLB_STATE_user_pcid_flush_mask) | 176 PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask |
| 177 | 177 |
| 178.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req 179 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI | 178.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req |
| 180 mov %cr3, \scratch_reg 181 182 ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID 183 184 /* 185 * Test if the ASID needs a flush. 186 */ 187 movq \scratch_reg, \scratch_reg2 --- 13 unchanged lines hidden (view full) --- 201.Lwrcr3_pcid_\@: 202 /* Flip the ASID to the user version */ 203 orq $(PTI_USER_PCID_MASK), \scratch_reg 204 205.Lwrcr3_\@: 206 /* Flip the PGD to the user version */ 207 orq $(PTI_USER_PGTABLE_MASK), \scratch_reg 208 mov \scratch_reg, %cr3 | 179 mov %cr3, \scratch_reg 180 181 ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID 182 183 /* 184 * Test if the ASID needs a flush. 185 */ 186 movq \scratch_reg, \scratch_reg2 --- 13 unchanged lines hidden (view full) --- 200.Lwrcr3_pcid_\@: 201 /* Flip the ASID to the user version */ 202 orq $(PTI_USER_PCID_MASK), \scratch_reg 203 204.Lwrcr3_\@: 205 /* Flip the PGD to the user version */ 206 orq $(PTI_USER_PGTABLE_MASK), \scratch_reg 207 mov \scratch_reg, %cr3 |
| 208.endm 209 210.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req 211 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 212 SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2 |
|
| 209.Lend_\@: 210.endm 211 212.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req | 213.Lend_\@: 214.endm 215 216.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req |
| 217 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
|
| 213 pushq %rax | 218 pushq %rax |
| 214 SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax | 219 SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax |
| 215 popq %rax | 220 popq %rax |
| 221.Lend_\@: |
|
| 216.endm 217 218.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req 219 ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI 220 movq %cr3, \scratch_reg 221 movq \scratch_reg, \save_reg 222 /* 223 * Test the user pagetable bit. If set, then the user page tables --- 40 unchanged lines hidden (view full) --- 264 /* 265 * The CR3 write could be avoided when not changing its value, 266 * but would require a CR3 read *and* a scratch register. 267 */ 268 movq \save_reg, %cr3 269.Lend_\@: 270.endm 271 | 222.endm 223 224.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req 225 ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI 226 movq %cr3, \scratch_reg 227 movq \scratch_reg, \save_reg 228 /* 229 * Test the user pagetable bit. If set, then the user page tables --- 40 unchanged lines hidden (view full) --- 270 /* 271 * The CR3 write could be avoided when not changing its value, 272 * but would require a CR3 read *and* a scratch register. 273 */ 274 movq \save_reg, %cr3 275.Lend_\@: 276.endm 277 |
| 272#else /* CONFIG_PAGE_TABLE_ISOLATION=n: */ | 278#else /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION=n: */ |
| 273 274.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req 275.endm 276.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req 277.endm 278.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req 279.endm 280.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req --- 142 unchanged lines hidden --- | 279 280.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req 281.endm 282.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req 283.endm 284.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req 285.endm 286.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req --- 142 unchanged lines hidden --- |