tlb.c (cf79f291f985662150363b4a93d16f88f12643bc) tlb.c (ea4654e0885348f0faa47f6d7b44a08d75ad16e9)
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/init.h>
3
4#include <linux/mm.h>
5#include <linux/spinlock.h>
6#include <linux/smp.h>
7#include <linux/interrupt.h>
8#include <linux/export.h>

--- 75 unchanged lines hidden (view full) ---

84 * for each mm. Corresponds to kPCID + 2048.
85 *
86 */
87
88/* There are 12 bits of space for ASIDS in CR3 */
89#define CR3_HW_ASID_BITS 12
90
91/*
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/init.h>
3
4#include <linux/mm.h>
5#include <linux/spinlock.h>
6#include <linux/smp.h>
7#include <linux/interrupt.h>
8#include <linux/export.h>

--- 75 unchanged lines hidden (view full) ---

84 * for each mm. Corresponds to kPCID + 2048.
85 *
86 */
87
88/* There are 12 bits of space for ASIDS in CR3 */
89#define CR3_HW_ASID_BITS 12
90
91/*
92 * When enabled, PAGE_TABLE_ISOLATION consumes a single bit for
92 * When enabled, MITIGATION_PAGE_TABLE_ISOLATION consumes a single bit for
93 * user/kernel switches
94 */
93 * user/kernel switches
94 */
95#ifdef CONFIG_PAGE_TABLE_ISOLATION
95#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
96# define PTI_CONSUMED_PCID_BITS 1
97#else
98# define PTI_CONSUMED_PCID_BITS 0
99#endif
100
101#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
102
103/*

--- 5 unchanged lines hidden (view full) ---

109
110/*
111 * Given @asid, compute kPCID
112 */
113static inline u16 kern_pcid(u16 asid)
114{
115 VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
116
96# define PTI_CONSUMED_PCID_BITS 1
97#else
98# define PTI_CONSUMED_PCID_BITS 0
99#endif
100
101#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
102
103/*

--- 5 unchanged lines hidden (view full) ---

109
110/*
111 * Given @asid, compute kPCID
112 */
113static inline u16 kern_pcid(u16 asid)
114{
115 VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
116
117#ifdef CONFIG_PAGE_TABLE_ISOLATION
117#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
118 /*
119 * Make sure that the dynamic ASID space does not conflict with the
120 * bit we are using to switch between user and kernel ASIDs.
121 */
122 BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));
123
124 /*
125 * The ASID being passed in here should have respected the

--- 18 unchanged lines hidden (view full) ---

144}
145
146/*
147 * Given @asid, compute uPCID
148 */
149static inline u16 user_pcid(u16 asid)
150{
151 u16 ret = kern_pcid(asid);
118 /*
119 * Make sure that the dynamic ASID space does not conflict with the
120 * bit we are using to switch between user and kernel ASIDs.
121 */
122 BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));
123
124 /*
125 * The ASID being passed in here should have respected the

--- 18 unchanged lines hidden (view full) ---

144}
145
146/*
147 * Given @asid, compute uPCID
148 */
149static inline u16 user_pcid(u16 asid)
150{
151 u16 ret = kern_pcid(asid);
152#ifdef CONFIG_PAGE_TABLE_ISOLATION
152#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
153 ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
154#endif
155 return ret;
156}
157
158static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
159{
160 unsigned long cr3 = __sme_pa(pgd) | lam;

--- 96 unchanged lines hidden (view full) ---

257 * Given an ASID, flush the corresponding user ASID. We can delay this
258 * until the next time we switch to it.
259 *
260 * See SWITCH_TO_USER_CR3.
261 */
262static inline void invalidate_user_asid(u16 asid)
263{
264 /* There is no user ASID if address space separation is off */
153 ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
154#endif
155 return ret;
156}
157
158static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
159{
160 unsigned long cr3 = __sme_pa(pgd) | lam;

--- 96 unchanged lines hidden (view full) ---

257 * Given an ASID, flush the corresponding user ASID. We can delay this
258 * until the next time we switch to it.
259 *
260 * See SWITCH_TO_USER_CR3.
261 */
262static inline void invalidate_user_asid(u16 asid)
263{
264 /* There is no user ASID if address space separation is off */
265 if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
265 if (!IS_ENABLED(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION))
266 return;
267
268 /*
269 * We only have a single ASID if PCID is off and the CR3
270 * write will have flushed it.
271 */
272 if (!cpu_feature_enabled(X86_FEATURE_PCID))
273 return;

--- 1080 unchanged lines hidden --- 		266 return;
267
268 /*
269 * We only have a single ASID if PCID is off and the CR3
270 * write will have flushed it.
271 */
272 if (!cpu_feature_enabled(X86_FEATURE_PCID))
273 return;

--- 1080 unchanged lines hidden ---