xref: /freebsd/sys/amd64/include/pcpu.h (revision d3d381b2b194b4d24853e92eecef55f262688d1a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) Peter Wemm <peter@netplex.com.au>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #ifndef _MACHINE_PCPU_H_
32 #define	_MACHINE_PCPU_H_
33 
34 #ifndef _SYS_CDEFS_H_
35 #error "sys/cdefs.h is a prerequisite for this file"
36 #endif
37 
38 #define	PC_PTI_STACK_SZ	16
39 /*
40  * The SMP parts are setup in pmap.c and locore.s for the BSP, and
41  * mp_machdep.c sets up the data for the AP's to "see" when they awake.
42  * The reason for doing it via a struct is so that an array of pointers
43  * to each CPU's data can be set up for things like "check curproc on all
44  * other processors"
45  */
46 #define	PCPU_MD_FIELDS							\
47 	char	pc_monitorbuf[128] __aligned(128); /* cache line */	\
48 	struct	pcpu *pc_prvspace;	/* Self-reference */		\
49 	struct	pmap *pc_curpmap;					\
50 	struct	amd64tss *pc_tssp;	/* TSS segment active on CPU */	\
51 	struct	amd64tss *pc_commontssp;/* Common TSS for the CPU */	\
52 	uint64_t pc_kcr3;						\
53 	uint64_t pc_ucr3;						\
54 	uint64_t pc_saved_ucr3;						\
55 	register_t pc_rsp0;						\
56 	register_t pc_scratch_rsp;	/* User %rsp in syscall */	\
57 	register_t pc_scratch_rax;					\
58 	u_int	pc_apic_id;						\
59 	u_int   pc_acpi_id;		/* ACPI CPU id */		\
60 	/* Pointer to the CPU %fs descriptor */				\
61 	struct user_segment_descriptor	*pc_fs32p;			\
62 	/* Pointer to the CPU %gs descriptor */				\
63 	struct user_segment_descriptor	*pc_gs32p;			\
64 	/* Pointer to the CPU LDT descriptor */				\
65 	struct system_segment_descriptor *pc_ldt;			\
66 	/* Pointer to the CPU TSS descriptor */				\
67 	struct system_segment_descriptor *pc_tss;			\
68 	uint64_t	pc_pm_save_cnt;					\
69 	u_int	pc_cmci_mask;		/* MCx banks for CMCI */	\
70 	uint64_t pc_dbreg[16];		/* ddb debugging regs */	\
71 	uint64_t pc_pti_stack[PC_PTI_STACK_SZ];				\
72 	register_t pc_pti_rsp0;						\
73 	int pc_dbreg_cmd;		/* ddb debugging reg cmd */	\
74 	u_int	pc_vcpu_id;		/* Xen vCPU ID */		\
75 	uint32_t pc_pcid_next;						\
76 	uint32_t pc_pcid_gen;						\
77 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
78 	uint32_t pc_ibpb_set;						\
79 	char	__pad[3288]		/* pad to UMA_PCPU_ALLOC_SIZE */
80 
81 #define	PC_DBREG_CMD_NONE	0
82 #define	PC_DBREG_CMD_LOAD	1
83 
84 #ifdef _KERNEL
85 
86 #if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF)
87 
88 /*
89  * Evaluates to the byte offset of the per-cpu variable name.
90  */
91 #define	__pcpu_offset(name)						\
92 	__offsetof(struct pcpu, name)
93 
94 /*
95  * Evaluates to the type of the per-cpu variable name.
96  */
97 #define	__pcpu_type(name)						\
98 	__typeof(((struct pcpu *)0)->name)
99 
100 /*
101  * Evaluates to the address of the per-cpu variable name.
102  */
103 #define	__PCPU_PTR(name) __extension__ ({				\
104 	__pcpu_type(name) *__p;						\
105 									\
106 	__asm __volatile("movq %%gs:%1,%0; addq %2,%0"			\
107 	    : "=r" (__p)						\
108 	    : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))),	\
109 	      "i" (__pcpu_offset(name)));				\
110 									\
111 	__p;								\
112 })
113 
114 /*
115  * Evaluates to the value of the per-cpu variable name.
116  */
117 #define	__PCPU_GET(name) __extension__ ({				\
118 	__pcpu_type(name) __res;					\
119 	struct __s {							\
120 		u_char	__b[MIN(sizeof(__pcpu_type(name)), 8)];		\
121 	} __s;								\
122 									\
123 	if (sizeof(__res) == 1 || sizeof(__res) == 2 ||			\
124 	    sizeof(__res) == 4 || sizeof(__res) == 8) {			\
125 		__asm __volatile("mov %%gs:%1,%0"			\
126 		    : "=r" (__s)					\
127 		    : "m" (*(struct __s *)(__pcpu_offset(name))));	\
128 		*(struct __s *)(void *)&__res = __s;			\
129 	} else {							\
130 		__res = *__PCPU_PTR(name);				\
131 	}								\
132 	__res;								\
133 })
134 
135 /*
136  * Adds the value to the per-cpu counter name.  The implementation
137  * must be atomic with respect to interrupts.
138  */
139 #define	__PCPU_ADD(name, val) do {					\
140 	__pcpu_type(name) __val;					\
141 	struct __s {							\
142 		u_char	__b[MIN(sizeof(__pcpu_type(name)), 8)];		\
143 	} __s;								\
144 									\
145 	__val = (val);							\
146 	if (sizeof(__val) == 1 || sizeof(__val) == 2 ||			\
147 	    sizeof(__val) == 4 || sizeof(__val) == 8) {			\
148 		__s = *(struct __s *)(void *)&__val;			\
149 		__asm __volatile("add %1,%%gs:%0"			\
150 		    : "=m" (*(struct __s *)(__pcpu_offset(name)))	\
151 		    : "r" (__s));					\
152 	} else								\
153 		*__PCPU_PTR(name) += __val;				\
154 } while (0)
155 
156 /*
157  * Increments the value of the per-cpu counter name.  The implementation
158  * must be atomic with respect to interrupts.
159  */
160 #define	__PCPU_INC(name) do {						\
161 	CTASSERT(sizeof(__pcpu_type(name)) == 1 ||			\
162 	    sizeof(__pcpu_type(name)) == 2 ||				\
163 	    sizeof(__pcpu_type(name)) == 4 ||				\
164 	    sizeof(__pcpu_type(name)) == 8);				\
165 	if (sizeof(__pcpu_type(name)) == 1) {				\
166 		__asm __volatile("incb %%gs:%0"				\
167 		    : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\
168 		    : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\
169 	} else if (sizeof(__pcpu_type(name)) == 2) {			\
170 		__asm __volatile("incw %%gs:%0"				\
171 		    : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\
172 		    : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\
173 	} else if (sizeof(__pcpu_type(name)) == 4) {			\
174 		__asm __volatile("incl %%gs:%0"				\
175 		    : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\
176 		    : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\
177 	} else if (sizeof(__pcpu_type(name)) == 8) {			\
178 		__asm __volatile("incq %%gs:%0"				\
179 		    : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\
180 		    : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\
181 	}								\
182 } while (0)
183 
184 /*
185  * Sets the value of the per-cpu variable name to value val.
186  */
187 #define	__PCPU_SET(name, val) {						\
188 	__pcpu_type(name) __val;					\
189 	struct __s {							\
190 		u_char	__b[MIN(sizeof(__pcpu_type(name)), 8)];		\
191 	} __s;								\
192 									\
193 	__val = (val);							\
194 	if (sizeof(__val) == 1 || sizeof(__val) == 2 ||			\
195 	    sizeof(__val) == 4 || sizeof(__val) == 8) {			\
196 		__s = *(struct __s *)(void *)&__val;			\
197 		__asm __volatile("mov %1,%%gs:%0"			\
198 		    : "=m" (*(struct __s *)(__pcpu_offset(name)))	\
199 		    : "r" (__s));					\
200 	} else {							\
201 		*__PCPU_PTR(name) = __val;				\
202 	}								\
203 }
204 
205 #define	get_pcpu() __extension__ ({					\
206 	struct pcpu *__pc;						\
207 									\
208 	__asm __volatile("movq %%gs:%1,%0"				\
209 	    : "=r" (__pc)						\
210 	    : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))));	\
211 	__pc;								\
212 })
213 
214 #define	PCPU_GET(member)	__PCPU_GET(pc_ ## member)
215 #define	PCPU_ADD(member, val)	__PCPU_ADD(pc_ ## member, val)
216 #define	PCPU_INC(member)	__PCPU_INC(pc_ ## member)
217 #define	PCPU_PTR(member)	__PCPU_PTR(pc_ ## member)
218 #define	PCPU_SET(member, val)	__PCPU_SET(pc_ ## member, val)
219 
220 #define	OFFSETOF_CURTHREAD	0
221 #ifdef __clang__
222 #pragma clang diagnostic push
223 #pragma clang diagnostic ignored "-Wnull-dereference"
224 #endif
225 static __inline __pure2 struct thread *
226 __curthread(void)
227 {
228 	struct thread *td;
229 
230 	__asm("movq %%gs:%1,%0" : "=r" (td)
231 	    : "m" (*(char *)OFFSETOF_CURTHREAD));
232 	return (td);
233 }
234 #ifdef __clang__
235 #pragma clang diagnostic pop
236 #endif
237 #define	curthread		(__curthread())
238 
239 #define	OFFSETOF_CURPCB		32
240 static __inline __pure2 struct pcb *
241 __curpcb(void)
242 {
243 	struct pcb *pcb;
244 
245 	__asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB));
246 	return (pcb);
247 }
248 #define	curpcb		(__curpcb())
249 
250 #define	IS_BSP()	(PCPU_GET(cpuid) == 0)
251 
252 #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */
253 
254 #error "this file needs to be ported to your compiler"
255 
256 #endif /* __GNUCLIKE_ASM && __GNUCLIKE___TYPEOF */
257 
258 #endif /* _KERNEL */
259 
260 #endif /* !_MACHINE_PCPU_H_ */
261