xref: /linux/arch/x86/include/asm/desc.h (revision 0a30908b9149b2b332ccf817261125a634765566) 
1 #ifndef _ASM_X86_DESC_H
2 #define _ASM_X86_DESC_H
3 
4 #include <asm/desc_defs.h>
5 #include <asm/ldt.h>
6 #include <asm/mmu.h>
7 #include <asm/fixmap.h>
8 #include <asm/irq_vectors.h>
9 
10 #include <linux/smp.h>
11 #include <linux/percpu.h>
12 
13 static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info)
14 {
15 	desc->limit0		= info->limit & 0x0ffff;
16 
17 	desc->base0		= (info->base_addr & 0x0000ffff);
18 	desc->base1		= (info->base_addr & 0x00ff0000) >> 16;
19 
20 	desc->type		= (info->read_exec_only ^ 1) << 1;
21 	desc->type	       |= info->contents << 2;
22 
23 	desc->s			= 1;
24 	desc->dpl		= 0x3;
25 	desc->p			= info->seg_not_present ^ 1;
26 	desc->limit1		= (info->limit & 0xf0000) >> 16;
27 	desc->avl		= info->useable;
28 	desc->d			= info->seg_32bit;
29 	desc->g			= info->limit_in_pages;
30 
31 	desc->base2		= (info->base_addr & 0xff000000) >> 24;
32 	/*
33 	 * Don't allow setting of the lm bit. It would confuse
34 	 * user_64bit_mode and would get overridden by sysret anyway.
35 	 */
36 	desc->l			= 0;
37 }
38 
39 extern struct desc_ptr idt_descr;
40 extern gate_desc idt_table[];
41 extern const struct desc_ptr debug_idt_descr;
42 extern gate_desc debug_idt_table[];
43 
44 struct gdt_page {
45 	struct desc_struct gdt[GDT_ENTRIES];
46 } __attribute__((aligned(PAGE_SIZE)));
47 
48 DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
49 
50 /* Provide the original GDT */
51 static inline struct desc_struct *get_cpu_gdt_rw(unsigned int cpu)
52 {
53 	return per_cpu(gdt_page, cpu).gdt;
54 }
55 
56 /* Provide the current original GDT */
57 static inline struct desc_struct *get_current_gdt_rw(void)
58 {
59 	return this_cpu_ptr(&gdt_page)->gdt;
60 }
61 
62 /* Get the fixmap index for a specific processor */
63 static inline unsigned int get_cpu_gdt_ro_index(int cpu)
64 {
65 	return FIX_GDT_REMAP_BEGIN + cpu;
66 }
67 
68 /* Provide the fixmap address of the remapped GDT */
69 static inline struct desc_struct *get_cpu_gdt_ro(int cpu)
70 {
71 	unsigned int idx = get_cpu_gdt_ro_index(cpu);
72 	return (struct desc_struct *)__fix_to_virt(idx);
73 }
74 
75 /* Provide the current read-only GDT */
76 static inline struct desc_struct *get_current_gdt_ro(void)
77 {
78 	return get_cpu_gdt_ro(smp_processor_id());
79 }
80 
81 /* Provide the physical address of the GDT page. */
82 static inline phys_addr_t get_cpu_gdt_paddr(unsigned int cpu)
83 {
84 	return per_cpu_ptr_to_phys(get_cpu_gdt_rw(cpu));
85 }
86 
87 static inline void pack_gate(gate_desc *gate, unsigned type, unsigned long func,
88 			     unsigned dpl, unsigned ist, unsigned seg)
89 {
90 	gate->offset_low	= (u16) func;
91 	gate->bits.p		= 1;
92 	gate->bits.dpl		= dpl;
93 	gate->bits.zero		= 0;
94 	gate->bits.type		= type;
95 	gate->offset_middle	= (u16) (func >> 16);
96 #ifdef CONFIG_X86_64
97 	gate->segment		= __KERNEL_CS;
98 	gate->bits.ist		= ist;
99 	gate->reserved		= 0;
100 	gate->offset_high	= (u32) (func >> 32);
101 #else
102 	gate->segment		= seg;
103 	gate->bits.ist		= 0;
104 #endif
105 }
106 
107 static inline int desc_empty(const void *ptr)
108 {
109 	const u32 *desc = ptr;
110 
111 	return !(desc[0] | desc[1]);
112 }
113 
114 #ifdef CONFIG_PARAVIRT
115 #include <asm/paravirt.h>
116 #else
117 #define load_TR_desc()				native_load_tr_desc()
118 #define load_gdt(dtr)				native_load_gdt(dtr)
119 #define load_idt(dtr)				native_load_idt(dtr)
120 #define load_tr(tr)				asm volatile("ltr %0"::"m" (tr))
121 #define load_ldt(ldt)				asm volatile("lldt %0"::"m" (ldt))
122 
123 #define store_gdt(dtr)				native_store_gdt(dtr)
124 #define store_idt(dtr)				native_store_idt(dtr)
125 #define store_tr(tr)				(tr = native_store_tr())
126 
127 #define load_TLS(t, cpu)			native_load_tls(t, cpu)
128 #define set_ldt					native_set_ldt
129 
130 #define write_ldt_entry(dt, entry, desc)	native_write_ldt_entry(dt, entry, desc)
131 #define write_gdt_entry(dt, entry, desc, type)	native_write_gdt_entry(dt, entry, desc, type)
132 #define write_idt_entry(dt, entry, g)		native_write_idt_entry(dt, entry, g)
133 
134 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
135 {
136 }
137 
138 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
139 {
140 }
141 #endif	/* CONFIG_PARAVIRT */
142 
143 #define store_ldt(ldt) asm("sldt %0" : "=m"(ldt))
144 
145 static inline void native_write_idt_entry(gate_desc *idt, int entry, const gate_desc *gate)
146 {
147 	memcpy(&idt[entry], gate, sizeof(*gate));
148 }
149 
150 static inline void native_write_ldt_entry(struct desc_struct *ldt, int entry, const void *desc)
151 {
152 	memcpy(&ldt[entry], desc, 8);
153 }
154 
155 static inline void
156 native_write_gdt_entry(struct desc_struct *gdt, int entry, const void *desc, int type)
157 {
158 	unsigned int size;
159 
160 	switch (type) {
161 	case DESC_TSS:	size = sizeof(tss_desc);	break;
162 	case DESC_LDT:	size = sizeof(ldt_desc);	break;
163 	default:	size = sizeof(*gdt);		break;
164 	}
165 
166 	memcpy(&gdt[entry], desc, size);
167 }
168 
169 static inline void set_tssldt_descriptor(void *d, unsigned long addr,
170 					 unsigned type, unsigned size)
171 {
172 	struct ldttss_desc *desc = d;
173 
174 	memset(desc, 0, sizeof(*desc));
175 
176 	desc->limit0		= (u16) size;
177 	desc->base0		= (u16) addr;
178 	desc->base1		= (addr >> 16) & 0xFF;
179 	desc->type		= type;
180 	desc->p			= 1;
181 	desc->limit1		= (size >> 16) & 0xF;
182 	desc->base2		= (addr >> 24) & 0xFF;
183 #ifdef CONFIG_X86_64
184 	desc->base3		= (u32) (addr >> 32);
185 #endif
186 }
187 
188 static inline void __set_tss_desc(unsigned cpu, unsigned int entry, void *addr)
189 {
190 	struct desc_struct *d = get_cpu_gdt_rw(cpu);
191 	tss_desc tss;
192 
193 	set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS,
194 			      __KERNEL_TSS_LIMIT);
195 	write_gdt_entry(d, entry, &tss, DESC_TSS);
196 }
197 
198 #define set_tss_desc(cpu, addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr)
199 
200 static inline void native_set_ldt(const void *addr, unsigned int entries)
201 {
202 	if (likely(entries == 0))
203 		asm volatile("lldt %w0"::"q" (0));
204 	else {
205 		unsigned cpu = smp_processor_id();
206 		ldt_desc ldt;
207 
208 		set_tssldt_descriptor(&ldt, (unsigned long)addr, DESC_LDT,
209 				      entries * LDT_ENTRY_SIZE - 1);
210 		write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_LDT,
211 				&ldt, DESC_LDT);
212 		asm volatile("lldt %w0"::"q" (GDT_ENTRY_LDT*8));
213 	}
214 }
215 
216 static inline void native_load_gdt(const struct desc_ptr *dtr)
217 {
218 	asm volatile("lgdt %0"::"m" (*dtr));
219 }
220 
221 static inline void native_load_idt(const struct desc_ptr *dtr)
222 {
223 	asm volatile("lidt %0"::"m" (*dtr));
224 }
225 
226 static inline void native_store_gdt(struct desc_ptr *dtr)
227 {
228 	asm volatile("sgdt %0":"=m" (*dtr));
229 }
230 
231 static inline void native_store_idt(struct desc_ptr *dtr)
232 {
233 	asm volatile("sidt %0":"=m" (*dtr));
234 }
235 
236 /*
237  * The LTR instruction marks the TSS GDT entry as busy. On 64-bit, the GDT is
238  * a read-only remapping. To prevent a page fault, the GDT is switched to the
239  * original writeable version when needed.
240  */
241 #ifdef CONFIG_X86_64
242 static inline void native_load_tr_desc(void)
243 {
244 	struct desc_ptr gdt;
245 	int cpu = raw_smp_processor_id();
246 	bool restore = 0;
247 	struct desc_struct *fixmap_gdt;
248 
249 	native_store_gdt(&gdt);
250 	fixmap_gdt = get_cpu_gdt_ro(cpu);
251 
252 	/*
253 	 * If the current GDT is the read-only fixmap, swap to the original
254 	 * writeable version. Swap back at the end.
255 	 */
256 	if (gdt.address == (unsigned long)fixmap_gdt) {
257 		load_direct_gdt(cpu);
258 		restore = 1;
259 	}
260 	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
261 	if (restore)
262 		load_fixmap_gdt(cpu);
263 }
264 #else
265 static inline void native_load_tr_desc(void)
266 {
267 	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
268 }
269 #endif
270 
271 static inline unsigned long native_store_tr(void)
272 {
273 	unsigned long tr;
274 
275 	asm volatile("str %0":"=r" (tr));
276 
277 	return tr;
278 }
279 
280 static inline void native_load_tls(struct thread_struct *t, unsigned int cpu)
281 {
282 	struct desc_struct *gdt = get_cpu_gdt_rw(cpu);
283 	unsigned int i;
284 
285 	for (i = 0; i < GDT_ENTRY_TLS_ENTRIES; i++)
286 		gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i];
287 }
288 
289 DECLARE_PER_CPU(bool, __tss_limit_invalid);
290 
291 static inline void force_reload_TR(void)
292 {
293 	struct desc_struct *d = get_current_gdt_rw();
294 	tss_desc tss;
295 
296 	memcpy(&tss, &d[GDT_ENTRY_TSS], sizeof(tss_desc));
297 
298 	/*
299 	 * LTR requires an available TSS, and the TSS is currently
300 	 * busy.  Make it be available so that LTR will work.
301 	 */
302 	tss.type = DESC_TSS;
303 	write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);
304 
305 	load_TR_desc();
306 	this_cpu_write(__tss_limit_invalid, false);
307 }
308 
309 /*
310  * Call this if you need the TSS limit to be correct, which should be the case
311  * if and only if you have TIF_IO_BITMAP set or you're switching to a task
312  * with TIF_IO_BITMAP set.
313  */
314 static inline void refresh_tss_limit(void)
315 {
316 	DEBUG_LOCKS_WARN_ON(preemptible());
317 
318 	if (unlikely(this_cpu_read(__tss_limit_invalid)))
319 		force_reload_TR();
320 }
321 
322 /*
323  * If you do something evil that corrupts the cached TSS limit (I'm looking
324  * at you, VMX exits), call this function.
325  *
326  * The optimization here is that the TSS limit only matters for Linux if the
327  * IO bitmap is in use.  If the TSS limit gets forced to its minimum value,
328  * everything works except that IO bitmap will be ignored and all CPL 3 IO
329  * instructions will #GP, which is exactly what we want for normal tasks.
330  */
331 static inline void invalidate_tss_limit(void)
332 {
333 	DEBUG_LOCKS_WARN_ON(preemptible());
334 
335 	if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
336 		force_reload_TR();
337 	else
338 		this_cpu_write(__tss_limit_invalid, true);
339 }
340 
341 /* This intentionally ignores lm, since 32-bit apps don't have that field. */
342 #define LDT_empty(info)					\
343 	((info)->base_addr		== 0	&&	\
344 	 (info)->limit			== 0	&&	\
345 	 (info)->contents		== 0	&&	\
346 	 (info)->read_exec_only		== 1	&&	\
347 	 (info)->seg_32bit		== 0	&&	\
348 	 (info)->limit_in_pages		== 0	&&	\
349 	 (info)->seg_not_present	== 1	&&	\
350 	 (info)->useable		== 0)
351 
352 /* Lots of programs expect an all-zero user_desc to mean "no segment at all". */
353 static inline bool LDT_zero(const struct user_desc *info)
354 {
355 	return (info->base_addr		== 0 &&
356 		info->limit		== 0 &&
357 		info->contents		== 0 &&
358 		info->read_exec_only	== 0 &&
359 		info->seg_32bit		== 0 &&
360 		info->limit_in_pages	== 0 &&
361 		info->seg_not_present	== 0 &&
362 		info->useable		== 0);
363 }
364 
365 static inline void clear_LDT(void)
366 {
367 	set_ldt(NULL, 0);
368 }
369 
370 static inline unsigned long get_desc_base(const struct desc_struct *desc)
371 {
372 	return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
373 }
374 
375 static inline void set_desc_base(struct desc_struct *desc, unsigned long base)
376 {
377 	desc->base0 = base & 0xffff;
378 	desc->base1 = (base >> 16) & 0xff;
379 	desc->base2 = (base >> 24) & 0xff;
380 }
381 
382 static inline unsigned long get_desc_limit(const struct desc_struct *desc)
383 {
384 	return desc->limit0 | (desc->limit1 << 16);
385 }
386 
387 static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
388 {
389 	desc->limit0 = limit & 0xffff;
390 	desc->limit1 = (limit >> 16) & 0xf;
391 }
392 
393 #ifdef CONFIG_X86_64
394 static inline void set_nmi_gate(int gate, void *addr)
395 {
396 	gate_desc s;
397 
398 	pack_gate(&s, GATE_INTERRUPT, (unsigned long)addr, 0, 0, __KERNEL_CS);
399 	write_idt_entry(debug_idt_table, gate, &s);
400 }
401 #endif
402 
403 static inline void _set_gate(int gate, unsigned type, const void *addr,
404 			     unsigned dpl, unsigned ist, unsigned seg)
405 {
406 	gate_desc s;
407 
408 	pack_gate(&s, type, (unsigned long)addr, dpl, ist, seg);
409 	/*
410 	 * does not need to be atomic because it is only done once at
411 	 * setup time
412 	 */
413 	write_idt_entry(idt_table, gate, &s);
414 }
415 
416 static inline void set_intr_gate(unsigned int n, const void *addr)
417 {
418 	BUG_ON(n > 0xFF);
419 	_set_gate(n, GATE_INTERRUPT, addr, 0, 0, __KERNEL_CS);
420 }
421 
422 extern unsigned long used_vectors[];
423 
424 static inline void alloc_system_vector(int vector)
425 {
426 	BUG_ON(vector < FIRST_SYSTEM_VECTOR);
427 	if (!test_bit(vector, used_vectors)) {
428 		set_bit(vector, used_vectors);
429 	} else {
430 		BUG();
431 	}
432 }
433 
434 #define alloc_intr_gate(n, addr)				\
435 	do {							\
436 		alloc_system_vector(n);				\
437 		set_intr_gate(n, addr);				\
438 	} while (0)
439 
440 /*
441  * This routine sets up an interrupt gate at directory privilege level 3.
442  */
443 static inline void set_system_intr_gate(unsigned int n, void *addr)
444 {
445 	BUG_ON((unsigned)n > 0xFF);
446 	_set_gate(n, GATE_INTERRUPT, addr, 0x3, 0, __KERNEL_CS);
447 }
448 
449 static inline void set_task_gate(unsigned int n, unsigned int gdt_entry)
450 {
451 	BUG_ON((unsigned)n > 0xFF);
452 	_set_gate(n, GATE_TASK, (void *)0, 0, 0, (gdt_entry<<3));
453 }
454 
455 static inline void set_intr_gate_ist(int n, void *addr, unsigned ist)
456 {
457 	BUG_ON((unsigned)n > 0xFF);
458 	_set_gate(n, GATE_INTERRUPT, addr, 0, ist, __KERNEL_CS);
459 }
460 
461 static inline void set_system_intr_gate_ist(int n, void *addr, unsigned ist)
462 {
463 	BUG_ON((unsigned)n > 0xFF);
464 	_set_gate(n, GATE_INTERRUPT, addr, 0x3, ist, __KERNEL_CS);
465 }
466 
467 #ifdef CONFIG_X86_64
468 DECLARE_PER_CPU(u32, debug_idt_ctr);
469 static inline bool is_debug_idt_enabled(void)
470 {
471 	if (this_cpu_read(debug_idt_ctr))
472 		return true;
473 
474 	return false;
475 }
476 
477 static inline void load_debug_idt(void)
478 {
479 	load_idt((const struct desc_ptr *)&debug_idt_descr);
480 }
481 #else
482 static inline bool is_debug_idt_enabled(void)
483 {
484 	return false;
485 }
486 
487 static inline void load_debug_idt(void)
488 {
489 }
490 #endif
491 
492 /*
493  * The load_current_idt() must be called with interrupts disabled
494  * to avoid races. That way the IDT will always be set back to the expected
495  * descriptor. It's also called when a CPU is being initialized, and
496  * that doesn't need to disable interrupts, as nothing should be
497  * bothering the CPU then.
498  */
499 static inline void load_current_idt(void)
500 {
501 	if (is_debug_idt_enabled())
502 		load_debug_idt();
503 	else
504 		load_idt((const struct desc_ptr *)&idt_descr);
505 }
506 
507 extern void idt_setup_early_handler(void);
508 extern void idt_setup_early_traps(void);
509 
510 #ifdef CONFIG_X86_64
511 extern void idt_setup_early_pf(void);
512 extern void idt_setup_debugidt_traps(void);
513 #else
514 static inline void idt_setup_early_pf(void) { }
515 static inline void idt_setup_debugidt_traps(void) { }
516 #endif
517 
518 extern void idt_invalidate(void *addr);
519 
520 #endif /* _ASM_X86_DESC_H */
521