1 /* 2 * Copyright (C) 1995 Linus Torvalds 3 * 4 * Pentium III FXSR, SSE support 5 * Gareth Hughes <gareth@valinux.com>, May 2000 6 * 7 * X86-64 port 8 * Andi Kleen. 9 * 10 * CPU hotplug support - ashok.raj@intel.com 11 */ 12 13 /* 14 * This file handles the architecture-dependent parts of process handling.. 15 */ 16 17 #include <stdarg.h> 18 19 #include <linux/cpu.h> 20 #include <linux/errno.h> 21 #include <linux/sched.h> 22 #include <linux/fs.h> 23 #include <linux/kernel.h> 24 #include <linux/mm.h> 25 #include <linux/elfcore.h> 26 #include <linux/smp.h> 27 #include <linux/slab.h> 28 #include <linux/user.h> 29 #include <linux/a.out.h> 30 #include <linux/interrupt.h> 31 #include <linux/utsname.h> 32 #include <linux/delay.h> 33 #include <linux/module.h> 34 #include <linux/ptrace.h> 35 #include <linux/random.h> 36 #include <linux/notifier.h> 37 #include <linux/kprobes.h> 38 #include <linux/kdebug.h> 39 #include <linux/tick.h> 40 41 #include <asm/uaccess.h> 42 #include <asm/pgtable.h> 43 #include <asm/system.h> 44 #include <asm/io.h> 45 #include <asm/processor.h> 46 #include <asm/i387.h> 47 #include <asm/mmu_context.h> 48 #include <asm/pda.h> 49 #include <asm/prctl.h> 50 #include <asm/desc.h> 51 #include <asm/proto.h> 52 #include <asm/ia32.h> 53 #include <asm/idle.h> 54 55 asmlinkage extern void ret_from_fork(void); 56 57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; 58 59 unsigned long boot_option_idle_override = 0; 60 EXPORT_SYMBOL(boot_option_idle_override); 61 62 /* 63 * Powermanagement idle function, if any.. 64 */ 65 void (*pm_idle)(void); 66 EXPORT_SYMBOL(pm_idle); 67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state); 68 69 static ATOMIC_NOTIFIER_HEAD(idle_notifier); 70 71 void idle_notifier_register(struct notifier_block *n) 72 { 73 atomic_notifier_chain_register(&idle_notifier, n); 74 } 75 76 void enter_idle(void) 77 { 78 write_pda(isidle, 1); 79 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); 80 } 81 82 static void __exit_idle(void) 83 { 84 if (test_and_clear_bit_pda(0, isidle) == 0) 85 return; 86 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); 87 } 88 89 /* Called from interrupts to signify idle end */ 90 void exit_idle(void) 91 { 92 /* idle loop has pid 0 */ 93 if (current->pid) 94 return; 95 __exit_idle(); 96 } 97 98 /* 99 * We use this if we don't have any better 100 * idle routine.. 101 */ 102 void default_idle(void) 103 { 104 current_thread_info()->status &= ~TS_POLLING; 105 /* 106 * TS_POLLING-cleared state must be visible before we 107 * test NEED_RESCHED: 108 */ 109 smp_mb(); 110 local_irq_disable(); 111 if (!need_resched()) { 112 ktime_t t0, t1; 113 u64 t0n, t1n; 114 115 t0 = ktime_get(); 116 t0n = ktime_to_ns(t0); 117 safe_halt(); /* enables interrupts racelessly */ 118 local_irq_disable(); 119 t1 = ktime_get(); 120 t1n = ktime_to_ns(t1); 121 sched_clock_idle_wakeup_event(t1n - t0n); 122 } 123 local_irq_enable(); 124 current_thread_info()->status |= TS_POLLING; 125 } 126 127 /* 128 * On SMP it's slightly faster (but much more power-consuming!) 129 * to poll the ->need_resched flag instead of waiting for the 130 * cross-CPU IPI to arrive. Use this option with caution. 131 */ 132 static void poll_idle(void) 133 { 134 local_irq_enable(); 135 cpu_relax(); 136 } 137 138 #ifdef CONFIG_HOTPLUG_CPU 139 DECLARE_PER_CPU(int, cpu_state); 140 141 #include <asm/nmi.h> 142 /* We halt the CPU with physical CPU hotplug */ 143 static inline void play_dead(void) 144 { 145 idle_task_exit(); 146 wbinvd(); 147 mb(); 148 /* Ack it */ 149 __get_cpu_var(cpu_state) = CPU_DEAD; 150 151 local_irq_disable(); 152 while (1) 153 halt(); 154 } 155 #else 156 static inline void play_dead(void) 157 { 158 BUG(); 159 } 160 #endif /* CONFIG_HOTPLUG_CPU */ 161 162 /* 163 * The idle thread. There's no useful work to be 164 * done, so just try to conserve power and have a 165 * low exit latency (ie sit in a loop waiting for 166 * somebody to say that they'd like to reschedule) 167 */ 168 void cpu_idle(void) 169 { 170 current_thread_info()->status |= TS_POLLING; 171 /* endless idle loop with no priority at all */ 172 while (1) { 173 tick_nohz_stop_sched_tick(); 174 while (!need_resched()) { 175 void (*idle)(void); 176 177 if (__get_cpu_var(cpu_idle_state)) 178 __get_cpu_var(cpu_idle_state) = 0; 179 180 rmb(); 181 idle = pm_idle; 182 if (!idle) 183 idle = default_idle; 184 if (cpu_is_offline(smp_processor_id())) 185 play_dead(); 186 /* 187 * Idle routines should keep interrupts disabled 188 * from here on, until they go to idle. 189 * Otherwise, idle callbacks can misfire. 190 */ 191 local_irq_disable(); 192 enter_idle(); 193 idle(); 194 /* In many cases the interrupt that ended idle 195 has already called exit_idle. But some idle 196 loops can be woken up without interrupt. */ 197 __exit_idle(); 198 } 199 200 tick_nohz_restart_sched_tick(); 201 preempt_enable_no_resched(); 202 schedule(); 203 preempt_disable(); 204 } 205 } 206 207 static void do_nothing(void *unused) 208 { 209 } 210 211 void cpu_idle_wait(void) 212 { 213 unsigned int cpu, this_cpu = get_cpu(); 214 cpumask_t map, tmp = current->cpus_allowed; 215 216 set_cpus_allowed(current, cpumask_of_cpu(this_cpu)); 217 put_cpu(); 218 219 cpus_clear(map); 220 for_each_online_cpu(cpu) { 221 per_cpu(cpu_idle_state, cpu) = 1; 222 cpu_set(cpu, map); 223 } 224 225 __get_cpu_var(cpu_idle_state) = 0; 226 227 wmb(); 228 do { 229 ssleep(1); 230 for_each_online_cpu(cpu) { 231 if (cpu_isset(cpu, map) && !per_cpu(cpu_idle_state, cpu)) 232 cpu_clear(cpu, map); 233 } 234 cpus_and(map, map, cpu_online_map); 235 /* 236 * We waited 1 sec, if a CPU still did not call idle 237 * it may be because it is in idle and not waking up 238 * because it has nothing to do. 239 * Give all the remaining CPUS a kick. 240 */ 241 smp_call_function_mask(map, do_nothing, 0, 0); 242 } while (!cpus_empty(map)); 243 244 set_cpus_allowed(current, tmp); 245 } 246 EXPORT_SYMBOL_GPL(cpu_idle_wait); 247 248 /* 249 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI, 250 * which can obviate IPI to trigger checking of need_resched. 251 * We execute MONITOR against need_resched and enter optimized wait state 252 * through MWAIT. Whenever someone changes need_resched, we would be woken 253 * up from MWAIT (without an IPI). 254 * 255 * New with Core Duo processors, MWAIT can take some hints based on CPU 256 * capability. 257 */ 258 void mwait_idle_with_hints(unsigned long ax, unsigned long cx) 259 { 260 if (!need_resched()) { 261 __monitor((void *)¤t_thread_info()->flags, 0, 0); 262 smp_mb(); 263 if (!need_resched()) 264 __mwait(ax, cx); 265 } 266 } 267 268 /* Default MONITOR/MWAIT with no hints, used for default C1 state */ 269 static void mwait_idle(void) 270 { 271 if (!need_resched()) { 272 __monitor((void *)¤t_thread_info()->flags, 0, 0); 273 smp_mb(); 274 if (!need_resched()) 275 __sti_mwait(0, 0); 276 else 277 local_irq_enable(); 278 } else { 279 local_irq_enable(); 280 } 281 } 282 283 284 static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c) 285 { 286 if (force_mwait) 287 return 1; 288 /* Any C1 states supported? */ 289 return c->cpuid_level >= 5 && ((cpuid_edx(5) >> 4) & 0xf) > 0; 290 } 291 292 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) 293 { 294 static int selected; 295 296 if (selected) 297 return; 298 #ifdef CONFIG_X86_SMP 299 if (pm_idle == poll_idle && smp_num_siblings > 1) { 300 printk(KERN_WARNING "WARNING: polling idle and HT enabled," 301 " performance may degrade.\n"); 302 } 303 #endif 304 if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) { 305 /* 306 * Skip, if setup has overridden idle. 307 * One CPU supports mwait => All CPUs supports mwait 308 */ 309 if (!pm_idle) { 310 printk(KERN_INFO "using mwait in idle threads.\n"); 311 pm_idle = mwait_idle; 312 } 313 } 314 selected = 1; 315 } 316 317 static int __init idle_setup(char *str) 318 { 319 if (!strcmp(str, "poll")) { 320 printk("using polling idle threads.\n"); 321 pm_idle = poll_idle; 322 } else if (!strcmp(str, "mwait")) 323 force_mwait = 1; 324 else 325 return -1; 326 327 boot_option_idle_override = 1; 328 return 0; 329 } 330 early_param("idle", idle_setup); 331 332 /* Prints also some state that isn't saved in the pt_regs */ 333 void __show_regs(struct pt_regs * regs) 334 { 335 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; 336 unsigned long d0, d1, d2, d3, d6, d7; 337 unsigned int fsindex, gsindex; 338 unsigned int ds, cs, es; 339 340 printk("\n"); 341 print_modules(); 342 printk("Pid: %d, comm: %.20s %s %s %.*s\n", 343 current->pid, current->comm, print_tainted(), 344 init_utsname()->release, 345 (int)strcspn(init_utsname()->version, " "), 346 init_utsname()->version); 347 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip); 348 printk_address(regs->ip, 1); 349 printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->sp, 350 regs->flags); 351 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n", 352 regs->ax, regs->bx, regs->cx); 353 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n", 354 regs->dx, regs->si, regs->di); 355 printk("RBP: %016lx R08: %016lx R09: %016lx\n", 356 regs->bp, regs->r8, regs->r9); 357 printk("R10: %016lx R11: %016lx R12: %016lx\n", 358 regs->r10, regs->r11, regs->r12); 359 printk("R13: %016lx R14: %016lx R15: %016lx\n", 360 regs->r13, regs->r14, regs->r15); 361 362 asm("movl %%ds,%0" : "=r" (ds)); 363 asm("movl %%cs,%0" : "=r" (cs)); 364 asm("movl %%es,%0" : "=r" (es)); 365 asm("movl %%fs,%0" : "=r" (fsindex)); 366 asm("movl %%gs,%0" : "=r" (gsindex)); 367 368 rdmsrl(MSR_FS_BASE, fs); 369 rdmsrl(MSR_GS_BASE, gs); 370 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); 371 372 cr0 = read_cr0(); 373 cr2 = read_cr2(); 374 cr3 = read_cr3(); 375 cr4 = read_cr4(); 376 377 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n", 378 fs,fsindex,gs,gsindex,shadowgs); 379 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0); 380 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4); 381 382 get_debugreg(d0, 0); 383 get_debugreg(d1, 1); 384 get_debugreg(d2, 2); 385 printk("DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2); 386 get_debugreg(d3, 3); 387 get_debugreg(d6, 6); 388 get_debugreg(d7, 7); 389 printk("DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7); 390 } 391 392 void show_regs(struct pt_regs *regs) 393 { 394 printk("CPU %d:", smp_processor_id()); 395 __show_regs(regs); 396 show_trace(NULL, regs, (void *)(regs + 1), regs->bp); 397 } 398 399 /* 400 * Free current thread data structures etc.. 401 */ 402 void exit_thread(void) 403 { 404 struct task_struct *me = current; 405 struct thread_struct *t = &me->thread; 406 407 if (me->thread.io_bitmap_ptr) { 408 struct tss_struct *tss = &per_cpu(init_tss, get_cpu()); 409 410 kfree(t->io_bitmap_ptr); 411 t->io_bitmap_ptr = NULL; 412 clear_thread_flag(TIF_IO_BITMAP); 413 /* 414 * Careful, clear this in the TSS too: 415 */ 416 memset(tss->io_bitmap, 0xff, t->io_bitmap_max); 417 t->io_bitmap_max = 0; 418 put_cpu(); 419 } 420 } 421 422 void flush_thread(void) 423 { 424 struct task_struct *tsk = current; 425 426 if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) { 427 clear_tsk_thread_flag(tsk, TIF_ABI_PENDING); 428 if (test_tsk_thread_flag(tsk, TIF_IA32)) { 429 clear_tsk_thread_flag(tsk, TIF_IA32); 430 } else { 431 set_tsk_thread_flag(tsk, TIF_IA32); 432 current_thread_info()->status |= TS_COMPAT; 433 } 434 } 435 clear_tsk_thread_flag(tsk, TIF_DEBUG); 436 437 tsk->thread.debugreg0 = 0; 438 tsk->thread.debugreg1 = 0; 439 tsk->thread.debugreg2 = 0; 440 tsk->thread.debugreg3 = 0; 441 tsk->thread.debugreg6 = 0; 442 tsk->thread.debugreg7 = 0; 443 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 444 /* 445 * Forget coprocessor state.. 446 */ 447 clear_fpu(tsk); 448 clear_used_math(); 449 } 450 451 void release_thread(struct task_struct *dead_task) 452 { 453 if (dead_task->mm) { 454 if (dead_task->mm->context.size) { 455 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n", 456 dead_task->comm, 457 dead_task->mm->context.ldt, 458 dead_task->mm->context.size); 459 BUG(); 460 } 461 } 462 } 463 464 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr) 465 { 466 struct user_desc ud = { 467 .base_addr = addr, 468 .limit = 0xfffff, 469 .seg_32bit = 1, 470 .limit_in_pages = 1, 471 .useable = 1, 472 }; 473 struct desc_struct *desc = t->thread.tls_array; 474 desc += tls; 475 fill_ldt(desc, &ud); 476 } 477 478 static inline u32 read_32bit_tls(struct task_struct *t, int tls) 479 { 480 return get_desc_base(&t->thread.tls_array[tls]); 481 } 482 483 /* 484 * This gets called before we allocate a new thread and copy 485 * the current task into it. 486 */ 487 void prepare_to_copy(struct task_struct *tsk) 488 { 489 unlazy_fpu(tsk); 490 } 491 492 int copy_thread(int nr, unsigned long clone_flags, unsigned long sp, 493 unsigned long unused, 494 struct task_struct * p, struct pt_regs * regs) 495 { 496 int err; 497 struct pt_regs * childregs; 498 struct task_struct *me = current; 499 500 childregs = ((struct pt_regs *) 501 (THREAD_SIZE + task_stack_page(p))) - 1; 502 *childregs = *regs; 503 504 childregs->ax = 0; 505 childregs->sp = sp; 506 if (sp == ~0UL) 507 childregs->sp = (unsigned long)childregs; 508 509 p->thread.sp = (unsigned long) childregs; 510 p->thread.sp0 = (unsigned long) (childregs+1); 511 p->thread.usersp = me->thread.usersp; 512 513 set_tsk_thread_flag(p, TIF_FORK); 514 515 p->thread.fs = me->thread.fs; 516 p->thread.gs = me->thread.gs; 517 518 asm("mov %%gs,%0" : "=m" (p->thread.gsindex)); 519 asm("mov %%fs,%0" : "=m" (p->thread.fsindex)); 520 asm("mov %%es,%0" : "=m" (p->thread.es)); 521 asm("mov %%ds,%0" : "=m" (p->thread.ds)); 522 523 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) { 524 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); 525 if (!p->thread.io_bitmap_ptr) { 526 p->thread.io_bitmap_max = 0; 527 return -ENOMEM; 528 } 529 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr, 530 IO_BITMAP_BYTES); 531 set_tsk_thread_flag(p, TIF_IO_BITMAP); 532 } 533 534 /* 535 * Set a new TLS for the child thread? 536 */ 537 if (clone_flags & CLONE_SETTLS) { 538 #ifdef CONFIG_IA32_EMULATION 539 if (test_thread_flag(TIF_IA32)) 540 err = do_set_thread_area(p, -1, 541 (struct user_desc __user *)childregs->si, 0); 542 else 543 #endif 544 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8); 545 if (err) 546 goto out; 547 } 548 err = 0; 549 out: 550 if (err && p->thread.io_bitmap_ptr) { 551 kfree(p->thread.io_bitmap_ptr); 552 p->thread.io_bitmap_max = 0; 553 } 554 return err; 555 } 556 557 /* 558 * This special macro can be used to load a debugging register 559 */ 560 #define loaddebug(thread, r) set_debugreg(thread->debugreg ## r, r) 561 562 static inline void __switch_to_xtra(struct task_struct *prev_p, 563 struct task_struct *next_p, 564 struct tss_struct *tss) 565 { 566 struct thread_struct *prev, *next; 567 unsigned long debugctl; 568 569 prev = &prev_p->thread, 570 next = &next_p->thread; 571 572 debugctl = prev->debugctlmsr; 573 if (next->ds_area_msr != prev->ds_area_msr) { 574 /* we clear debugctl to make sure DS 575 * is not in use when we change it */ 576 debugctl = 0; 577 wrmsrl(MSR_IA32_DEBUGCTLMSR, 0); 578 wrmsrl(MSR_IA32_DS_AREA, next->ds_area_msr); 579 } 580 581 if (next->debugctlmsr != debugctl) 582 wrmsrl(MSR_IA32_DEBUGCTLMSR, next->debugctlmsr); 583 584 if (test_tsk_thread_flag(next_p, TIF_DEBUG)) { 585 loaddebug(next, 0); 586 loaddebug(next, 1); 587 loaddebug(next, 2); 588 loaddebug(next, 3); 589 /* no 4 and 5 */ 590 loaddebug(next, 6); 591 loaddebug(next, 7); 592 } 593 594 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { 595 /* 596 * Copy the relevant range of the IO bitmap. 597 * Normally this is 128 bytes or less: 598 */ 599 memcpy(tss->io_bitmap, next->io_bitmap_ptr, 600 max(prev->io_bitmap_max, next->io_bitmap_max)); 601 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) { 602 /* 603 * Clear any possible leftover bits: 604 */ 605 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); 606 } 607 608 if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) 609 ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS); 610 611 if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS)) 612 ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES); 613 } 614 615 /* 616 * switch_to(x,y) should switch tasks from x to y. 617 * 618 * This could still be optimized: 619 * - fold all the options into a flag word and test it with a single test. 620 * - could test fs/gs bitsliced 621 * 622 * Kprobes not supported here. Set the probe on schedule instead. 623 */ 624 struct task_struct * 625 __switch_to(struct task_struct *prev_p, struct task_struct *next_p) 626 { 627 struct thread_struct *prev = &prev_p->thread, 628 *next = &next_p->thread; 629 int cpu = smp_processor_id(); 630 struct tss_struct *tss = &per_cpu(init_tss, cpu); 631 632 /* we're going to use this soon, after a few expensive things */ 633 if (next_p->fpu_counter>5) 634 prefetch(&next->i387.fxsave); 635 636 /* 637 * Reload esp0, LDT and the page table pointer: 638 */ 639 load_sp0(tss, next); 640 641 /* 642 * Switch DS and ES. 643 * This won't pick up thread selector changes, but I guess that is ok. 644 */ 645 asm volatile("mov %%es,%0" : "=m" (prev->es)); 646 if (unlikely(next->es | prev->es)) 647 loadsegment(es, next->es); 648 649 asm volatile ("mov %%ds,%0" : "=m" (prev->ds)); 650 if (unlikely(next->ds | prev->ds)) 651 loadsegment(ds, next->ds); 652 653 load_TLS(next, cpu); 654 655 /* 656 * Switch FS and GS. 657 */ 658 { 659 unsigned fsindex; 660 asm volatile("movl %%fs,%0" : "=r" (fsindex)); 661 /* segment register != 0 always requires a reload. 662 also reload when it has changed. 663 when prev process used 64bit base always reload 664 to avoid an information leak. */ 665 if (unlikely(fsindex | next->fsindex | prev->fs)) { 666 loadsegment(fs, next->fsindex); 667 /* check if the user used a selector != 0 668 * if yes clear 64bit base, since overloaded base 669 * is always mapped to the Null selector 670 */ 671 if (fsindex) 672 prev->fs = 0; 673 } 674 /* when next process has a 64bit base use it */ 675 if (next->fs) 676 wrmsrl(MSR_FS_BASE, next->fs); 677 prev->fsindex = fsindex; 678 } 679 { 680 unsigned gsindex; 681 asm volatile("movl %%gs,%0" : "=r" (gsindex)); 682 if (unlikely(gsindex | next->gsindex | prev->gs)) { 683 load_gs_index(next->gsindex); 684 if (gsindex) 685 prev->gs = 0; 686 } 687 if (next->gs) 688 wrmsrl(MSR_KERNEL_GS_BASE, next->gs); 689 prev->gsindex = gsindex; 690 } 691 692 /* Must be after DS reload */ 693 unlazy_fpu(prev_p); 694 695 /* 696 * Switch the PDA and FPU contexts. 697 */ 698 prev->usersp = read_pda(oldrsp); 699 write_pda(oldrsp, next->usersp); 700 write_pda(pcurrent, next_p); 701 702 write_pda(kernelstack, 703 (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET); 704 #ifdef CONFIG_CC_STACKPROTECTOR 705 write_pda(stack_canary, next_p->stack_canary); 706 /* 707 * Build time only check to make sure the stack_canary is at 708 * offset 40 in the pda; this is a gcc ABI requirement 709 */ 710 BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40); 711 #endif 712 713 /* 714 * Now maybe reload the debug registers and handle I/O bitmaps 715 */ 716 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT || 717 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) 718 __switch_to_xtra(prev_p, next_p, tss); 719 720 /* If the task has used fpu the last 5 timeslices, just do a full 721 * restore of the math state immediately to avoid the trap; the 722 * chances of needing FPU soon are obviously high now 723 */ 724 if (next_p->fpu_counter>5) 725 math_state_restore(); 726 return prev_p; 727 } 728 729 /* 730 * sys_execve() executes a new program. 731 */ 732 asmlinkage 733 long sys_execve(char __user *name, char __user * __user *argv, 734 char __user * __user *envp, struct pt_regs regs) 735 { 736 long error; 737 char * filename; 738 739 filename = getname(name); 740 error = PTR_ERR(filename); 741 if (IS_ERR(filename)) 742 return error; 743 error = do_execve(filename, argv, envp, ®s); 744 putname(filename); 745 return error; 746 } 747 748 void set_personality_64bit(void) 749 { 750 /* inherit personality from parent */ 751 752 /* Make sure to be in 64bit mode */ 753 clear_thread_flag(TIF_IA32); 754 755 /* TBD: overwrites user setup. Should have two bits. 756 But 64bit processes have always behaved this way, 757 so it's not too bad. The main problem is just that 758 32bit childs are affected again. */ 759 current->personality &= ~READ_IMPLIES_EXEC; 760 } 761 762 asmlinkage long sys_fork(struct pt_regs *regs) 763 { 764 return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL); 765 } 766 767 asmlinkage long 768 sys_clone(unsigned long clone_flags, unsigned long newsp, 769 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs) 770 { 771 if (!newsp) 772 newsp = regs->sp; 773 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid); 774 } 775 776 /* 777 * This is trivial, and on the face of it looks like it 778 * could equally well be done in user mode. 779 * 780 * Not so, for quite unobvious reasons - register pressure. 781 * In user mode vfork() cannot have a stack frame, and if 782 * done by calling the "clone()" system call directly, you 783 * do not have enough call-clobbered registers to hold all 784 * the information you need. 785 */ 786 asmlinkage long sys_vfork(struct pt_regs *regs) 787 { 788 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0, 789 NULL, NULL); 790 } 791 792 unsigned long get_wchan(struct task_struct *p) 793 { 794 unsigned long stack; 795 u64 fp,ip; 796 int count = 0; 797 798 if (!p || p == current || p->state==TASK_RUNNING) 799 return 0; 800 stack = (unsigned long)task_stack_page(p); 801 if (p->thread.sp < stack || p->thread.sp > stack+THREAD_SIZE) 802 return 0; 803 fp = *(u64 *)(p->thread.sp); 804 do { 805 if (fp < (unsigned long)stack || 806 fp > (unsigned long)stack+THREAD_SIZE) 807 return 0; 808 ip = *(u64 *)(fp+8); 809 if (!in_sched_functions(ip)) 810 return ip; 811 fp = *(u64 *)fp; 812 } while (count++ < 16); 813 return 0; 814 } 815 816 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr) 817 { 818 int ret = 0; 819 int doit = task == current; 820 int cpu; 821 822 switch (code) { 823 case ARCH_SET_GS: 824 if (addr >= TASK_SIZE_OF(task)) 825 return -EPERM; 826 cpu = get_cpu(); 827 /* handle small bases via the GDT because that's faster to 828 switch. */ 829 if (addr <= 0xffffffff) { 830 set_32bit_tls(task, GS_TLS, addr); 831 if (doit) { 832 load_TLS(&task->thread, cpu); 833 load_gs_index(GS_TLS_SEL); 834 } 835 task->thread.gsindex = GS_TLS_SEL; 836 task->thread.gs = 0; 837 } else { 838 task->thread.gsindex = 0; 839 task->thread.gs = addr; 840 if (doit) { 841 load_gs_index(0); 842 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr); 843 } 844 } 845 put_cpu(); 846 break; 847 case ARCH_SET_FS: 848 /* Not strictly needed for fs, but do it for symmetry 849 with gs */ 850 if (addr >= TASK_SIZE_OF(task)) 851 return -EPERM; 852 cpu = get_cpu(); 853 /* handle small bases via the GDT because that's faster to 854 switch. */ 855 if (addr <= 0xffffffff) { 856 set_32bit_tls(task, FS_TLS, addr); 857 if (doit) { 858 load_TLS(&task->thread, cpu); 859 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL)); 860 } 861 task->thread.fsindex = FS_TLS_SEL; 862 task->thread.fs = 0; 863 } else { 864 task->thread.fsindex = 0; 865 task->thread.fs = addr; 866 if (doit) { 867 /* set the selector to 0 to not confuse 868 __switch_to */ 869 asm volatile("movl %0,%%fs" :: "r" (0)); 870 ret = checking_wrmsrl(MSR_FS_BASE, addr); 871 } 872 } 873 put_cpu(); 874 break; 875 case ARCH_GET_FS: { 876 unsigned long base; 877 if (task->thread.fsindex == FS_TLS_SEL) 878 base = read_32bit_tls(task, FS_TLS); 879 else if (doit) 880 rdmsrl(MSR_FS_BASE, base); 881 else 882 base = task->thread.fs; 883 ret = put_user(base, (unsigned long __user *)addr); 884 break; 885 } 886 case ARCH_GET_GS: { 887 unsigned long base; 888 unsigned gsindex; 889 if (task->thread.gsindex == GS_TLS_SEL) 890 base = read_32bit_tls(task, GS_TLS); 891 else if (doit) { 892 asm("movl %%gs,%0" : "=r" (gsindex)); 893 if (gsindex) 894 rdmsrl(MSR_KERNEL_GS_BASE, base); 895 else 896 base = task->thread.gs; 897 } 898 else 899 base = task->thread.gs; 900 ret = put_user(base, (unsigned long __user *)addr); 901 break; 902 } 903 904 default: 905 ret = -EINVAL; 906 break; 907 } 908 909 return ret; 910 } 911 912 long sys_arch_prctl(int code, unsigned long addr) 913 { 914 return do_arch_prctl(current, code, addr); 915 } 916 917 unsigned long arch_align_stack(unsigned long sp) 918 { 919 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) 920 sp -= get_random_int() % 8192; 921 return sp & ~0xf; 922 } 923 924 unsigned long arch_randomize_brk(struct mm_struct *mm) 925 { 926 unsigned long range_end = mm->brk + 0x02000000; 927 return randomize_range(mm->brk, range_end, 0) ? : mm->brk; 928 } 929