1 /* 2 * Copyright (C) 1995 Linus Torvalds 3 * 4 * Pentium III FXSR, SSE support 5 * Gareth Hughes <gareth@valinux.com>, May 2000 6 * 7 * X86-64 port 8 * Andi Kleen. 9 * 10 * CPU hotplug support - ashok.raj@intel.com 11 */ 12 13 /* 14 * This file handles the architecture-dependent parts of process handling.. 15 */ 16 17 #include <stdarg.h> 18 19 #include <linux/cpu.h> 20 #include <linux/errno.h> 21 #include <linux/sched.h> 22 #include <linux/fs.h> 23 #include <linux/kernel.h> 24 #include <linux/mm.h> 25 #include <linux/elfcore.h> 26 #include <linux/smp.h> 27 #include <linux/slab.h> 28 #include <linux/user.h> 29 #include <linux/interrupt.h> 30 #include <linux/utsname.h> 31 #include <linux/delay.h> 32 #include <linux/module.h> 33 #include <linux/ptrace.h> 34 #include <linux/random.h> 35 #include <linux/notifier.h> 36 #include <linux/kprobes.h> 37 #include <linux/kdebug.h> 38 #include <linux/tick.h> 39 40 #include <asm/uaccess.h> 41 #include <asm/pgtable.h> 42 #include <asm/system.h> 43 #include <asm/io.h> 44 #include <asm/processor.h> 45 #include <asm/i387.h> 46 #include <asm/mmu_context.h> 47 #include <asm/pda.h> 48 #include <asm/prctl.h> 49 #include <asm/desc.h> 50 #include <asm/proto.h> 51 #include <asm/ia32.h> 52 #include <asm/idle.h> 53 54 asmlinkage extern void ret_from_fork(void); 55 56 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; 57 58 unsigned long boot_option_idle_override = 0; 59 EXPORT_SYMBOL(boot_option_idle_override); 60 61 /* 62 * Powermanagement idle function, if any.. 63 */ 64 void (*pm_idle)(void); 65 EXPORT_SYMBOL(pm_idle); 66 static DEFINE_PER_CPU(unsigned int, cpu_idle_state); 67 68 static ATOMIC_NOTIFIER_HEAD(idle_notifier); 69 70 void idle_notifier_register(struct notifier_block *n) 71 { 72 atomic_notifier_chain_register(&idle_notifier, n); 73 } 74 75 void enter_idle(void) 76 { 77 write_pda(isidle, 1); 78 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); 79 } 80 81 static void __exit_idle(void) 82 { 83 if (test_and_clear_bit_pda(0, isidle) == 0) 84 return; 85 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); 86 } 87 88 /* Called from interrupts to signify idle end */ 89 void exit_idle(void) 90 { 91 /* idle loop has pid 0 */ 92 if (current->pid) 93 return; 94 __exit_idle(); 95 } 96 97 /* 98 * We use this if we don't have any better 99 * idle routine.. 100 */ 101 void default_idle(void) 102 { 103 current_thread_info()->status &= ~TS_POLLING; 104 /* 105 * TS_POLLING-cleared state must be visible before we 106 * test NEED_RESCHED: 107 */ 108 smp_mb(); 109 local_irq_disable(); 110 if (!need_resched()) { 111 ktime_t t0, t1; 112 u64 t0n, t1n; 113 114 t0 = ktime_get(); 115 t0n = ktime_to_ns(t0); 116 safe_halt(); /* enables interrupts racelessly */ 117 local_irq_disable(); 118 t1 = ktime_get(); 119 t1n = ktime_to_ns(t1); 120 sched_clock_idle_wakeup_event(t1n - t0n); 121 } 122 local_irq_enable(); 123 current_thread_info()->status |= TS_POLLING; 124 } 125 126 /* 127 * On SMP it's slightly faster (but much more power-consuming!) 128 * to poll the ->need_resched flag instead of waiting for the 129 * cross-CPU IPI to arrive. Use this option with caution. 130 */ 131 static void poll_idle(void) 132 { 133 local_irq_enable(); 134 cpu_relax(); 135 } 136 137 #ifdef CONFIG_HOTPLUG_CPU 138 DECLARE_PER_CPU(int, cpu_state); 139 140 #include <asm/nmi.h> 141 /* We halt the CPU with physical CPU hotplug */ 142 static inline void play_dead(void) 143 { 144 idle_task_exit(); 145 wbinvd(); 146 mb(); 147 /* Ack it */ 148 __get_cpu_var(cpu_state) = CPU_DEAD; 149 150 local_irq_disable(); 151 while (1) 152 halt(); 153 } 154 #else 155 static inline void play_dead(void) 156 { 157 BUG(); 158 } 159 #endif /* CONFIG_HOTPLUG_CPU */ 160 161 /* 162 * The idle thread. There's no useful work to be 163 * done, so just try to conserve power and have a 164 * low exit latency (ie sit in a loop waiting for 165 * somebody to say that they'd like to reschedule) 166 */ 167 void cpu_idle(void) 168 { 169 current_thread_info()->status |= TS_POLLING; 170 /* endless idle loop with no priority at all */ 171 while (1) { 172 tick_nohz_stop_sched_tick(); 173 while (!need_resched()) { 174 void (*idle)(void); 175 176 if (__get_cpu_var(cpu_idle_state)) 177 __get_cpu_var(cpu_idle_state) = 0; 178 179 rmb(); 180 idle = pm_idle; 181 if (!idle) 182 idle = default_idle; 183 if (cpu_is_offline(smp_processor_id())) 184 play_dead(); 185 /* 186 * Idle routines should keep interrupts disabled 187 * from here on, until they go to idle. 188 * Otherwise, idle callbacks can misfire. 189 */ 190 local_irq_disable(); 191 enter_idle(); 192 idle(); 193 /* In many cases the interrupt that ended idle 194 has already called exit_idle. But some idle 195 loops can be woken up without interrupt. */ 196 __exit_idle(); 197 } 198 199 tick_nohz_restart_sched_tick(); 200 preempt_enable_no_resched(); 201 schedule(); 202 preempt_disable(); 203 } 204 } 205 206 static void do_nothing(void *unused) 207 { 208 } 209 210 void cpu_idle_wait(void) 211 { 212 unsigned int cpu, this_cpu = get_cpu(); 213 cpumask_t map, tmp = current->cpus_allowed; 214 215 set_cpus_allowed(current, cpumask_of_cpu(this_cpu)); 216 put_cpu(); 217 218 cpus_clear(map); 219 for_each_online_cpu(cpu) { 220 per_cpu(cpu_idle_state, cpu) = 1; 221 cpu_set(cpu, map); 222 } 223 224 __get_cpu_var(cpu_idle_state) = 0; 225 226 wmb(); 227 do { 228 ssleep(1); 229 for_each_online_cpu(cpu) { 230 if (cpu_isset(cpu, map) && !per_cpu(cpu_idle_state, cpu)) 231 cpu_clear(cpu, map); 232 } 233 cpus_and(map, map, cpu_online_map); 234 /* 235 * We waited 1 sec, if a CPU still did not call idle 236 * it may be because it is in idle and not waking up 237 * because it has nothing to do. 238 * Give all the remaining CPUS a kick. 239 */ 240 smp_call_function_mask(map, do_nothing, 0, 0); 241 } while (!cpus_empty(map)); 242 243 set_cpus_allowed(current, tmp); 244 } 245 EXPORT_SYMBOL_GPL(cpu_idle_wait); 246 247 /* 248 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI, 249 * which can obviate IPI to trigger checking of need_resched. 250 * We execute MONITOR against need_resched and enter optimized wait state 251 * through MWAIT. Whenever someone changes need_resched, we would be woken 252 * up from MWAIT (without an IPI). 253 * 254 * New with Core Duo processors, MWAIT can take some hints based on CPU 255 * capability. 256 */ 257 void mwait_idle_with_hints(unsigned long ax, unsigned long cx) 258 { 259 if (!need_resched()) { 260 __monitor((void *)¤t_thread_info()->flags, 0, 0); 261 smp_mb(); 262 if (!need_resched()) 263 __mwait(ax, cx); 264 } 265 } 266 267 /* Default MONITOR/MWAIT with no hints, used for default C1 state */ 268 static void mwait_idle(void) 269 { 270 if (!need_resched()) { 271 __monitor((void *)¤t_thread_info()->flags, 0, 0); 272 smp_mb(); 273 if (!need_resched()) 274 __sti_mwait(0, 0); 275 else 276 local_irq_enable(); 277 } else { 278 local_irq_enable(); 279 } 280 } 281 282 283 static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c) 284 { 285 if (force_mwait) 286 return 1; 287 /* Any C1 states supported? */ 288 return c->cpuid_level >= 5 && ((cpuid_edx(5) >> 4) & 0xf) > 0; 289 } 290 291 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) 292 { 293 static int selected; 294 295 if (selected) 296 return; 297 #ifdef CONFIG_X86_SMP 298 if (pm_idle == poll_idle && smp_num_siblings > 1) { 299 printk(KERN_WARNING "WARNING: polling idle and HT enabled," 300 " performance may degrade.\n"); 301 } 302 #endif 303 if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) { 304 /* 305 * Skip, if setup has overridden idle. 306 * One CPU supports mwait => All CPUs supports mwait 307 */ 308 if (!pm_idle) { 309 printk(KERN_INFO "using mwait in idle threads.\n"); 310 pm_idle = mwait_idle; 311 } 312 } 313 selected = 1; 314 } 315 316 static int __init idle_setup(char *str) 317 { 318 if (!strcmp(str, "poll")) { 319 printk("using polling idle threads.\n"); 320 pm_idle = poll_idle; 321 } else if (!strcmp(str, "mwait")) 322 force_mwait = 1; 323 else 324 return -1; 325 326 boot_option_idle_override = 1; 327 return 0; 328 } 329 early_param("idle", idle_setup); 330 331 /* Prints also some state that isn't saved in the pt_regs */ 332 void __show_regs(struct pt_regs * regs) 333 { 334 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; 335 unsigned long d0, d1, d2, d3, d6, d7; 336 unsigned int fsindex, gsindex; 337 unsigned int ds, cs, es; 338 339 printk("\n"); 340 print_modules(); 341 printk("Pid: %d, comm: %.20s %s %s %.*s\n", 342 current->pid, current->comm, print_tainted(), 343 init_utsname()->release, 344 (int)strcspn(init_utsname()->version, " "), 345 init_utsname()->version); 346 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip); 347 printk_address(regs->ip, 1); 348 printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->sp, 349 regs->flags); 350 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n", 351 regs->ax, regs->bx, regs->cx); 352 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n", 353 regs->dx, regs->si, regs->di); 354 printk("RBP: %016lx R08: %016lx R09: %016lx\n", 355 regs->bp, regs->r8, regs->r9); 356 printk("R10: %016lx R11: %016lx R12: %016lx\n", 357 regs->r10, regs->r11, regs->r12); 358 printk("R13: %016lx R14: %016lx R15: %016lx\n", 359 regs->r13, regs->r14, regs->r15); 360 361 asm("movl %%ds,%0" : "=r" (ds)); 362 asm("movl %%cs,%0" : "=r" (cs)); 363 asm("movl %%es,%0" : "=r" (es)); 364 asm("movl %%fs,%0" : "=r" (fsindex)); 365 asm("movl %%gs,%0" : "=r" (gsindex)); 366 367 rdmsrl(MSR_FS_BASE, fs); 368 rdmsrl(MSR_GS_BASE, gs); 369 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); 370 371 cr0 = read_cr0(); 372 cr2 = read_cr2(); 373 cr3 = read_cr3(); 374 cr4 = read_cr4(); 375 376 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n", 377 fs,fsindex,gs,gsindex,shadowgs); 378 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0); 379 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4); 380 381 get_debugreg(d0, 0); 382 get_debugreg(d1, 1); 383 get_debugreg(d2, 2); 384 printk("DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2); 385 get_debugreg(d3, 3); 386 get_debugreg(d6, 6); 387 get_debugreg(d7, 7); 388 printk("DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7); 389 } 390 391 void show_regs(struct pt_regs *regs) 392 { 393 printk("CPU %d:", smp_processor_id()); 394 __show_regs(regs); 395 show_trace(NULL, regs, (void *)(regs + 1), regs->bp); 396 } 397 398 /* 399 * Free current thread data structures etc.. 400 */ 401 void exit_thread(void) 402 { 403 struct task_struct *me = current; 404 struct thread_struct *t = &me->thread; 405 406 if (me->thread.io_bitmap_ptr) { 407 struct tss_struct *tss = &per_cpu(init_tss, get_cpu()); 408 409 kfree(t->io_bitmap_ptr); 410 t->io_bitmap_ptr = NULL; 411 clear_thread_flag(TIF_IO_BITMAP); 412 /* 413 * Careful, clear this in the TSS too: 414 */ 415 memset(tss->io_bitmap, 0xff, t->io_bitmap_max); 416 t->io_bitmap_max = 0; 417 put_cpu(); 418 } 419 } 420 421 void flush_thread(void) 422 { 423 struct task_struct *tsk = current; 424 425 if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) { 426 clear_tsk_thread_flag(tsk, TIF_ABI_PENDING); 427 if (test_tsk_thread_flag(tsk, TIF_IA32)) { 428 clear_tsk_thread_flag(tsk, TIF_IA32); 429 } else { 430 set_tsk_thread_flag(tsk, TIF_IA32); 431 current_thread_info()->status |= TS_COMPAT; 432 } 433 } 434 clear_tsk_thread_flag(tsk, TIF_DEBUG); 435 436 tsk->thread.debugreg0 = 0; 437 tsk->thread.debugreg1 = 0; 438 tsk->thread.debugreg2 = 0; 439 tsk->thread.debugreg3 = 0; 440 tsk->thread.debugreg6 = 0; 441 tsk->thread.debugreg7 = 0; 442 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 443 /* 444 * Forget coprocessor state.. 445 */ 446 clear_fpu(tsk); 447 clear_used_math(); 448 } 449 450 void release_thread(struct task_struct *dead_task) 451 { 452 if (dead_task->mm) { 453 if (dead_task->mm->context.size) { 454 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n", 455 dead_task->comm, 456 dead_task->mm->context.ldt, 457 dead_task->mm->context.size); 458 BUG(); 459 } 460 } 461 } 462 463 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr) 464 { 465 struct user_desc ud = { 466 .base_addr = addr, 467 .limit = 0xfffff, 468 .seg_32bit = 1, 469 .limit_in_pages = 1, 470 .useable = 1, 471 }; 472 struct desc_struct *desc = t->thread.tls_array; 473 desc += tls; 474 fill_ldt(desc, &ud); 475 } 476 477 static inline u32 read_32bit_tls(struct task_struct *t, int tls) 478 { 479 return get_desc_base(&t->thread.tls_array[tls]); 480 } 481 482 /* 483 * This gets called before we allocate a new thread and copy 484 * the current task into it. 485 */ 486 void prepare_to_copy(struct task_struct *tsk) 487 { 488 unlazy_fpu(tsk); 489 } 490 491 int copy_thread(int nr, unsigned long clone_flags, unsigned long sp, 492 unsigned long unused, 493 struct task_struct * p, struct pt_regs * regs) 494 { 495 int err; 496 struct pt_regs * childregs; 497 struct task_struct *me = current; 498 499 childregs = ((struct pt_regs *) 500 (THREAD_SIZE + task_stack_page(p))) - 1; 501 *childregs = *regs; 502 503 childregs->ax = 0; 504 childregs->sp = sp; 505 if (sp == ~0UL) 506 childregs->sp = (unsigned long)childregs; 507 508 p->thread.sp = (unsigned long) childregs; 509 p->thread.sp0 = (unsigned long) (childregs+1); 510 p->thread.usersp = me->thread.usersp; 511 512 set_tsk_thread_flag(p, TIF_FORK); 513 514 p->thread.fs = me->thread.fs; 515 p->thread.gs = me->thread.gs; 516 517 asm("mov %%gs,%0" : "=m" (p->thread.gsindex)); 518 asm("mov %%fs,%0" : "=m" (p->thread.fsindex)); 519 asm("mov %%es,%0" : "=m" (p->thread.es)); 520 asm("mov %%ds,%0" : "=m" (p->thread.ds)); 521 522 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) { 523 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); 524 if (!p->thread.io_bitmap_ptr) { 525 p->thread.io_bitmap_max = 0; 526 return -ENOMEM; 527 } 528 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr, 529 IO_BITMAP_BYTES); 530 set_tsk_thread_flag(p, TIF_IO_BITMAP); 531 } 532 533 /* 534 * Set a new TLS for the child thread? 535 */ 536 if (clone_flags & CLONE_SETTLS) { 537 #ifdef CONFIG_IA32_EMULATION 538 if (test_thread_flag(TIF_IA32)) 539 err = do_set_thread_area(p, -1, 540 (struct user_desc __user *)childregs->si, 0); 541 else 542 #endif 543 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8); 544 if (err) 545 goto out; 546 } 547 err = 0; 548 out: 549 if (err && p->thread.io_bitmap_ptr) { 550 kfree(p->thread.io_bitmap_ptr); 551 p->thread.io_bitmap_max = 0; 552 } 553 return err; 554 } 555 556 /* 557 * This special macro can be used to load a debugging register 558 */ 559 #define loaddebug(thread, r) set_debugreg(thread->debugreg ## r, r) 560 561 static inline void __switch_to_xtra(struct task_struct *prev_p, 562 struct task_struct *next_p, 563 struct tss_struct *tss) 564 { 565 struct thread_struct *prev, *next; 566 unsigned long debugctl; 567 568 prev = &prev_p->thread, 569 next = &next_p->thread; 570 571 debugctl = prev->debugctlmsr; 572 if (next->ds_area_msr != prev->ds_area_msr) { 573 /* we clear debugctl to make sure DS 574 * is not in use when we change it */ 575 debugctl = 0; 576 wrmsrl(MSR_IA32_DEBUGCTLMSR, 0); 577 wrmsrl(MSR_IA32_DS_AREA, next->ds_area_msr); 578 } 579 580 if (next->debugctlmsr != debugctl) 581 wrmsrl(MSR_IA32_DEBUGCTLMSR, next->debugctlmsr); 582 583 if (test_tsk_thread_flag(next_p, TIF_DEBUG)) { 584 loaddebug(next, 0); 585 loaddebug(next, 1); 586 loaddebug(next, 2); 587 loaddebug(next, 3); 588 /* no 4 and 5 */ 589 loaddebug(next, 6); 590 loaddebug(next, 7); 591 } 592 593 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { 594 /* 595 * Copy the relevant range of the IO bitmap. 596 * Normally this is 128 bytes or less: 597 */ 598 memcpy(tss->io_bitmap, next->io_bitmap_ptr, 599 max(prev->io_bitmap_max, next->io_bitmap_max)); 600 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) { 601 /* 602 * Clear any possible leftover bits: 603 */ 604 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); 605 } 606 607 if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) 608 ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS); 609 610 if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS)) 611 ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES); 612 } 613 614 /* 615 * switch_to(x,y) should switch tasks from x to y. 616 * 617 * This could still be optimized: 618 * - fold all the options into a flag word and test it with a single test. 619 * - could test fs/gs bitsliced 620 * 621 * Kprobes not supported here. Set the probe on schedule instead. 622 */ 623 struct task_struct * 624 __switch_to(struct task_struct *prev_p, struct task_struct *next_p) 625 { 626 struct thread_struct *prev = &prev_p->thread, 627 *next = &next_p->thread; 628 int cpu = smp_processor_id(); 629 struct tss_struct *tss = &per_cpu(init_tss, cpu); 630 631 /* we're going to use this soon, after a few expensive things */ 632 if (next_p->fpu_counter>5) 633 prefetch(&next->i387.fxsave); 634 635 /* 636 * Reload esp0, LDT and the page table pointer: 637 */ 638 load_sp0(tss, next); 639 640 /* 641 * Switch DS and ES. 642 * This won't pick up thread selector changes, but I guess that is ok. 643 */ 644 asm volatile("mov %%es,%0" : "=m" (prev->es)); 645 if (unlikely(next->es | prev->es)) 646 loadsegment(es, next->es); 647 648 asm volatile ("mov %%ds,%0" : "=m" (prev->ds)); 649 if (unlikely(next->ds | prev->ds)) 650 loadsegment(ds, next->ds); 651 652 load_TLS(next, cpu); 653 654 /* 655 * Switch FS and GS. 656 */ 657 { 658 unsigned fsindex; 659 asm volatile("movl %%fs,%0" : "=r" (fsindex)); 660 /* segment register != 0 always requires a reload. 661 also reload when it has changed. 662 when prev process used 64bit base always reload 663 to avoid an information leak. */ 664 if (unlikely(fsindex | next->fsindex | prev->fs)) { 665 loadsegment(fs, next->fsindex); 666 /* check if the user used a selector != 0 667 * if yes clear 64bit base, since overloaded base 668 * is always mapped to the Null selector 669 */ 670 if (fsindex) 671 prev->fs = 0; 672 } 673 /* when next process has a 64bit base use it */ 674 if (next->fs) 675 wrmsrl(MSR_FS_BASE, next->fs); 676 prev->fsindex = fsindex; 677 } 678 { 679 unsigned gsindex; 680 asm volatile("movl %%gs,%0" : "=r" (gsindex)); 681 if (unlikely(gsindex | next->gsindex | prev->gs)) { 682 load_gs_index(next->gsindex); 683 if (gsindex) 684 prev->gs = 0; 685 } 686 if (next->gs) 687 wrmsrl(MSR_KERNEL_GS_BASE, next->gs); 688 prev->gsindex = gsindex; 689 } 690 691 /* Must be after DS reload */ 692 unlazy_fpu(prev_p); 693 694 /* 695 * Switch the PDA and FPU contexts. 696 */ 697 prev->usersp = read_pda(oldrsp); 698 write_pda(oldrsp, next->usersp); 699 write_pda(pcurrent, next_p); 700 701 write_pda(kernelstack, 702 (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET); 703 #ifdef CONFIG_CC_STACKPROTECTOR 704 write_pda(stack_canary, next_p->stack_canary); 705 /* 706 * Build time only check to make sure the stack_canary is at 707 * offset 40 in the pda; this is a gcc ABI requirement 708 */ 709 BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40); 710 #endif 711 712 /* 713 * Now maybe reload the debug registers and handle I/O bitmaps 714 */ 715 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT || 716 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) 717 __switch_to_xtra(prev_p, next_p, tss); 718 719 /* If the task has used fpu the last 5 timeslices, just do a full 720 * restore of the math state immediately to avoid the trap; the 721 * chances of needing FPU soon are obviously high now 722 */ 723 if (next_p->fpu_counter>5) 724 math_state_restore(); 725 return prev_p; 726 } 727 728 /* 729 * sys_execve() executes a new program. 730 */ 731 asmlinkage 732 long sys_execve(char __user *name, char __user * __user *argv, 733 char __user * __user *envp, struct pt_regs regs) 734 { 735 long error; 736 char * filename; 737 738 filename = getname(name); 739 error = PTR_ERR(filename); 740 if (IS_ERR(filename)) 741 return error; 742 error = do_execve(filename, argv, envp, ®s); 743 putname(filename); 744 return error; 745 } 746 747 void set_personality_64bit(void) 748 { 749 /* inherit personality from parent */ 750 751 /* Make sure to be in 64bit mode */ 752 clear_thread_flag(TIF_IA32); 753 754 /* TBD: overwrites user setup. Should have two bits. 755 But 64bit processes have always behaved this way, 756 so it's not too bad. The main problem is just that 757 32bit childs are affected again. */ 758 current->personality &= ~READ_IMPLIES_EXEC; 759 } 760 761 asmlinkage long sys_fork(struct pt_regs *regs) 762 { 763 return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL); 764 } 765 766 asmlinkage long 767 sys_clone(unsigned long clone_flags, unsigned long newsp, 768 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs) 769 { 770 if (!newsp) 771 newsp = regs->sp; 772 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid); 773 } 774 775 /* 776 * This is trivial, and on the face of it looks like it 777 * could equally well be done in user mode. 778 * 779 * Not so, for quite unobvious reasons - register pressure. 780 * In user mode vfork() cannot have a stack frame, and if 781 * done by calling the "clone()" system call directly, you 782 * do not have enough call-clobbered registers to hold all 783 * the information you need. 784 */ 785 asmlinkage long sys_vfork(struct pt_regs *regs) 786 { 787 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0, 788 NULL, NULL); 789 } 790 791 unsigned long get_wchan(struct task_struct *p) 792 { 793 unsigned long stack; 794 u64 fp,ip; 795 int count = 0; 796 797 if (!p || p == current || p->state==TASK_RUNNING) 798 return 0; 799 stack = (unsigned long)task_stack_page(p); 800 if (p->thread.sp < stack || p->thread.sp > stack+THREAD_SIZE) 801 return 0; 802 fp = *(u64 *)(p->thread.sp); 803 do { 804 if (fp < (unsigned long)stack || 805 fp > (unsigned long)stack+THREAD_SIZE) 806 return 0; 807 ip = *(u64 *)(fp+8); 808 if (!in_sched_functions(ip)) 809 return ip; 810 fp = *(u64 *)fp; 811 } while (count++ < 16); 812 return 0; 813 } 814 815 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr) 816 { 817 int ret = 0; 818 int doit = task == current; 819 int cpu; 820 821 switch (code) { 822 case ARCH_SET_GS: 823 if (addr >= TASK_SIZE_OF(task)) 824 return -EPERM; 825 cpu = get_cpu(); 826 /* handle small bases via the GDT because that's faster to 827 switch. */ 828 if (addr <= 0xffffffff) { 829 set_32bit_tls(task, GS_TLS, addr); 830 if (doit) { 831 load_TLS(&task->thread, cpu); 832 load_gs_index(GS_TLS_SEL); 833 } 834 task->thread.gsindex = GS_TLS_SEL; 835 task->thread.gs = 0; 836 } else { 837 task->thread.gsindex = 0; 838 task->thread.gs = addr; 839 if (doit) { 840 load_gs_index(0); 841 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr); 842 } 843 } 844 put_cpu(); 845 break; 846 case ARCH_SET_FS: 847 /* Not strictly needed for fs, but do it for symmetry 848 with gs */ 849 if (addr >= TASK_SIZE_OF(task)) 850 return -EPERM; 851 cpu = get_cpu(); 852 /* handle small bases via the GDT because that's faster to 853 switch. */ 854 if (addr <= 0xffffffff) { 855 set_32bit_tls(task, FS_TLS, addr); 856 if (doit) { 857 load_TLS(&task->thread, cpu); 858 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL)); 859 } 860 task->thread.fsindex = FS_TLS_SEL; 861 task->thread.fs = 0; 862 } else { 863 task->thread.fsindex = 0; 864 task->thread.fs = addr; 865 if (doit) { 866 /* set the selector to 0 to not confuse 867 __switch_to */ 868 asm volatile("movl %0,%%fs" :: "r" (0)); 869 ret = checking_wrmsrl(MSR_FS_BASE, addr); 870 } 871 } 872 put_cpu(); 873 break; 874 case ARCH_GET_FS: { 875 unsigned long base; 876 if (task->thread.fsindex == FS_TLS_SEL) 877 base = read_32bit_tls(task, FS_TLS); 878 else if (doit) 879 rdmsrl(MSR_FS_BASE, base); 880 else 881 base = task->thread.fs; 882 ret = put_user(base, (unsigned long __user *)addr); 883 break; 884 } 885 case ARCH_GET_GS: { 886 unsigned long base; 887 unsigned gsindex; 888 if (task->thread.gsindex == GS_TLS_SEL) 889 base = read_32bit_tls(task, GS_TLS); 890 else if (doit) { 891 asm("movl %%gs,%0" : "=r" (gsindex)); 892 if (gsindex) 893 rdmsrl(MSR_KERNEL_GS_BASE, base); 894 else 895 base = task->thread.gs; 896 } 897 else 898 base = task->thread.gs; 899 ret = put_user(base, (unsigned long __user *)addr); 900 break; 901 } 902 903 default: 904 ret = -EINVAL; 905 break; 906 } 907 908 return ret; 909 } 910 911 long sys_arch_prctl(int code, unsigned long addr) 912 { 913 return do_arch_prctl(current, code, addr); 914 } 915 916 unsigned long arch_align_stack(unsigned long sp) 917 { 918 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) 919 sp -= get_random_int() % 8192; 920 return sp & ~0xf; 921 } 922 923 unsigned long arch_randomize_brk(struct mm_struct *mm) 924 { 925 unsigned long range_end = mm->brk + 0x02000000; 926 return randomize_range(mm->brk, range_end, 0) ? : mm->brk; 927 } 928