blob: cbbc6adc1a92004c902954f10777528f6b964178 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/kernel/process.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Pentium III FXSR, SSE support
7 * Gareth Hughes <gareth@valinux.com>, May 2000
8 *
9 * X86-64 port
10 * Andi Kleen.
Ashok Raj76e4f662005-06-25 14:55:00 -070011 *
12 * CPU hotplug support - ashok.raj@intel.com
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
14
15/*
16 * This file handles the architecture-dependent parts of process handling..
17 */
18
19#include <stdarg.h>
20
Ashok Raj76e4f662005-06-25 14:55:00 -070021#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/errno.h>
23#include <linux/sched.h>
24#include <linux/kernel.h>
25#include <linux/mm.h>
26#include <linux/elfcore.h>
27#include <linux/smp.h>
28#include <linux/slab.h>
29#include <linux/user.h>
30#include <linux/module.h>
31#include <linux/a.out.h>
32#include <linux/interrupt.h>
33#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <linux/ptrace.h>
35#include <linux/utsname.h>
36#include <linux/random.h>
Andi Kleen95833c82006-01-11 22:44:36 +010037#include <linux/notifier.h>
bibo maoc6fd91f2006-03-26 01:38:20 -080038#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
40#include <asm/uaccess.h>
41#include <asm/pgtable.h>
42#include <asm/system.h>
43#include <asm/io.h>
44#include <asm/processor.h>
45#include <asm/i387.h>
46#include <asm/mmu_context.h>
47#include <asm/pda.h>
48#include <asm/prctl.h>
49#include <asm/kdebug.h>
50#include <asm/desc.h>
51#include <asm/proto.h>
52#include <asm/ia32.h>
Andi Kleen95833c82006-01-11 22:44:36 +010053#include <asm/idle.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55asmlinkage extern void ret_from_fork(void);
56
57unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059unsigned long boot_option_idle_override = 0;
60EXPORT_SYMBOL(boot_option_idle_override);
61
62/*
63 * Powermanagement idle function, if any..
64 */
65void (*pm_idle)(void);
Andi Kleen2ee60e172006-06-26 13:59:44 +020066EXPORT_SYMBOL(pm_idle);
Linus Torvalds1da177e2005-04-16 15:20:36 -070067static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
68
Alan Sterne041c682006-03-27 01:16:30 -080069static ATOMIC_NOTIFIER_HEAD(idle_notifier);
Andi Kleen95833c82006-01-11 22:44:36 +010070
71void idle_notifier_register(struct notifier_block *n)
72{
Alan Sterne041c682006-03-27 01:16:30 -080073 atomic_notifier_chain_register(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010074}
75EXPORT_SYMBOL_GPL(idle_notifier_register);
76
77void idle_notifier_unregister(struct notifier_block *n)
78{
Alan Sterne041c682006-03-27 01:16:30 -080079 atomic_notifier_chain_unregister(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010080}
81EXPORT_SYMBOL(idle_notifier_unregister);
82
Andi Kleen95833c82006-01-11 22:44:36 +010083void enter_idle(void)
84{
Andi Kleena15da492006-09-26 10:52:40 +020085 write_pda(isidle, 1);
Alan Sterne041c682006-03-27 01:16:30 -080086 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010087}
88
89static void __exit_idle(void)
90{
Andi Kleen94468682006-11-14 16:57:46 +010091 if (test_and_clear_bit_pda(0, isidle) == 0)
Andi Kleena15da492006-09-26 10:52:40 +020092 return;
Alan Sterne041c682006-03-27 01:16:30 -080093 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010094}
95
96/* Called from interrupts to signify idle end */
97void exit_idle(void)
98{
Andi Kleena15da492006-09-26 10:52:40 +020099 /* idle loop has pid 0 */
100 if (current->pid)
Andi Kleen95833c82006-01-11 22:44:36 +0100101 return;
102 __exit_idle();
103}
104
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105/*
106 * We use this if we don't have any better
107 * idle routine..
108 */
Adrian Bunkcdb04522006-03-24 03:15:57 -0800109static void default_idle(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110{
Andi Kleen495ab9c2006-06-26 13:59:11 +0200111 current_thread_info()->status &= ~TS_POLLING;
Ingo Molnar0888f062006-12-22 01:11:56 -0800112 /*
113 * TS_POLLING-cleared state must be visible before we
114 * test NEED_RESCHED:
115 */
116 smp_mb();
Andi Kleen72690a22006-12-07 02:14:03 +0100117 local_irq_disable();
118 if (!need_resched()) {
119 /* Enables interrupts one instruction before HLT.
120 x86 special cases this so there is no race. */
121 safe_halt();
122 } else
123 local_irq_enable();
Andi Kleen495ab9c2006-06-26 13:59:11 +0200124 current_thread_info()->status |= TS_POLLING;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125}
126
127/*
128 * On SMP it's slightly faster (but much more power-consuming!)
129 * to poll the ->need_resched flag instead of waiting for the
130 * cross-CPU IPI to arrive. Use this option with caution.
131 */
132static void poll_idle (void)
133{
Venkatesh Pallipadid331e732006-12-07 02:14:13 +0100134 local_irq_enable();
Andi Kleen72690a22006-12-07 02:14:03 +0100135 cpu_relax();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136}
137
138void cpu_idle_wait(void)
139{
140 unsigned int cpu, this_cpu = get_cpu();
Ingo Molnardc1829a2006-11-17 14:26:18 +0100141 cpumask_t map, tmp = current->cpus_allowed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
143 set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
144 put_cpu();
145
146 cpus_clear(map);
147 for_each_online_cpu(cpu) {
148 per_cpu(cpu_idle_state, cpu) = 1;
149 cpu_set(cpu, map);
150 }
151
152 __get_cpu_var(cpu_idle_state) = 0;
153
154 wmb();
155 do {
156 ssleep(1);
157 for_each_online_cpu(cpu) {
Andi Kleena88cde12005-11-05 17:25:54 +0100158 if (cpu_isset(cpu, map) &&
159 !per_cpu(cpu_idle_state, cpu))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 cpu_clear(cpu, map);
161 }
162 cpus_and(map, map, cpu_online_map);
163 } while (!cpus_empty(map));
Ingo Molnardc1829a2006-11-17 14:26:18 +0100164
165 set_cpus_allowed(current, tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166}
167EXPORT_SYMBOL_GPL(cpu_idle_wait);
168
Ashok Raj76e4f662005-06-25 14:55:00 -0700169#ifdef CONFIG_HOTPLUG_CPU
170DECLARE_PER_CPU(int, cpu_state);
171
172#include <asm/nmi.h>
Shaohua Li1fa744e2006-01-06 00:12:20 -0800173/* We halt the CPU with physical CPU hotplug */
Ashok Raj76e4f662005-06-25 14:55:00 -0700174static inline void play_dead(void)
175{
176 idle_task_exit();
177 wbinvd();
178 mb();
179 /* Ack it */
180 __get_cpu_var(cpu_state) = CPU_DEAD;
181
Shaohua Li1fa744e2006-01-06 00:12:20 -0800182 local_irq_disable();
Ashok Raj76e4f662005-06-25 14:55:00 -0700183 while (1)
Shaohua Li1fa744e2006-01-06 00:12:20 -0800184 halt();
Ashok Raj76e4f662005-06-25 14:55:00 -0700185}
186#else
187static inline void play_dead(void)
188{
189 BUG();
190}
191#endif /* CONFIG_HOTPLUG_CPU */
192
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193/*
194 * The idle thread. There's no useful work to be
195 * done, so just try to conserve power and have a
196 * low exit latency (ie sit in a loop waiting for
197 * somebody to say that they'd like to reschedule)
198 */
199void cpu_idle (void)
200{
Andi Kleen495ab9c2006-06-26 13:59:11 +0200201 current_thread_info()->status |= TS_POLLING;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 /* endless idle loop with no priority at all */
203 while (1) {
204 while (!need_resched()) {
205 void (*idle)(void);
206
207 if (__get_cpu_var(cpu_idle_state))
208 __get_cpu_var(cpu_idle_state) = 0;
209
210 rmb();
211 idle = pm_idle;
212 if (!idle)
213 idle = default_idle;
Ashok Raj76e4f662005-06-25 14:55:00 -0700214 if (cpu_is_offline(smp_processor_id()))
215 play_dead();
Venkatesh Pallipadid331e732006-12-07 02:14:13 +0100216 /*
217 * Idle routines should keep interrupts disabled
218 * from here on, until they go to idle.
219 * Otherwise, idle callbacks can misfire.
220 */
221 local_irq_disable();
Andi Kleen95833c82006-01-11 22:44:36 +0100222 enter_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 idle();
Andi Kleena15da492006-09-26 10:52:40 +0200224 /* In many cases the interrupt that ended idle
225 has already called exit_idle. But some idle
226 loops can be woken up without interrupt. */
Andi Kleen95833c82006-01-11 22:44:36 +0100227 __exit_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 }
229
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800230 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 schedule();
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800232 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 }
234}
235
236/*
237 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
238 * which can obviate IPI to trigger checking of need_resched.
239 * We execute MONITOR against need_resched and enter optimized wait state
240 * through MWAIT. Whenever someone changes need_resched, we would be woken
241 * up from MWAIT (without an IPI).
Venkatesh Pallipadi991528d2006-09-25 16:28:13 -0700242 *
243 * New with Core Duo processors, MWAIT can take some hints based on CPU
244 * capability.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 */
Venkatesh Pallipadi991528d2006-09-25 16:28:13 -0700246void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
247{
248 if (!need_resched()) {
249 __monitor((void *)&current_thread_info()->flags, 0, 0);
250 smp_mb();
251 if (!need_resched())
252 __mwait(eax, ecx);
253 }
254}
255
256/* Default MONITOR/MWAIT with no hints, used for default C1 state */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257static void mwait_idle(void)
258{
Venkatesh Pallipadid331e732006-12-07 02:14:13 +0100259 if (!need_resched()) {
260 __monitor((void *)&current_thread_info()->flags, 0, 0);
261 smp_mb();
262 if (!need_resched())
263 __sti_mwait(0, 0);
264 else
265 local_irq_enable();
266 } else {
267 local_irq_enable();
268 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269}
270
Ashok Raje6982c62005-06-25 14:54:58 -0700271void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272{
273 static int printed;
274 if (cpu_has(c, X86_FEATURE_MWAIT)) {
275 /*
276 * Skip, if setup has overridden idle.
277 * One CPU supports mwait => All CPUs supports mwait
278 */
279 if (!pm_idle) {
280 if (!printed) {
281 printk("using mwait in idle threads.\n");
282 printed = 1;
283 }
284 pm_idle = mwait_idle;
285 }
286 }
287}
288
289static int __init idle_setup (char *str)
290{
291 if (!strncmp(str, "poll", 4)) {
292 printk("using polling idle threads.\n");
293 pm_idle = poll_idle;
294 }
295
296 boot_option_idle_override = 1;
297 return 1;
298}
299
300__setup("idle=", idle_setup);
301
302/* Prints also some state that isn't saved in the pt_regs */
303void __show_regs(struct pt_regs * regs)
304{
305 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
306 unsigned int fsindex,gsindex;
307 unsigned int ds,cs,es;
308
309 printk("\n");
310 print_modules();
Andi Kleen9acf23c2005-09-12 18:49:24 +0200311 printk("Pid: %d, comm: %.20s %s %s %.*s\n",
312 current->pid, current->comm, print_tainted(),
Serge E. Hallyn96b644b2006-10-02 02:18:13 -0700313 init_utsname()->release,
314 (int)strcspn(init_utsname()->version, " "),
315 init_utsname()->version);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
317 printk_address(regs->rip);
Ingo Molnar3ac94932006-07-03 00:24:36 -0700318 printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp,
Andi Kleena88cde12005-11-05 17:25:54 +0100319 regs->eflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
321 regs->rax, regs->rbx, regs->rcx);
322 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
323 regs->rdx, regs->rsi, regs->rdi);
324 printk("RBP: %016lx R08: %016lx R09: %016lx\n",
325 regs->rbp, regs->r8, regs->r9);
326 printk("R10: %016lx R11: %016lx R12: %016lx\n",
327 regs->r10, regs->r11, regs->r12);
328 printk("R13: %016lx R14: %016lx R15: %016lx\n",
329 regs->r13, regs->r14, regs->r15);
330
331 asm("movl %%ds,%0" : "=r" (ds));
332 asm("movl %%cs,%0" : "=r" (cs));
333 asm("movl %%es,%0" : "=r" (es));
334 asm("movl %%fs,%0" : "=r" (fsindex));
335 asm("movl %%gs,%0" : "=r" (gsindex));
336
337 rdmsrl(MSR_FS_BASE, fs);
338 rdmsrl(MSR_GS_BASE, gs);
339 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
340
341 asm("movq %%cr0, %0": "=r" (cr0));
342 asm("movq %%cr2, %0": "=r" (cr2));
343 asm("movq %%cr3, %0": "=r" (cr3));
344 asm("movq %%cr4, %0": "=r" (cr4));
345
346 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
347 fs,fsindex,gs,gsindex,shadowgs);
348 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
349 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
350}
351
352void show_regs(struct pt_regs *regs)
353{
Zwane Mwaikamboc078d322005-09-06 15:16:16 -0700354 printk("CPU %d:", smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 __show_regs(regs);
Jan Beulichb538ed22006-06-26 13:57:32 +0200356 show_trace(NULL, regs, (void *)(regs + 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357}
358
359/*
360 * Free current thread data structures etc..
361 */
362void exit_thread(void)
363{
364 struct task_struct *me = current;
365 struct thread_struct *t = &me->thread;
Rusty Lynch73649da2005-06-23 00:09:23 -0700366
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 if (me->thread.io_bitmap_ptr) {
368 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
369
370 kfree(t->io_bitmap_ptr);
371 t->io_bitmap_ptr = NULL;
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200372 clear_thread_flag(TIF_IO_BITMAP);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 /*
374 * Careful, clear this in the TSS too:
375 */
376 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
377 t->io_bitmap_max = 0;
378 put_cpu();
379 }
380}
381
382void flush_thread(void)
383{
384 struct task_struct *tsk = current;
385 struct thread_info *t = current_thread_info();
386
Andi Kleen4d9bc792006-06-26 13:57:19 +0200387 if (t->flags & _TIF_ABI_PENDING) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
Andi Kleen4d9bc792006-06-26 13:57:19 +0200389 if (t->flags & _TIF_IA32)
390 current_thread_info()->status |= TS_COMPAT;
391 }
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200392 t->flags &= ~_TIF_DEBUG;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393
394 tsk->thread.debugreg0 = 0;
395 tsk->thread.debugreg1 = 0;
396 tsk->thread.debugreg2 = 0;
397 tsk->thread.debugreg3 = 0;
398 tsk->thread.debugreg6 = 0;
399 tsk->thread.debugreg7 = 0;
400 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
401 /*
402 * Forget coprocessor state..
403 */
404 clear_fpu(tsk);
405 clear_used_math();
406}
407
408void release_thread(struct task_struct *dead_task)
409{
410 if (dead_task->mm) {
411 if (dead_task->mm->context.size) {
412 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
413 dead_task->comm,
414 dead_task->mm->context.ldt,
415 dead_task->mm->context.size);
416 BUG();
417 }
418 }
419}
420
421static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
422{
423 struct user_desc ud = {
424 .base_addr = addr,
425 .limit = 0xfffff,
426 .seg_32bit = 1,
427 .limit_in_pages = 1,
428 .useable = 1,
429 };
430 struct n_desc_struct *desc = (void *)t->thread.tls_array;
431 desc += tls;
432 desc->a = LDT_entry_a(&ud);
433 desc->b = LDT_entry_b(&ud);
434}
435
436static inline u32 read_32bit_tls(struct task_struct *t, int tls)
437{
438 struct desc_struct *desc = (void *)t->thread.tls_array;
439 desc += tls;
440 return desc->base0 |
441 (((u32)desc->base1) << 16) |
442 (((u32)desc->base2) << 24);
443}
444
445/*
446 * This gets called before we allocate a new thread and copy
447 * the current task into it.
448 */
449void prepare_to_copy(struct task_struct *tsk)
450{
451 unlazy_fpu(tsk);
452}
453
454int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
455 unsigned long unused,
456 struct task_struct * p, struct pt_regs * regs)
457{
458 int err;
459 struct pt_regs * childregs;
460 struct task_struct *me = current;
461
Andi Kleena88cde12005-11-05 17:25:54 +0100462 childregs = ((struct pt_regs *)
Al Viro57eafdc2006-01-12 01:05:39 -0800463 (THREAD_SIZE + task_stack_page(p))) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 *childregs = *regs;
465
466 childregs->rax = 0;
467 childregs->rsp = rsp;
Andi Kleena88cde12005-11-05 17:25:54 +0100468 if (rsp == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 childregs->rsp = (unsigned long)childregs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470
471 p->thread.rsp = (unsigned long) childregs;
472 p->thread.rsp0 = (unsigned long) (childregs+1);
473 p->thread.userrsp = me->thread.userrsp;
474
Al Viroe4f17c42006-01-12 01:05:38 -0800475 set_tsk_thread_flag(p, TIF_FORK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
477 p->thread.fs = me->thread.fs;
478 p->thread.gs = me->thread.gs;
479
H. J. Lufd51f662005-05-01 08:58:48 -0700480 asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
481 asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
482 asm("mov %%es,%0" : "=m" (p->thread.es));
483 asm("mov %%ds,%0" : "=m" (p->thread.ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200485 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
487 if (!p->thread.io_bitmap_ptr) {
488 p->thread.io_bitmap_max = 0;
489 return -ENOMEM;
490 }
Andi Kleena88cde12005-11-05 17:25:54 +0100491 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
492 IO_BITMAP_BYTES);
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200493 set_tsk_thread_flag(p, TIF_IO_BITMAP);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 }
495
496 /*
497 * Set a new TLS for the child thread?
498 */
499 if (clone_flags & CLONE_SETTLS) {
500#ifdef CONFIG_IA32_EMULATION
501 if (test_thread_flag(TIF_IA32))
502 err = ia32_child_tls(p, childregs);
503 else
504#endif
505 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
506 if (err)
507 goto out;
508 }
509 err = 0;
510out:
511 if (err && p->thread.io_bitmap_ptr) {
512 kfree(p->thread.io_bitmap_ptr);
513 p->thread.io_bitmap_max = 0;
514 }
515 return err;
516}
517
518/*
519 * This special macro can be used to load a debugging register
520 */
Jan Beulich2b514e72006-03-25 16:29:22 +0100521#define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200523static inline void __switch_to_xtra(struct task_struct *prev_p,
524 struct task_struct *next_p,
525 struct tss_struct *tss)
526{
527 struct thread_struct *prev, *next;
528
529 prev = &prev_p->thread,
530 next = &next_p->thread;
531
532 if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
533 loaddebug(next, 0);
534 loaddebug(next, 1);
535 loaddebug(next, 2);
536 loaddebug(next, 3);
537 /* no 4 and 5 */
538 loaddebug(next, 6);
539 loaddebug(next, 7);
540 }
541
542 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
543 /*
544 * Copy the relevant range of the IO bitmap.
545 * Normally this is 128 bytes or less:
546 */
547 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
548 max(prev->io_bitmap_max, next->io_bitmap_max));
549 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
550 /*
551 * Clear any possible leftover bits:
552 */
553 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
554 }
555}
556
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557/*
558 * switch_to(x,y) should switch tasks from x to y.
559 *
560 * This could still be optimized:
561 * - fold all the options into a flag word and test it with a single test.
562 * - could test fs/gs bitsliced
Andi Kleen099f3182006-02-03 21:51:38 +0100563 *
564 * Kprobes not supported here. Set the probe on schedule instead.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 */
Andi Kleen099f3182006-02-03 21:51:38 +0100566__kprobes struct task_struct *
Andi Kleena88cde12005-11-05 17:25:54 +0100567__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568{
569 struct thread_struct *prev = &prev_p->thread,
570 *next = &next_p->thread;
571 int cpu = smp_processor_id();
572 struct tss_struct *tss = &per_cpu(init_tss, cpu);
573
Arjan van de Vene07e23e2006-09-26 10:52:36 +0200574 /* we're going to use this soon, after a few expensive things */
575 if (next_p->fpu_counter>5)
576 prefetch(&next->i387.fxsave);
577
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 /*
579 * Reload esp0, LDT and the page table pointer:
580 */
581 tss->rsp0 = next->rsp0;
582
583 /*
584 * Switch DS and ES.
585 * This won't pick up thread selector changes, but I guess that is ok.
586 */
H. J. Lufd51f662005-05-01 08:58:48 -0700587 asm volatile("mov %%es,%0" : "=m" (prev->es));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 if (unlikely(next->es | prev->es))
589 loadsegment(es, next->es);
590
H. J. Lufd51f662005-05-01 08:58:48 -0700591 asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 if (unlikely(next->ds | prev->ds))
593 loadsegment(ds, next->ds);
594
595 load_TLS(next, cpu);
596
597 /*
598 * Switch FS and GS.
599 */
600 {
601 unsigned fsindex;
602 asm volatile("movl %%fs,%0" : "=r" (fsindex));
603 /* segment register != 0 always requires a reload.
604 also reload when it has changed.
605 when prev process used 64bit base always reload
606 to avoid an information leak. */
607 if (unlikely(fsindex | next->fsindex | prev->fs)) {
608 loadsegment(fs, next->fsindex);
609 /* check if the user used a selector != 0
610 * if yes clear 64bit base, since overloaded base
611 * is always mapped to the Null selector
612 */
613 if (fsindex)
614 prev->fs = 0;
615 }
616 /* when next process has a 64bit base use it */
617 if (next->fs)
618 wrmsrl(MSR_FS_BASE, next->fs);
619 prev->fsindex = fsindex;
620 }
621 {
622 unsigned gsindex;
623 asm volatile("movl %%gs,%0" : "=r" (gsindex));
624 if (unlikely(gsindex | next->gsindex | prev->gs)) {
625 load_gs_index(next->gsindex);
626 if (gsindex)
627 prev->gs = 0;
628 }
629 if (next->gs)
630 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
631 prev->gsindex = gsindex;
632 }
633
Andi Kleen0a5ace22006-10-05 18:47:22 +0200634 /* Must be after DS reload */
635 unlazy_fpu(prev_p);
636
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 /*
Jan Beulich45948d72006-03-25 16:29:25 +0100638 * Switch the PDA and FPU contexts.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 */
640 prev->userrsp = read_pda(oldrsp);
641 write_pda(oldrsp, next->userrsp);
642 write_pda(pcurrent, next_p);
Andi Kleen18bd0572006-04-20 02:36:45 +0200643
Andi Kleena88cde12005-11-05 17:25:54 +0100644 write_pda(kernelstack,
Andi Kleen7b0bda72006-09-26 10:52:39 +0200645 (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
Arjan van de Ven0a425402006-09-26 10:52:38 +0200646#ifdef CONFIG_CC_STACKPROTECTOR
647 write_pda(stack_canary, next_p->stack_canary);
648 /*
649 * Build time only check to make sure the stack_canary is at
650 * offset 40 in the pda; this is a gcc ABI requirement
651 */
652 BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
653#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
655 /*
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200656 * Now maybe reload the debug registers and handle I/O bitmaps
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 */
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200658 if (unlikely((task_thread_info(next_p)->flags & _TIF_WORK_CTXSW))
659 || test_tsk_thread_flag(prev_p, TIF_IO_BITMAP))
660 __switch_to_xtra(prev_p, next_p, tss);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661
Arjan van de Vene07e23e2006-09-26 10:52:36 +0200662 /* If the task has used fpu the last 5 timeslices, just do a full
663 * restore of the math state immediately to avoid the trap; the
664 * chances of needing FPU soon are obviously high now
665 */
666 if (next_p->fpu_counter>5)
667 math_state_restore();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 return prev_p;
669}
670
671/*
672 * sys_execve() executes a new program.
673 */
674asmlinkage
675long sys_execve(char __user *name, char __user * __user *argv,
676 char __user * __user *envp, struct pt_regs regs)
677{
678 long error;
679 char * filename;
680
681 filename = getname(name);
682 error = PTR_ERR(filename);
683 if (IS_ERR(filename))
684 return error;
685 error = do_execve(filename, argv, envp, &regs);
686 if (error == 0) {
687 task_lock(current);
688 current->ptrace &= ~PT_DTRACE;
689 task_unlock(current);
690 }
691 putname(filename);
692 return error;
693}
694
695void set_personality_64bit(void)
696{
697 /* inherit personality from parent */
698
699 /* Make sure to be in 64bit mode */
700 clear_thread_flag(TIF_IA32);
701
702 /* TBD: overwrites user setup. Should have two bits.
703 But 64bit processes have always behaved this way,
704 so it's not too bad. The main problem is just that
705 32bit childs are affected again. */
706 current->personality &= ~READ_IMPLIES_EXEC;
707}
708
709asmlinkage long sys_fork(struct pt_regs *regs)
710{
711 return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
712}
713
Andi Kleena88cde12005-11-05 17:25:54 +0100714asmlinkage long
715sys_clone(unsigned long clone_flags, unsigned long newsp,
716 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717{
718 if (!newsp)
719 newsp = regs->rsp;
720 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
721}
722
723/*
724 * This is trivial, and on the face of it looks like it
725 * could equally well be done in user mode.
726 *
727 * Not so, for quite unobvious reasons - register pressure.
728 * In user mode vfork() cannot have a stack frame, and if
729 * done by calling the "clone()" system call directly, you
730 * do not have enough call-clobbered registers to hold all
731 * the information you need.
732 */
733asmlinkage long sys_vfork(struct pt_regs *regs)
734{
735 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
736 NULL, NULL);
737}
738
739unsigned long get_wchan(struct task_struct *p)
740{
741 unsigned long stack;
742 u64 fp,rip;
743 int count = 0;
744
745 if (!p || p == current || p->state==TASK_RUNNING)
746 return 0;
Al Viro57eafdc2006-01-12 01:05:39 -0800747 stack = (unsigned long)task_stack_page(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
749 return 0;
750 fp = *(u64 *)(p->thread.rsp);
751 do {
Andi Kleena88cde12005-11-05 17:25:54 +0100752 if (fp < (unsigned long)stack ||
753 fp > (unsigned long)stack+THREAD_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 return 0;
755 rip = *(u64 *)(fp+8);
756 if (!in_sched_functions(rip))
757 return rip;
758 fp = *(u64 *)fp;
759 } while (count++ < 16);
760 return 0;
761}
762
763long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
764{
765 int ret = 0;
766 int doit = task == current;
767 int cpu;
768
769 switch (code) {
770 case ARCH_SET_GS:
Suresh Siddha84929802005-06-21 17:14:32 -0700771 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 return -EPERM;
773 cpu = get_cpu();
774 /* handle small bases via the GDT because that's faster to
775 switch. */
776 if (addr <= 0xffffffff) {
777 set_32bit_tls(task, GS_TLS, addr);
778 if (doit) {
779 load_TLS(&task->thread, cpu);
780 load_gs_index(GS_TLS_SEL);
781 }
782 task->thread.gsindex = GS_TLS_SEL;
783 task->thread.gs = 0;
784 } else {
785 task->thread.gsindex = 0;
786 task->thread.gs = addr;
787 if (doit) {
Andi Kleena88cde12005-11-05 17:25:54 +0100788 load_gs_index(0);
789 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 }
791 }
792 put_cpu();
793 break;
794 case ARCH_SET_FS:
795 /* Not strictly needed for fs, but do it for symmetry
796 with gs */
Suresh Siddha84929802005-06-21 17:14:32 -0700797 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798 return -EPERM;
799 cpu = get_cpu();
800 /* handle small bases via the GDT because that's faster to
801 switch. */
802 if (addr <= 0xffffffff) {
803 set_32bit_tls(task, FS_TLS, addr);
804 if (doit) {
805 load_TLS(&task->thread, cpu);
Andi Kleena88cde12005-11-05 17:25:54 +0100806 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 }
808 task->thread.fsindex = FS_TLS_SEL;
809 task->thread.fs = 0;
810 } else {
811 task->thread.fsindex = 0;
812 task->thread.fs = addr;
813 if (doit) {
814 /* set the selector to 0 to not confuse
815 __switch_to */
Andi Kleena88cde12005-11-05 17:25:54 +0100816 asm volatile("movl %0,%%fs" :: "r" (0));
817 ret = checking_wrmsrl(MSR_FS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 }
819 }
820 put_cpu();
821 break;
822 case ARCH_GET_FS: {
823 unsigned long base;
824 if (task->thread.fsindex == FS_TLS_SEL)
825 base = read_32bit_tls(task, FS_TLS);
Andi Kleena88cde12005-11-05 17:25:54 +0100826 else if (doit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 rdmsrl(MSR_FS_BASE, base);
Andi Kleena88cde12005-11-05 17:25:54 +0100828 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 base = task->thread.fs;
830 ret = put_user(base, (unsigned long __user *)addr);
831 break;
832 }
833 case ARCH_GET_GS: {
834 unsigned long base;
John Blackwood97c28032006-04-07 19:50:25 +0200835 unsigned gsindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 if (task->thread.gsindex == GS_TLS_SEL)
837 base = read_32bit_tls(task, GS_TLS);
John Blackwood97c28032006-04-07 19:50:25 +0200838 else if (doit) {
839 asm("movl %%gs,%0" : "=r" (gsindex));
840 if (gsindex)
841 rdmsrl(MSR_KERNEL_GS_BASE, base);
842 else
843 base = task->thread.gs;
844 }
Andi Kleena88cde12005-11-05 17:25:54 +0100845 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 base = task->thread.gs;
847 ret = put_user(base, (unsigned long __user *)addr);
848 break;
849 }
850
851 default:
852 ret = -EINVAL;
853 break;
854 }
855
856 return ret;
857}
858
859long sys_arch_prctl(int code, unsigned long addr)
860{
861 return do_arch_prctl(current, code, addr);
862}
863
864/*
865 * Capture the user space registers if the task is not running (in user space)
866 */
867int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
868{
869 struct pt_regs *pp, ptregs;
870
Al Virobb049232006-01-12 01:05:38 -0800871 pp = task_pt_regs(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872
873 ptregs = *pp;
874 ptregs.cs &= 0xffff;
875 ptregs.ss &= 0xffff;
876
877 elf_core_copy_regs(regs, &ptregs);
878
879 return 1;
880}
881
882unsigned long arch_align_stack(unsigned long sp)
883{
Andi Kleenc16b63e2006-09-26 10:52:28 +0200884 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 sp -= get_random_int() % 8192;
886 return sp & ~0xf;
887}