blob: a4fa381db3c2de81b93d1f57708706d18d9702d3 [file] [log] [blame]
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001/*
Ingo Molnar57c0c152009-09-21 12:20:38 +02002 * Performance events core code:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 *
Ingo Molnar57c0c152009-09-21 12:20:38 +02009 * For licensing details see kernel-base/COPYING
Ingo Molnarcdd6c482009-09-21 12:02:48 +020010 */
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/file.h>
17#include <linux/poll.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090018#include <linux/slab.h>
Frederic Weisbecker76e1d902010-04-05 15:35:57 +020019#include <linux/hash.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020020#include <linux/sysfs.h>
21#include <linux/dcache.h>
22#include <linux/percpu.h>
23#include <linux/ptrace.h>
24#include <linux/vmstat.h>
Peter Zijlstra906010b2009-09-21 16:08:49 +020025#include <linux/vmalloc.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020026#include <linux/hardirq.h>
27#include <linux/rculist.h>
28#include <linux/uaccess.h>
29#include <linux/syscalls.h>
30#include <linux/anon_inodes.h>
31#include <linux/kernel_stat.h>
32#include <linux/perf_event.h>
Li Zefan6fb29152009-10-15 11:21:42 +080033#include <linux/ftrace_event.h>
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +020034#include <linux/hw_breakpoint.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020035
36#include <asm/irq_regs.h>
37
38/*
39 * Each CPU has a list of per CPU events:
40 */
Xiao Guangrongaa5452d2009-12-09 11:28:13 +080041static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +020042
43int perf_max_events __read_mostly = 1;
44static int perf_reserved_percpu __read_mostly;
45static int perf_overcommit __read_mostly = 1;
46
47static atomic_t nr_events __read_mostly;
48static atomic_t nr_mmap_events __read_mostly;
49static atomic_t nr_comm_events __read_mostly;
50static atomic_t nr_task_events __read_mostly;
51
52/*
53 * perf event paranoia level:
54 * -1 - not paranoid at all
55 * 0 - disallow raw tracepoint access for unpriv
56 * 1 - disallow cpu events for unpriv
57 * 2 - disallow kernel profiling for unpriv
58 */
59int sysctl_perf_event_paranoid __read_mostly = 1;
60
Ingo Molnarcdd6c482009-09-21 12:02:48 +020061int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
62
63/*
64 * max perf event sample rate
65 */
66int sysctl_perf_event_sample_rate __read_mostly = 100000;
67
68static atomic64_t perf_event_id;
69
70/*
71 * Lock for (sysadmin-configurable) event reservations:
72 */
73static DEFINE_SPINLOCK(perf_resource_lock);
74
75/*
76 * Architecture provided APIs - weak aliases:
77 */
78extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
79{
80 return NULL;
81}
82
83void __weak hw_perf_disable(void) { barrier(); }
84void __weak hw_perf_enable(void) { barrier(); }
85
Ingo Molnarcdd6c482009-09-21 12:02:48 +020086void __weak perf_event_print_debug(void) { }
87
88static DEFINE_PER_CPU(int, perf_disable_count);
89
Ingo Molnarcdd6c482009-09-21 12:02:48 +020090void perf_disable(void)
91{
Peter Zijlstra32975a42010-03-06 19:49:19 +010092 if (!__get_cpu_var(perf_disable_count)++)
93 hw_perf_disable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +020094}
95
96void perf_enable(void)
97{
Peter Zijlstra32975a42010-03-06 19:49:19 +010098 if (!--__get_cpu_var(perf_disable_count))
Ingo Molnarcdd6c482009-09-21 12:02:48 +020099 hw_perf_enable();
100}
101
102static void get_ctx(struct perf_event_context *ctx)
103{
104 WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
105}
106
107static void free_ctx(struct rcu_head *head)
108{
109 struct perf_event_context *ctx;
110
111 ctx = container_of(head, struct perf_event_context, rcu_head);
112 kfree(ctx);
113}
114
115static void put_ctx(struct perf_event_context *ctx)
116{
117 if (atomic_dec_and_test(&ctx->refcount)) {
118 if (ctx->parent_ctx)
119 put_ctx(ctx->parent_ctx);
120 if (ctx->task)
121 put_task_struct(ctx->task);
122 call_rcu(&ctx->rcu_head, free_ctx);
123 }
124}
125
126static void unclone_ctx(struct perf_event_context *ctx)
127{
128 if (ctx->parent_ctx) {
129 put_ctx(ctx->parent_ctx);
130 ctx->parent_ctx = NULL;
131 }
132}
133
134/*
135 * If we inherit events we want to return the parent event id
136 * to userspace.
137 */
138static u64 primary_event_id(struct perf_event *event)
139{
140 u64 id = event->id;
141
142 if (event->parent)
143 id = event->parent->id;
144
145 return id;
146}
147
148/*
149 * Get the perf_event_context for a task and lock it.
150 * This has to cope with with the fact that until it is locked,
151 * the context could get moved to another task.
152 */
153static struct perf_event_context *
154perf_lock_task_context(struct task_struct *task, unsigned long *flags)
155{
156 struct perf_event_context *ctx;
157
158 rcu_read_lock();
159 retry:
160 ctx = rcu_dereference(task->perf_event_ctxp);
161 if (ctx) {
162 /*
163 * If this context is a clone of another, it might
164 * get swapped for another underneath us by
165 * perf_event_task_sched_out, though the
166 * rcu_read_lock() protects us from any context
167 * getting freed. Lock the context and check if it
168 * got swapped before we could get the lock, and retry
169 * if so. If we locked the right context, then it
170 * can't get swapped on us any more.
171 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100172 raw_spin_lock_irqsave(&ctx->lock, *flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200173 if (ctx != rcu_dereference(task->perf_event_ctxp)) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100174 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200175 goto retry;
176 }
177
178 if (!atomic_inc_not_zero(&ctx->refcount)) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100179 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200180 ctx = NULL;
181 }
182 }
183 rcu_read_unlock();
184 return ctx;
185}
186
187/*
188 * Get the context for a task and increment its pin_count so it
189 * can't get swapped to another task. This also increments its
190 * reference count so that the context can't get freed.
191 */
192static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
193{
194 struct perf_event_context *ctx;
195 unsigned long flags;
196
197 ctx = perf_lock_task_context(task, &flags);
198 if (ctx) {
199 ++ctx->pin_count;
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100200 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200201 }
202 return ctx;
203}
204
205static void perf_unpin_context(struct perf_event_context *ctx)
206{
207 unsigned long flags;
208
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100209 raw_spin_lock_irqsave(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200210 --ctx->pin_count;
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100211 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200212 put_ctx(ctx);
213}
214
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100215static inline u64 perf_clock(void)
216{
Peter Zijlstra24691ea2010-02-26 16:36:23 +0100217 return cpu_clock(raw_smp_processor_id());
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100218}
219
220/*
221 * Update the record of the current time in a context.
222 */
223static void update_context_time(struct perf_event_context *ctx)
224{
225 u64 now = perf_clock();
226
227 ctx->time += now - ctx->timestamp;
228 ctx->timestamp = now;
229}
230
231/*
232 * Update the total_time_enabled and total_time_running fields for a event.
233 */
234static void update_event_times(struct perf_event *event)
235{
236 struct perf_event_context *ctx = event->ctx;
237 u64 run_end;
238
239 if (event->state < PERF_EVENT_STATE_INACTIVE ||
240 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
241 return;
242
Peter Zijlstraacd1d7c2009-11-23 15:00:36 +0100243 if (ctx->is_active)
244 run_end = ctx->time;
245 else
246 run_end = event->tstamp_stopped;
247
248 event->total_time_enabled = run_end - event->tstamp_enabled;
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100249
250 if (event->state == PERF_EVENT_STATE_INACTIVE)
251 run_end = event->tstamp_stopped;
252 else
253 run_end = ctx->time;
254
255 event->total_time_running = run_end - event->tstamp_running;
256}
257
Peter Zijlstra96c21a42010-05-11 16:19:10 +0200258/*
259 * Update total_time_enabled and total_time_running for all events in a group.
260 */
261static void update_group_times(struct perf_event *leader)
262{
263 struct perf_event *event;
264
265 update_event_times(leader);
266 list_for_each_entry(event, &leader->sibling_list, group_entry)
267 update_event_times(event);
268}
269
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100270static struct list_head *
271ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
272{
273 if (event->attr.pinned)
274 return &ctx->pinned_groups;
275 else
276 return &ctx->flexible_groups;
277}
278
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200279/*
280 * Add a event from the lists for its context.
281 * Must be called with ctx->mutex and ctx->lock held.
282 */
283static void
284list_add_event(struct perf_event *event, struct perf_event_context *ctx)
285{
286 struct perf_event *group_leader = event->group_leader;
287
288 /*
289 * Depending on whether it is a standalone or sibling event,
290 * add it straight to the context's event list, or to the group
291 * leader's sibling list:
292 */
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100293 if (group_leader == event) {
294 struct list_head *list;
295
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100296 if (is_software_event(event))
297 event->group_flags |= PERF_GROUP_SOFTWARE;
298
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100299 list = ctx_group_list(event, ctx);
300 list_add_tail(&event->group_entry, list);
301 } else {
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100302 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
303 !is_software_event(event))
304 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
305
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200306 list_add_tail(&event->group_entry, &group_leader->sibling_list);
307 group_leader->nr_siblings++;
308 }
309
310 list_add_rcu(&event->event_entry, &ctx->event_list);
311 ctx->nr_events++;
312 if (event->attr.inherit_stat)
313 ctx->nr_stat++;
314}
315
316/*
317 * Remove a event from the lists for its context.
318 * Must be called with ctx->mutex and ctx->lock held.
319 */
320static void
321list_del_event(struct perf_event *event, struct perf_event_context *ctx)
322{
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200323 if (list_empty(&event->group_entry))
324 return;
325 ctx->nr_events--;
326 if (event->attr.inherit_stat)
327 ctx->nr_stat--;
328
329 list_del_init(&event->group_entry);
330 list_del_rcu(&event->event_entry);
331
332 if (event->group_leader != event)
333 event->group_leader->nr_siblings--;
334
Peter Zijlstra96c21a42010-05-11 16:19:10 +0200335 update_group_times(event);
Stephane Eranianb2e74a22009-11-26 09:24:30 -0800336
337 /*
338 * If event was in error state, then keep it
339 * that way, otherwise bogus counts will be
340 * returned on read(). The only way to get out
341 * of error state is by explicit re-enabling
342 * of the event
343 */
344 if (event->state > PERF_EVENT_STATE_OFF)
345 event->state = PERF_EVENT_STATE_OFF;
Peter Zijlstra050735b2010-05-11 11:51:53 +0200346}
347
348static void
349perf_destroy_group(struct perf_event *event, struct perf_event_context *ctx)
350{
351 struct perf_event *sibling, *tmp;
Peter Zijlstra2e2af502009-11-23 11:37:25 +0100352
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200353 /*
354 * If this was a group event with sibling events then
355 * upgrade the siblings to singleton events by adding them
356 * to the context list directly:
357 */
358 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100359 struct list_head *list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200360
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100361 list = ctx_group_list(event, ctx);
362 list_move_tail(&sibling->group_entry, list);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200363 sibling->group_leader = sibling;
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100364
365 /* Inherit group flags from the previous leader */
366 sibling->group_flags = event->group_flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200367 }
368}
369
370static void
371event_sched_out(struct perf_event *event,
372 struct perf_cpu_context *cpuctx,
373 struct perf_event_context *ctx)
374{
375 if (event->state != PERF_EVENT_STATE_ACTIVE)
376 return;
377
378 event->state = PERF_EVENT_STATE_INACTIVE;
379 if (event->pending_disable) {
380 event->pending_disable = 0;
381 event->state = PERF_EVENT_STATE_OFF;
382 }
383 event->tstamp_stopped = ctx->time;
384 event->pmu->disable(event);
385 event->oncpu = -1;
386
387 if (!is_software_event(event))
388 cpuctx->active_oncpu--;
389 ctx->nr_active--;
390 if (event->attr.exclusive || !cpuctx->active_oncpu)
391 cpuctx->exclusive = 0;
392}
393
394static void
395group_sched_out(struct perf_event *group_event,
396 struct perf_cpu_context *cpuctx,
397 struct perf_event_context *ctx)
398{
399 struct perf_event *event;
400
401 if (group_event->state != PERF_EVENT_STATE_ACTIVE)
402 return;
403
404 event_sched_out(group_event, cpuctx, ctx);
405
406 /*
407 * Schedule out siblings (if any):
408 */
409 list_for_each_entry(event, &group_event->sibling_list, group_entry)
410 event_sched_out(event, cpuctx, ctx);
411
412 if (group_event->attr.exclusive)
413 cpuctx->exclusive = 0;
414}
415
416/*
417 * Cross CPU call to remove a performance event
418 *
419 * We disable the event on the hardware level first. After that we
420 * remove it from the context list.
421 */
422static void __perf_event_remove_from_context(void *info)
423{
424 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
425 struct perf_event *event = info;
426 struct perf_event_context *ctx = event->ctx;
427
428 /*
429 * If this is a task context, we need to check whether it is
430 * the current task context of this cpu. If not it has been
431 * scheduled out before the smp call arrived.
432 */
433 if (ctx->task && cpuctx->task_ctx != ctx)
434 return;
435
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100436 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200437 /*
438 * Protect the list operation against NMI by disabling the
439 * events on a global level.
440 */
441 perf_disable();
442
443 event_sched_out(event, cpuctx, ctx);
444
445 list_del_event(event, ctx);
446
447 if (!ctx->task) {
448 /*
449 * Allow more per task events with respect to the
450 * reservation:
451 */
452 cpuctx->max_pertask =
453 min(perf_max_events - ctx->nr_events,
454 perf_max_events - perf_reserved_percpu);
455 }
456
457 perf_enable();
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100458 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200459}
460
461
462/*
463 * Remove the event from a task's (or a CPU's) list of events.
464 *
465 * Must be called with ctx->mutex held.
466 *
467 * CPU events are removed with a smp call. For task events we only
468 * call when the task is on a CPU.
469 *
470 * If event->ctx is a cloned context, callers must make sure that
471 * every task struct that event->ctx->task could possibly point to
472 * remains valid. This is OK when called from perf_release since
473 * that only calls us on the top-level context, which can't be a clone.
474 * When called from perf_event_exit_task, it's OK because the
475 * context has been detached from its task.
476 */
477static void perf_event_remove_from_context(struct perf_event *event)
478{
479 struct perf_event_context *ctx = event->ctx;
480 struct task_struct *task = ctx->task;
481
482 if (!task) {
483 /*
484 * Per cpu events are removed via an smp call and
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200485 * the removal is always successful.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200486 */
487 smp_call_function_single(event->cpu,
488 __perf_event_remove_from_context,
489 event, 1);
490 return;
491 }
492
493retry:
494 task_oncpu_function_call(task, __perf_event_remove_from_context,
495 event);
496
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100497 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200498 /*
499 * If the context is active we need to retry the smp call.
500 */
501 if (ctx->nr_active && !list_empty(&event->group_entry)) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100502 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200503 goto retry;
504 }
505
506 /*
507 * The lock prevents that this context is scheduled in so we
508 * can remove the event safely, if the call above did not
509 * succeed.
510 */
Peter Zijlstra6c2bfcb2009-11-23 11:37:24 +0100511 if (!list_empty(&event->group_entry))
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200512 list_del_event(event, ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100513 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200514}
515
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200516/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200517 * Cross CPU call to disable a performance event
518 */
519static void __perf_event_disable(void *info)
520{
521 struct perf_event *event = info;
522 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
523 struct perf_event_context *ctx = event->ctx;
524
525 /*
526 * If this is a per-task event, need to check whether this
527 * event's task is the current task on this cpu.
528 */
529 if (ctx->task && cpuctx->task_ctx != ctx)
530 return;
531
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100532 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200533
534 /*
535 * If the event is on, turn it off.
536 * If it is in error state, leave it in error state.
537 */
538 if (event->state >= PERF_EVENT_STATE_INACTIVE) {
539 update_context_time(ctx);
540 update_group_times(event);
541 if (event == event->group_leader)
542 group_sched_out(event, cpuctx, ctx);
543 else
544 event_sched_out(event, cpuctx, ctx);
545 event->state = PERF_EVENT_STATE_OFF;
546 }
547
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100548 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200549}
550
551/*
552 * Disable a event.
553 *
554 * If event->ctx is a cloned context, callers must make sure that
555 * every task struct that event->ctx->task could possibly point to
556 * remains valid. This condition is satisifed when called through
557 * perf_event_for_each_child or perf_event_for_each because they
558 * hold the top-level event's child_mutex, so any descendant that
559 * goes to exit will block in sync_child_event.
560 * When called from perf_pending_event it's OK because event->ctx
561 * is the current context on this CPU and preemption is disabled,
562 * hence we can't get into perf_event_task_sched_out for this context.
563 */
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100564void perf_event_disable(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200565{
566 struct perf_event_context *ctx = event->ctx;
567 struct task_struct *task = ctx->task;
568
569 if (!task) {
570 /*
571 * Disable the event on the cpu that it's on
572 */
573 smp_call_function_single(event->cpu, __perf_event_disable,
574 event, 1);
575 return;
576 }
577
578 retry:
579 task_oncpu_function_call(task, __perf_event_disable, event);
580
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100581 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200582 /*
583 * If the event is still active, we need to retry the cross-call.
584 */
585 if (event->state == PERF_EVENT_STATE_ACTIVE) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100586 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200587 goto retry;
588 }
589
590 /*
591 * Since we have the lock this context can't be scheduled
592 * in, so we can change the state safely.
593 */
594 if (event->state == PERF_EVENT_STATE_INACTIVE) {
595 update_group_times(event);
596 event->state = PERF_EVENT_STATE_OFF;
597 }
598
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100599 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200600}
601
602static int
603event_sched_in(struct perf_event *event,
604 struct perf_cpu_context *cpuctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +0100605 struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200606{
607 if (event->state <= PERF_EVENT_STATE_OFF)
608 return 0;
609
610 event->state = PERF_EVENT_STATE_ACTIVE;
Peter Zijlstra6e377382010-02-11 13:21:58 +0100611 event->oncpu = smp_processor_id();
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200612 /*
613 * The new state must be visible before we turn it on in the hardware:
614 */
615 smp_wmb();
616
617 if (event->pmu->enable(event)) {
618 event->state = PERF_EVENT_STATE_INACTIVE;
619 event->oncpu = -1;
620 return -EAGAIN;
621 }
622
623 event->tstamp_running += ctx->time - event->tstamp_stopped;
624
625 if (!is_software_event(event))
626 cpuctx->active_oncpu++;
627 ctx->nr_active++;
628
629 if (event->attr.exclusive)
630 cpuctx->exclusive = 1;
631
632 return 0;
633}
634
635static int
636group_sched_in(struct perf_event *group_event,
637 struct perf_cpu_context *cpuctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +0100638 struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200639{
Lin Ming6bde9b62010-04-23 13:56:00 +0800640 struct perf_event *event, *partial_group = NULL;
641 const struct pmu *pmu = group_event->pmu;
642 bool txn = false;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200643 int ret;
644
645 if (group_event->state == PERF_EVENT_STATE_OFF)
646 return 0;
647
Lin Ming6bde9b62010-04-23 13:56:00 +0800648 /* Check if group transaction availabe */
649 if (pmu->start_txn)
650 txn = true;
651
652 if (txn)
653 pmu->start_txn(pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200654
Peter Zijlstra6e377382010-02-11 13:21:58 +0100655 if (event_sched_in(group_event, cpuctx, ctx))
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200656 return -EAGAIN;
657
658 /*
659 * Schedule in siblings as one group (if any):
660 */
661 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
Peter Zijlstra6e377382010-02-11 13:21:58 +0100662 if (event_sched_in(event, cpuctx, ctx)) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200663 partial_group = event;
664 goto group_error;
665 }
666 }
667
Paul Mackerras6e851582010-05-08 20:58:00 +1000668 if (!txn)
669 return 0;
Lin Ming6bde9b62010-04-23 13:56:00 +0800670
Paul Mackerras6e851582010-05-08 20:58:00 +1000671 ret = pmu->commit_txn(pmu);
672 if (!ret) {
673 pmu->cancel_txn(pmu);
674 return 0;
Lin Ming6bde9b62010-04-23 13:56:00 +0800675 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200676
677group_error:
Lin Ming6bde9b62010-04-23 13:56:00 +0800678 if (txn)
679 pmu->cancel_txn(pmu);
680
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200681 /*
682 * Groups can be scheduled in as one unit only, so undo any
683 * partial group before returning:
684 */
685 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
686 if (event == partial_group)
687 break;
688 event_sched_out(event, cpuctx, ctx);
689 }
690 event_sched_out(group_event, cpuctx, ctx);
691
692 return -EAGAIN;
693}
694
695/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200696 * Work out whether we can put this event group on the CPU now.
697 */
698static int group_can_go_on(struct perf_event *event,
699 struct perf_cpu_context *cpuctx,
700 int can_add_hw)
701{
702 /*
703 * Groups consisting entirely of software events can always go on.
704 */
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100705 if (event->group_flags & PERF_GROUP_SOFTWARE)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200706 return 1;
707 /*
708 * If an exclusive group is already on, no other hardware
709 * events can go on.
710 */
711 if (cpuctx->exclusive)
712 return 0;
713 /*
714 * If this group is exclusive and there are already
715 * events on the CPU, it can't go on.
716 */
717 if (event->attr.exclusive && cpuctx->active_oncpu)
718 return 0;
719 /*
720 * Otherwise, try to add it if all previous groups were able
721 * to go on.
722 */
723 return can_add_hw;
724}
725
726static void add_event_to_ctx(struct perf_event *event,
727 struct perf_event_context *ctx)
728{
729 list_add_event(event, ctx);
730 event->tstamp_enabled = ctx->time;
731 event->tstamp_running = ctx->time;
732 event->tstamp_stopped = ctx->time;
733}
734
735/*
736 * Cross CPU call to install and enable a performance event
737 *
738 * Must be called with ctx->mutex held
739 */
740static void __perf_install_in_context(void *info)
741{
742 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
743 struct perf_event *event = info;
744 struct perf_event_context *ctx = event->ctx;
745 struct perf_event *leader = event->group_leader;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200746 int err;
747
748 /*
749 * If this is a task context, we need to check whether it is
750 * the current task context of this cpu. If not it has been
751 * scheduled out before the smp call arrived.
752 * Or possibly this is the right context but it isn't
753 * on this cpu because it had no events.
754 */
755 if (ctx->task && cpuctx->task_ctx != ctx) {
756 if (cpuctx->task_ctx || ctx->task != current)
757 return;
758 cpuctx->task_ctx = ctx;
759 }
760
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100761 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200762 ctx->is_active = 1;
763 update_context_time(ctx);
764
765 /*
766 * Protect the list operation against NMI by disabling the
767 * events on a global level. NOP for non NMI based events.
768 */
769 perf_disable();
770
771 add_event_to_ctx(event, ctx);
772
Peter Zijlstraf4c41762009-12-16 17:55:54 +0100773 if (event->cpu != -1 && event->cpu != smp_processor_id())
774 goto unlock;
775
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200776 /*
777 * Don't put the event on if it is disabled or if
778 * it is in a group and the group isn't on.
779 */
780 if (event->state != PERF_EVENT_STATE_INACTIVE ||
781 (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
782 goto unlock;
783
784 /*
785 * An exclusive event can't go on if there are already active
786 * hardware events, and no hardware event can go on if there
787 * is already an exclusive event on.
788 */
789 if (!group_can_go_on(event, cpuctx, 1))
790 err = -EEXIST;
791 else
Peter Zijlstra6e377382010-02-11 13:21:58 +0100792 err = event_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200793
794 if (err) {
795 /*
796 * This event couldn't go on. If it is in a group
797 * then we have to pull the whole group off.
798 * If the event group is pinned then put it in error state.
799 */
800 if (leader != event)
801 group_sched_out(leader, cpuctx, ctx);
802 if (leader->attr.pinned) {
803 update_group_times(leader);
804 leader->state = PERF_EVENT_STATE_ERROR;
805 }
806 }
807
808 if (!err && !ctx->task && cpuctx->max_pertask)
809 cpuctx->max_pertask--;
810
811 unlock:
812 perf_enable();
813
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100814 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200815}
816
817/*
818 * Attach a performance event to a context
819 *
820 * First we add the event to the list with the hardware enable bit
821 * in event->hw_config cleared.
822 *
823 * If the event is attached to a task which is on a CPU we use a smp
824 * call to enable it in the task context. The task might have been
825 * scheduled away, but we check this in the smp call again.
826 *
827 * Must be called with ctx->mutex held.
828 */
829static void
830perf_install_in_context(struct perf_event_context *ctx,
831 struct perf_event *event,
832 int cpu)
833{
834 struct task_struct *task = ctx->task;
835
836 if (!task) {
837 /*
838 * Per cpu events are installed via an smp call and
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200839 * the install is always successful.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200840 */
841 smp_call_function_single(cpu, __perf_install_in_context,
842 event, 1);
843 return;
844 }
845
846retry:
847 task_oncpu_function_call(task, __perf_install_in_context,
848 event);
849
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100850 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200851 /*
852 * we need to retry the smp call.
853 */
854 if (ctx->is_active && list_empty(&event->group_entry)) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100855 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200856 goto retry;
857 }
858
859 /*
860 * The lock prevents that this context is scheduled in so we
861 * can add the event safely, if it the call above did not
862 * succeed.
863 */
864 if (list_empty(&event->group_entry))
865 add_event_to_ctx(event, ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100866 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200867}
868
869/*
870 * Put a event into inactive state and update time fields.
871 * Enabling the leader of a group effectively enables all
872 * the group members that aren't explicitly disabled, so we
873 * have to update their ->tstamp_enabled also.
874 * Note: this works for group members as well as group leaders
875 * since the non-leader members' sibling_lists will be empty.
876 */
877static void __perf_event_mark_enabled(struct perf_event *event,
878 struct perf_event_context *ctx)
879{
880 struct perf_event *sub;
881
882 event->state = PERF_EVENT_STATE_INACTIVE;
883 event->tstamp_enabled = ctx->time - event->total_time_enabled;
884 list_for_each_entry(sub, &event->sibling_list, group_entry)
885 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
886 sub->tstamp_enabled =
887 ctx->time - sub->total_time_enabled;
888}
889
890/*
891 * Cross CPU call to enable a performance event
892 */
893static void __perf_event_enable(void *info)
894{
895 struct perf_event *event = info;
896 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
897 struct perf_event_context *ctx = event->ctx;
898 struct perf_event *leader = event->group_leader;
899 int err;
900
901 /*
902 * If this is a per-task event, need to check whether this
903 * event's task is the current task on this cpu.
904 */
905 if (ctx->task && cpuctx->task_ctx != ctx) {
906 if (cpuctx->task_ctx || ctx->task != current)
907 return;
908 cpuctx->task_ctx = ctx;
909 }
910
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100911 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200912 ctx->is_active = 1;
913 update_context_time(ctx);
914
915 if (event->state >= PERF_EVENT_STATE_INACTIVE)
916 goto unlock;
917 __perf_event_mark_enabled(event, ctx);
918
Peter Zijlstraf4c41762009-12-16 17:55:54 +0100919 if (event->cpu != -1 && event->cpu != smp_processor_id())
920 goto unlock;
921
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200922 /*
923 * If the event is in a group and isn't the group leader,
924 * then don't put it on unless the group is on.
925 */
926 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
927 goto unlock;
928
929 if (!group_can_go_on(event, cpuctx, 1)) {
930 err = -EEXIST;
931 } else {
932 perf_disable();
933 if (event == leader)
Peter Zijlstra6e377382010-02-11 13:21:58 +0100934 err = group_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200935 else
Peter Zijlstra6e377382010-02-11 13:21:58 +0100936 err = event_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200937 perf_enable();
938 }
939
940 if (err) {
941 /*
942 * If this event can't go on and it's part of a
943 * group, then the whole group has to come off.
944 */
945 if (leader != event)
946 group_sched_out(leader, cpuctx, ctx);
947 if (leader->attr.pinned) {
948 update_group_times(leader);
949 leader->state = PERF_EVENT_STATE_ERROR;
950 }
951 }
952
953 unlock:
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100954 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200955}
956
957/*
958 * Enable a event.
959 *
960 * If event->ctx is a cloned context, callers must make sure that
961 * every task struct that event->ctx->task could possibly point to
962 * remains valid. This condition is satisfied when called through
963 * perf_event_for_each_child or perf_event_for_each as described
964 * for perf_event_disable.
965 */
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100966void perf_event_enable(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200967{
968 struct perf_event_context *ctx = event->ctx;
969 struct task_struct *task = ctx->task;
970
971 if (!task) {
972 /*
973 * Enable the event on the cpu that it's on
974 */
975 smp_call_function_single(event->cpu, __perf_event_enable,
976 event, 1);
977 return;
978 }
979
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100980 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200981 if (event->state >= PERF_EVENT_STATE_INACTIVE)
982 goto out;
983
984 /*
985 * If the event is in error state, clear that first.
986 * That way, if we see the event in error state below, we
987 * know that it has gone back into error state, as distinct
988 * from the task having been scheduled away before the
989 * cross-call arrived.
990 */
991 if (event->state == PERF_EVENT_STATE_ERROR)
992 event->state = PERF_EVENT_STATE_OFF;
993
994 retry:
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100995 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200996 task_oncpu_function_call(task, __perf_event_enable, event);
997
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100998 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200999
1000 /*
1001 * If the context is active and the event is still off,
1002 * we need to retry the cross-call.
1003 */
1004 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1005 goto retry;
1006
1007 /*
1008 * Since we have the lock this context can't be scheduled
1009 * in, so we can change the state safely.
1010 */
1011 if (event->state == PERF_EVENT_STATE_OFF)
1012 __perf_event_mark_enabled(event, ctx);
1013
1014 out:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001015 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001016}
1017
1018static int perf_event_refresh(struct perf_event *event, int refresh)
1019{
1020 /*
1021 * not supported on inherited events
1022 */
1023 if (event->attr.inherit)
1024 return -EINVAL;
1025
1026 atomic_add(refresh, &event->event_limit);
1027 perf_event_enable(event);
1028
1029 return 0;
1030}
1031
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001032enum event_type_t {
1033 EVENT_FLEXIBLE = 0x1,
1034 EVENT_PINNED = 0x2,
1035 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
1036};
1037
1038static void ctx_sched_out(struct perf_event_context *ctx,
1039 struct perf_cpu_context *cpuctx,
1040 enum event_type_t event_type)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001041{
1042 struct perf_event *event;
1043
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001044 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001045 ctx->is_active = 0;
1046 if (likely(!ctx->nr_events))
1047 goto out;
1048 update_context_time(ctx);
1049
1050 perf_disable();
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001051 if (!ctx->nr_active)
1052 goto out_enable;
1053
1054 if (event_type & EVENT_PINNED)
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001055 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1056 group_sched_out(event, cpuctx, ctx);
1057
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001058 if (event_type & EVENT_FLEXIBLE)
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001059 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
Xiao Guangrong8c9ed8e2009-09-25 13:51:17 +08001060 group_sched_out(event, cpuctx, ctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001061
1062 out_enable:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001063 perf_enable();
1064 out:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001065 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001066}
1067
1068/*
1069 * Test whether two contexts are equivalent, i.e. whether they
1070 * have both been cloned from the same version of the same context
1071 * and they both have the same number of enabled events.
1072 * If the number of enabled events is the same, then the set
1073 * of enabled events should be the same, because these are both
1074 * inherited contexts, therefore we can't access individual events
1075 * in them directly with an fd; we can only enable/disable all
1076 * events via prctl, or enable/disable all events in a family
1077 * via ioctl, which will have the same effect on both contexts.
1078 */
1079static int context_equiv(struct perf_event_context *ctx1,
1080 struct perf_event_context *ctx2)
1081{
1082 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1083 && ctx1->parent_gen == ctx2->parent_gen
1084 && !ctx1->pin_count && !ctx2->pin_count;
1085}
1086
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001087static void __perf_event_sync_stat(struct perf_event *event,
1088 struct perf_event *next_event)
1089{
1090 u64 value;
1091
1092 if (!event->attr.inherit_stat)
1093 return;
1094
1095 /*
1096 * Update the event value, we cannot use perf_event_read()
1097 * because we're in the middle of a context switch and have IRQs
1098 * disabled, which upsets smp_call_function_single(), however
1099 * we know the event must be on the current CPU, therefore we
1100 * don't need to use it.
1101 */
1102 switch (event->state) {
1103 case PERF_EVENT_STATE_ACTIVE:
Peter Zijlstra3dbebf12009-11-20 22:19:52 +01001104 event->pmu->read(event);
1105 /* fall-through */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001106
1107 case PERF_EVENT_STATE_INACTIVE:
1108 update_event_times(event);
1109 break;
1110
1111 default:
1112 break;
1113 }
1114
1115 /*
1116 * In order to keep per-task stats reliable we need to flip the event
1117 * values when we flip the contexts.
1118 */
1119 value = atomic64_read(&next_event->count);
1120 value = atomic64_xchg(&event->count, value);
1121 atomic64_set(&next_event->count, value);
1122
1123 swap(event->total_time_enabled, next_event->total_time_enabled);
1124 swap(event->total_time_running, next_event->total_time_running);
1125
1126 /*
1127 * Since we swizzled the values, update the user visible data too.
1128 */
1129 perf_event_update_userpage(event);
1130 perf_event_update_userpage(next_event);
1131}
1132
1133#define list_next_entry(pos, member) \
1134 list_entry(pos->member.next, typeof(*pos), member)
1135
1136static void perf_event_sync_stat(struct perf_event_context *ctx,
1137 struct perf_event_context *next_ctx)
1138{
1139 struct perf_event *event, *next_event;
1140
1141 if (!ctx->nr_stat)
1142 return;
1143
Peter Zijlstra02ffdbc2009-11-20 22:19:50 +01001144 update_context_time(ctx);
1145
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001146 event = list_first_entry(&ctx->event_list,
1147 struct perf_event, event_entry);
1148
1149 next_event = list_first_entry(&next_ctx->event_list,
1150 struct perf_event, event_entry);
1151
1152 while (&event->event_entry != &ctx->event_list &&
1153 &next_event->event_entry != &next_ctx->event_list) {
1154
1155 __perf_event_sync_stat(event, next_event);
1156
1157 event = list_next_entry(event, event_entry);
1158 next_event = list_next_entry(next_event, event_entry);
1159 }
1160}
1161
1162/*
1163 * Called from scheduler to remove the events of the current task,
1164 * with interrupts disabled.
1165 *
1166 * We stop each event and update the event value in event->count.
1167 *
1168 * This does not protect us against NMI, but disable()
1169 * sets the disabled bit in the control field of event _before_
1170 * accessing the event control register. If a NMI hits, then it will
1171 * not restart the event.
1172 */
1173void perf_event_task_sched_out(struct task_struct *task,
Peter Zijlstra49f47432009-12-27 11:51:52 +01001174 struct task_struct *next)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001175{
Peter Zijlstra49f47432009-12-27 11:51:52 +01001176 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001177 struct perf_event_context *ctx = task->perf_event_ctxp;
1178 struct perf_event_context *next_ctx;
1179 struct perf_event_context *parent;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001180 int do_switch = 1;
1181
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +01001182 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001183
1184 if (likely(!ctx || !cpuctx->task_ctx))
1185 return;
1186
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001187 rcu_read_lock();
1188 parent = rcu_dereference(ctx->parent_ctx);
1189 next_ctx = next->perf_event_ctxp;
1190 if (parent && next_ctx &&
1191 rcu_dereference(next_ctx->parent_ctx) == parent) {
1192 /*
1193 * Looks like the two contexts are clones, so we might be
1194 * able to optimize the context switch. We lock both
1195 * contexts and check that they are clones under the
1196 * lock (including re-checking that neither has been
1197 * uncloned in the meantime). It doesn't matter which
1198 * order we take the locks because no other cpu could
1199 * be trying to lock both of these tasks.
1200 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001201 raw_spin_lock(&ctx->lock);
1202 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001203 if (context_equiv(ctx, next_ctx)) {
1204 /*
1205 * XXX do we need a memory barrier of sorts
1206 * wrt to rcu_dereference() of perf_event_ctxp
1207 */
1208 task->perf_event_ctxp = next_ctx;
1209 next->perf_event_ctxp = ctx;
1210 ctx->task = next;
1211 next_ctx->task = task;
1212 do_switch = 0;
1213
1214 perf_event_sync_stat(ctx, next_ctx);
1215 }
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001216 raw_spin_unlock(&next_ctx->lock);
1217 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001218 }
1219 rcu_read_unlock();
1220
1221 if (do_switch) {
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001222 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001223 cpuctx->task_ctx = NULL;
1224 }
1225}
1226
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001227static void task_ctx_sched_out(struct perf_event_context *ctx,
1228 enum event_type_t event_type)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001229{
1230 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1231
1232 if (!cpuctx->task_ctx)
1233 return;
1234
1235 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1236 return;
1237
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001238 ctx_sched_out(ctx, cpuctx, event_type);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001239 cpuctx->task_ctx = NULL;
1240}
1241
1242/*
1243 * Called with IRQs disabled
1244 */
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001245static void __perf_event_task_sched_out(struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001246{
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001247 task_ctx_sched_out(ctx, EVENT_ALL);
1248}
1249
1250/*
1251 * Called with IRQs disabled
1252 */
1253static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1254 enum event_type_t event_type)
1255{
1256 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001257}
1258
1259static void
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001260ctx_pinned_sched_in(struct perf_event_context *ctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01001261 struct perf_cpu_context *cpuctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001262{
1263 struct perf_event *event;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001264
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001265 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1266 if (event->state <= PERF_EVENT_STATE_OFF)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001267 continue;
Peter Zijlstra6e377382010-02-11 13:21:58 +01001268 if (event->cpu != -1 && event->cpu != smp_processor_id())
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001269 continue;
1270
Xiao Guangrong8c9ed8e2009-09-25 13:51:17 +08001271 if (group_can_go_on(event, cpuctx, 1))
Peter Zijlstra6e377382010-02-11 13:21:58 +01001272 group_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001273
1274 /*
1275 * If this pinned group hasn't been scheduled,
1276 * put it in error state.
1277 */
1278 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1279 update_group_times(event);
1280 event->state = PERF_EVENT_STATE_ERROR;
1281 }
1282 }
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001283}
1284
1285static void
1286ctx_flexible_sched_in(struct perf_event_context *ctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01001287 struct perf_cpu_context *cpuctx)
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001288{
1289 struct perf_event *event;
1290 int can_add_hw = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001291
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001292 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1293 /* Ignore events in OFF or ERROR state */
1294 if (event->state <= PERF_EVENT_STATE_OFF)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001295 continue;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001296 /*
1297 * Listen to the 'cpu' scheduling filter constraint
1298 * of events:
1299 */
Peter Zijlstra6e377382010-02-11 13:21:58 +01001300 if (event->cpu != -1 && event->cpu != smp_processor_id())
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001301 continue;
1302
Xiao Guangrong8c9ed8e2009-09-25 13:51:17 +08001303 if (group_can_go_on(event, cpuctx, can_add_hw))
Peter Zijlstra6e377382010-02-11 13:21:58 +01001304 if (group_sched_in(event, cpuctx, ctx))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001305 can_add_hw = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001306 }
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001307}
1308
1309static void
1310ctx_sched_in(struct perf_event_context *ctx,
1311 struct perf_cpu_context *cpuctx,
1312 enum event_type_t event_type)
1313{
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001314 raw_spin_lock(&ctx->lock);
1315 ctx->is_active = 1;
1316 if (likely(!ctx->nr_events))
1317 goto out;
1318
1319 ctx->timestamp = perf_clock();
1320
1321 perf_disable();
1322
1323 /*
1324 * First go through the list and put on any pinned groups
1325 * in order to give them the best chance of going on.
1326 */
1327 if (event_type & EVENT_PINNED)
Peter Zijlstra6e377382010-02-11 13:21:58 +01001328 ctx_pinned_sched_in(ctx, cpuctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001329
1330 /* Then walk through the lower prio flexible groups */
1331 if (event_type & EVENT_FLEXIBLE)
Peter Zijlstra6e377382010-02-11 13:21:58 +01001332 ctx_flexible_sched_in(ctx, cpuctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001333
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001334 perf_enable();
1335 out:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001336 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001337}
1338
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01001339static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1340 enum event_type_t event_type)
1341{
1342 struct perf_event_context *ctx = &cpuctx->ctx;
1343
1344 ctx_sched_in(ctx, cpuctx, event_type);
1345}
1346
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001347static void task_ctx_sched_in(struct task_struct *task,
1348 enum event_type_t event_type)
1349{
1350 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1351 struct perf_event_context *ctx = task->perf_event_ctxp;
1352
1353 if (likely(!ctx))
1354 return;
1355 if (cpuctx->task_ctx == ctx)
1356 return;
1357 ctx_sched_in(ctx, cpuctx, event_type);
1358 cpuctx->task_ctx = ctx;
1359}
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001360/*
1361 * Called from scheduler to add the events of the current task
1362 * with interrupts disabled.
1363 *
1364 * We restore the event value and then enable it.
1365 *
1366 * This does not protect us against NMI, but enable()
1367 * sets the enabled bit in the control field of event _before_
1368 * accessing the event control register. If a NMI hits, then it will
1369 * keep the event running.
1370 */
Peter Zijlstra49f47432009-12-27 11:51:52 +01001371void perf_event_task_sched_in(struct task_struct *task)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001372{
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01001373 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1374 struct perf_event_context *ctx = task->perf_event_ctxp;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001375
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01001376 if (likely(!ctx))
1377 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001378
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01001379 if (cpuctx->task_ctx == ctx)
1380 return;
1381
eranian@google.com9b33fa62010-03-10 22:26:05 -08001382 perf_disable();
1383
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01001384 /*
1385 * We want to keep the following priority order:
1386 * cpu pinned (that don't need to move), task pinned,
1387 * cpu flexible, task flexible.
1388 */
1389 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1390
1391 ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
1392 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1393 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
1394
1395 cpuctx->task_ctx = ctx;
eranian@google.com9b33fa62010-03-10 22:26:05 -08001396
1397 perf_enable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001398}
1399
1400#define MAX_INTERRUPTS (~0ULL)
1401
1402static void perf_log_throttle(struct perf_event *event, int enable);
1403
Peter Zijlstraabd50712010-01-26 18:50:16 +01001404static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
1405{
1406 u64 frequency = event->attr.sample_freq;
1407 u64 sec = NSEC_PER_SEC;
1408 u64 divisor, dividend;
1409
1410 int count_fls, nsec_fls, frequency_fls, sec_fls;
1411
1412 count_fls = fls64(count);
1413 nsec_fls = fls64(nsec);
1414 frequency_fls = fls64(frequency);
1415 sec_fls = 30;
1416
1417 /*
1418 * We got @count in @nsec, with a target of sample_freq HZ
1419 * the target period becomes:
1420 *
1421 * @count * 10^9
1422 * period = -------------------
1423 * @nsec * sample_freq
1424 *
1425 */
1426
1427 /*
1428 * Reduce accuracy by one bit such that @a and @b converge
1429 * to a similar magnitude.
1430 */
1431#define REDUCE_FLS(a, b) \
1432do { \
1433 if (a##_fls > b##_fls) { \
1434 a >>= 1; \
1435 a##_fls--; \
1436 } else { \
1437 b >>= 1; \
1438 b##_fls--; \
1439 } \
1440} while (0)
1441
1442 /*
1443 * Reduce accuracy until either term fits in a u64, then proceed with
1444 * the other, so that finally we can do a u64/u64 division.
1445 */
1446 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
1447 REDUCE_FLS(nsec, frequency);
1448 REDUCE_FLS(sec, count);
1449 }
1450
1451 if (count_fls + sec_fls > 64) {
1452 divisor = nsec * frequency;
1453
1454 while (count_fls + sec_fls > 64) {
1455 REDUCE_FLS(count, sec);
1456 divisor >>= 1;
1457 }
1458
1459 dividend = count * sec;
1460 } else {
1461 dividend = count * sec;
1462
1463 while (nsec_fls + frequency_fls > 64) {
1464 REDUCE_FLS(nsec, frequency);
1465 dividend >>= 1;
1466 }
1467
1468 divisor = nsec * frequency;
1469 }
1470
1471 return div64_u64(dividend, divisor);
1472}
1473
Stephane Eraniand76a0812010-02-08 17:06:01 +02001474static void perf_event_stop(struct perf_event *event)
1475{
1476 if (!event->pmu->stop)
1477 return event->pmu->disable(event);
1478
1479 return event->pmu->stop(event);
1480}
1481
1482static int perf_event_start(struct perf_event *event)
1483{
1484 if (!event->pmu->start)
1485 return event->pmu->enable(event);
1486
1487 return event->pmu->start(event);
1488}
1489
Peter Zijlstraabd50712010-01-26 18:50:16 +01001490static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001491{
1492 struct hw_perf_event *hwc = &event->hw;
1493 u64 period, sample_period;
1494 s64 delta;
1495
Peter Zijlstraabd50712010-01-26 18:50:16 +01001496 period = perf_calculate_period(event, nsec, count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001497
1498 delta = (s64)(period - hwc->sample_period);
1499 delta = (delta + 7) / 8; /* low pass filter */
1500
1501 sample_period = hwc->sample_period + delta;
1502
1503 if (!sample_period)
1504 sample_period = 1;
1505
1506 hwc->sample_period = sample_period;
Peter Zijlstraabd50712010-01-26 18:50:16 +01001507
1508 if (atomic64_read(&hwc->period_left) > 8*sample_period) {
1509 perf_disable();
Stephane Eraniand76a0812010-02-08 17:06:01 +02001510 perf_event_stop(event);
Peter Zijlstraabd50712010-01-26 18:50:16 +01001511 atomic64_set(&hwc->period_left, 0);
Stephane Eraniand76a0812010-02-08 17:06:01 +02001512 perf_event_start(event);
Peter Zijlstraabd50712010-01-26 18:50:16 +01001513 perf_enable();
1514 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001515}
1516
1517static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1518{
1519 struct perf_event *event;
1520 struct hw_perf_event *hwc;
Peter Zijlstraabd50712010-01-26 18:50:16 +01001521 u64 interrupts, now;
1522 s64 delta;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001523
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001524 raw_spin_lock(&ctx->lock);
Paul Mackerras03541f82009-10-14 16:58:03 +11001525 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001526 if (event->state != PERF_EVENT_STATE_ACTIVE)
1527 continue;
1528
Peter Zijlstra5d27c232009-12-17 13:16:32 +01001529 if (event->cpu != -1 && event->cpu != smp_processor_id())
1530 continue;
1531
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001532 hwc = &event->hw;
1533
1534 interrupts = hwc->interrupts;
1535 hwc->interrupts = 0;
1536
1537 /*
1538 * unthrottle events on the tick
1539 */
1540 if (interrupts == MAX_INTERRUPTS) {
1541 perf_log_throttle(event, 1);
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001542 perf_disable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001543 event->pmu->unthrottle(event);
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001544 perf_enable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001545 }
1546
1547 if (!event->attr.freq || !event->attr.sample_freq)
1548 continue;
1549
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001550 perf_disable();
Peter Zijlstraabd50712010-01-26 18:50:16 +01001551 event->pmu->read(event);
1552 now = atomic64_read(&event->count);
1553 delta = now - hwc->freq_count_stamp;
1554 hwc->freq_count_stamp = now;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001555
Peter Zijlstraabd50712010-01-26 18:50:16 +01001556 if (delta > 0)
1557 perf_adjust_period(event, TICK_NSEC, delta);
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001558 perf_enable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001559 }
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001560 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001561}
1562
1563/*
1564 * Round-robin a context's events:
1565 */
1566static void rotate_ctx(struct perf_event_context *ctx)
1567{
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001568 raw_spin_lock(&ctx->lock);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001569
Frederic Weisbeckere2864172010-01-09 21:05:28 +01001570 /* Rotate the first entry last of non-pinned groups */
Frederic Weisbeckere2864172010-01-09 21:05:28 +01001571 list_rotate_left(&ctx->flexible_groups);
1572
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001573 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001574}
1575
Peter Zijlstra49f47432009-12-27 11:51:52 +01001576void perf_event_task_tick(struct task_struct *curr)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001577{
1578 struct perf_cpu_context *cpuctx;
1579 struct perf_event_context *ctx;
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001580 int rotate = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001581
1582 if (!atomic_read(&nr_events))
1583 return;
1584
Peter Zijlstra49f47432009-12-27 11:51:52 +01001585 cpuctx = &__get_cpu_var(perf_cpu_context);
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001586 if (cpuctx->ctx.nr_events &&
1587 cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
1588 rotate = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001589
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001590 ctx = curr->perf_event_ctxp;
1591 if (ctx && ctx->nr_events && ctx->nr_events != ctx->nr_active)
1592 rotate = 1;
Peter Zijlstra9717e6c2010-01-28 13:57:44 +01001593
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001594 perf_ctx_adjust_freq(&cpuctx->ctx);
1595 if (ctx)
1596 perf_ctx_adjust_freq(ctx);
1597
Peter Zijlstrad4944a02010-03-08 13:51:20 +01001598 if (!rotate)
1599 return;
1600
1601 perf_disable();
Frederic Weisbecker7defb0f2010-01-17 12:15:31 +01001602 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001603 if (ctx)
Frederic Weisbecker7defb0f2010-01-17 12:15:31 +01001604 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001605
1606 rotate_ctx(&cpuctx->ctx);
1607 if (ctx)
1608 rotate_ctx(ctx);
1609
Frederic Weisbecker7defb0f2010-01-17 12:15:31 +01001610 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001611 if (ctx)
Frederic Weisbecker7defb0f2010-01-17 12:15:31 +01001612 task_ctx_sched_in(curr, EVENT_FLEXIBLE);
Peter Zijlstra9717e6c2010-01-28 13:57:44 +01001613 perf_enable();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001614}
1615
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001616static int event_enable_on_exec(struct perf_event *event,
1617 struct perf_event_context *ctx)
1618{
1619 if (!event->attr.enable_on_exec)
1620 return 0;
1621
1622 event->attr.enable_on_exec = 0;
1623 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1624 return 0;
1625
1626 __perf_event_mark_enabled(event, ctx);
1627
1628 return 1;
1629}
1630
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001631/*
1632 * Enable all of a task's events that have been marked enable-on-exec.
1633 * This expects task == current.
1634 */
1635static void perf_event_enable_on_exec(struct task_struct *task)
1636{
1637 struct perf_event_context *ctx;
1638 struct perf_event *event;
1639 unsigned long flags;
1640 int enabled = 0;
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001641 int ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001642
1643 local_irq_save(flags);
1644 ctx = task->perf_event_ctxp;
1645 if (!ctx || !ctx->nr_events)
1646 goto out;
1647
1648 __perf_event_task_sched_out(ctx);
1649
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001650 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001651
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001652 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1653 ret = event_enable_on_exec(event, ctx);
1654 if (ret)
1655 enabled = 1;
1656 }
1657
1658 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1659 ret = event_enable_on_exec(event, ctx);
1660 if (ret)
1661 enabled = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001662 }
1663
1664 /*
1665 * Unclone this context if we enabled any event.
1666 */
1667 if (enabled)
1668 unclone_ctx(ctx);
1669
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001670 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001671
Peter Zijlstra49f47432009-12-27 11:51:52 +01001672 perf_event_task_sched_in(task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001673 out:
1674 local_irq_restore(flags);
1675}
1676
1677/*
1678 * Cross CPU call to read the hardware event
1679 */
1680static void __perf_event_read(void *info)
1681{
1682 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1683 struct perf_event *event = info;
1684 struct perf_event_context *ctx = event->ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001685
1686 /*
1687 * If this is a task context, we need to check whether it is
1688 * the current task context of this cpu. If not it has been
1689 * scheduled out before the smp call arrived. In that case
1690 * event->count would have been updated to a recent sample
1691 * when the event was scheduled out.
1692 */
1693 if (ctx->task && cpuctx->task_ctx != ctx)
1694 return;
1695
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001696 raw_spin_lock(&ctx->lock);
Peter Zijlstra58e5ad12009-11-20 22:19:53 +01001697 update_context_time(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001698 update_event_times(event);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001699 raw_spin_unlock(&ctx->lock);
Peter Zijlstra2b8988c2009-11-20 22:19:54 +01001700
Peter Zijlstra58e5ad12009-11-20 22:19:53 +01001701 event->pmu->read(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001702}
1703
1704static u64 perf_event_read(struct perf_event *event)
1705{
1706 /*
1707 * If event is enabled and currently active on a CPU, update the
1708 * value in the event structure:
1709 */
1710 if (event->state == PERF_EVENT_STATE_ACTIVE) {
1711 smp_call_function_single(event->oncpu,
1712 __perf_event_read, event, 1);
1713 } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
Peter Zijlstra2b8988c2009-11-20 22:19:54 +01001714 struct perf_event_context *ctx = event->ctx;
1715 unsigned long flags;
1716
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001717 raw_spin_lock_irqsave(&ctx->lock, flags);
Peter Zijlstra2b8988c2009-11-20 22:19:54 +01001718 update_context_time(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001719 update_event_times(event);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001720 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001721 }
1722
1723 return atomic64_read(&event->count);
1724}
1725
1726/*
1727 * Initialize the perf_event context in a task_struct:
1728 */
1729static void
1730__perf_event_init_context(struct perf_event_context *ctx,
1731 struct task_struct *task)
1732{
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001733 raw_spin_lock_init(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001734 mutex_init(&ctx->mutex);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001735 INIT_LIST_HEAD(&ctx->pinned_groups);
1736 INIT_LIST_HEAD(&ctx->flexible_groups);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001737 INIT_LIST_HEAD(&ctx->event_list);
1738 atomic_set(&ctx->refcount, 1);
1739 ctx->task = task;
1740}
1741
1742static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1743{
1744 struct perf_event_context *ctx;
1745 struct perf_cpu_context *cpuctx;
1746 struct task_struct *task;
1747 unsigned long flags;
1748 int err;
1749
Peter Zijlstraf4c41762009-12-16 17:55:54 +01001750 if (pid == -1 && cpu != -1) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001751 /* Must be root to operate on a CPU event: */
1752 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1753 return ERR_PTR(-EACCES);
1754
Paul Mackerras0f624e72009-12-15 19:40:32 +11001755 if (cpu < 0 || cpu >= nr_cpumask_bits)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001756 return ERR_PTR(-EINVAL);
1757
1758 /*
1759 * We could be clever and allow to attach a event to an
1760 * offline CPU and activate it when the CPU comes up, but
1761 * that's for later.
1762 */
Rusty Russellf6325e32009-12-17 11:43:08 -06001763 if (!cpu_online(cpu))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001764 return ERR_PTR(-ENODEV);
1765
1766 cpuctx = &per_cpu(perf_cpu_context, cpu);
1767 ctx = &cpuctx->ctx;
1768 get_ctx(ctx);
1769
1770 return ctx;
1771 }
1772
1773 rcu_read_lock();
1774 if (!pid)
1775 task = current;
1776 else
1777 task = find_task_by_vpid(pid);
1778 if (task)
1779 get_task_struct(task);
1780 rcu_read_unlock();
1781
1782 if (!task)
1783 return ERR_PTR(-ESRCH);
1784
1785 /*
1786 * Can't attach events to a dying task.
1787 */
1788 err = -ESRCH;
1789 if (task->flags & PF_EXITING)
1790 goto errout;
1791
1792 /* Reuse ptrace permission checks for now. */
1793 err = -EACCES;
1794 if (!ptrace_may_access(task, PTRACE_MODE_READ))
1795 goto errout;
1796
1797 retry:
1798 ctx = perf_lock_task_context(task, &flags);
1799 if (ctx) {
1800 unclone_ctx(ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001801 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001802 }
1803
1804 if (!ctx) {
Xiao Guangrongaa5452d2009-12-09 11:28:13 +08001805 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001806 err = -ENOMEM;
1807 if (!ctx)
1808 goto errout;
1809 __perf_event_init_context(ctx, task);
1810 get_ctx(ctx);
1811 if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1812 /*
1813 * We raced with some other task; use
1814 * the context they set.
1815 */
1816 kfree(ctx);
1817 goto retry;
1818 }
1819 get_task_struct(task);
1820 }
1821
1822 put_task_struct(task);
1823 return ctx;
1824
1825 errout:
1826 put_task_struct(task);
1827 return ERR_PTR(err);
1828}
1829
Li Zefan6fb29152009-10-15 11:21:42 +08001830static void perf_event_free_filter(struct perf_event *event);
1831
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001832static void free_event_rcu(struct rcu_head *head)
1833{
1834 struct perf_event *event;
1835
1836 event = container_of(head, struct perf_event, rcu_head);
1837 if (event->ns)
1838 put_pid_ns(event->ns);
Li Zefan6fb29152009-10-15 11:21:42 +08001839 perf_event_free_filter(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001840 kfree(event);
1841}
1842
1843static void perf_pending_sync(struct perf_event *event);
1844
1845static void free_event(struct perf_event *event)
1846{
1847 perf_pending_sync(event);
1848
1849 if (!event->parent) {
1850 atomic_dec(&nr_events);
1851 if (event->attr.mmap)
1852 atomic_dec(&nr_mmap_events);
1853 if (event->attr.comm)
1854 atomic_dec(&nr_comm_events);
1855 if (event->attr.task)
1856 atomic_dec(&nr_task_events);
1857 }
1858
1859 if (event->output) {
1860 fput(event->output->filp);
1861 event->output = NULL;
1862 }
1863
1864 if (event->destroy)
1865 event->destroy(event);
1866
1867 put_ctx(event->ctx);
1868 call_rcu(&event->rcu_head, free_event_rcu);
1869}
1870
Arjan van de Venfb0459d2009-09-25 12:25:56 +02001871int perf_event_release_kernel(struct perf_event *event)
1872{
1873 struct perf_event_context *ctx = event->ctx;
1874
Peter Zijlstra050735b2010-05-11 11:51:53 +02001875 /*
1876 * Remove from the PMU, can't get re-enabled since we got
1877 * here because the last ref went.
1878 */
1879 perf_event_disable(event);
1880
Arjan van de Venfb0459d2009-09-25 12:25:56 +02001881 WARN_ON_ONCE(ctx->parent_ctx);
Peter Zijlstraa0507c82010-05-06 15:42:53 +02001882 /*
1883 * There are two ways this annotation is useful:
1884 *
1885 * 1) there is a lock recursion from perf_event_exit_task
1886 * see the comment there.
1887 *
1888 * 2) there is a lock-inversion with mmap_sem through
1889 * perf_event_read_group(), which takes faults while
1890 * holding ctx->mutex, however this is called after
1891 * the last filedesc died, so there is no possibility
1892 * to trigger the AB-BA case.
1893 */
1894 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
Peter Zijlstra050735b2010-05-11 11:51:53 +02001895 raw_spin_lock_irq(&ctx->lock);
1896 list_del_event(event, ctx);
1897 perf_destroy_group(event, ctx);
1898 raw_spin_unlock_irq(&ctx->lock);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02001899 mutex_unlock(&ctx->mutex);
1900
1901 mutex_lock(&event->owner->perf_event_mutex);
1902 list_del_init(&event->owner_entry);
1903 mutex_unlock(&event->owner->perf_event_mutex);
1904 put_task_struct(event->owner);
1905
1906 free_event(event);
1907
1908 return 0;
1909}
1910EXPORT_SYMBOL_GPL(perf_event_release_kernel);
1911
Peter Zijlstraa66a3052009-11-23 11:37:23 +01001912/*
1913 * Called when the last reference to the file is gone.
1914 */
1915static int perf_release(struct inode *inode, struct file *file)
1916{
1917 struct perf_event *event = file->private_data;
1918
1919 file->private_data = NULL;
1920
1921 return perf_event_release_kernel(event);
1922}
1923
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001924static int perf_event_read_size(struct perf_event *event)
1925{
1926 int entry = sizeof(u64); /* value */
1927 int size = 0;
1928 int nr = 1;
1929
1930 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1931 size += sizeof(u64);
1932
1933 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1934 size += sizeof(u64);
1935
1936 if (event->attr.read_format & PERF_FORMAT_ID)
1937 entry += sizeof(u64);
1938
1939 if (event->attr.read_format & PERF_FORMAT_GROUP) {
1940 nr += event->group_leader->nr_siblings;
1941 size += sizeof(u64);
1942 }
1943
1944 size += entry * nr;
1945
1946 return size;
1947}
1948
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001949u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001950{
1951 struct perf_event *child;
1952 u64 total = 0;
1953
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001954 *enabled = 0;
1955 *running = 0;
1956
Peter Zijlstra6f105812009-11-20 22:19:56 +01001957 mutex_lock(&event->child_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001958 total += perf_event_read(event);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001959 *enabled += event->total_time_enabled +
1960 atomic64_read(&event->child_total_time_enabled);
1961 *running += event->total_time_running +
1962 atomic64_read(&event->child_total_time_running);
1963
1964 list_for_each_entry(child, &event->child_list, child_list) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001965 total += perf_event_read(child);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001966 *enabled += child->total_time_enabled;
1967 *running += child->total_time_running;
1968 }
Peter Zijlstra6f105812009-11-20 22:19:56 +01001969 mutex_unlock(&event->child_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001970
1971 return total;
1972}
Arjan van de Venfb0459d2009-09-25 12:25:56 +02001973EXPORT_SYMBOL_GPL(perf_event_read_value);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001974
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001975static int perf_event_read_group(struct perf_event *event,
1976 u64 read_format, char __user *buf)
1977{
1978 struct perf_event *leader = event->group_leader, *sub;
Peter Zijlstra6f105812009-11-20 22:19:56 +01001979 int n = 0, size = 0, ret = -EFAULT;
1980 struct perf_event_context *ctx = leader->ctx;
Peter Zijlstraabf48682009-11-20 22:19:49 +01001981 u64 values[5];
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001982 u64 count, enabled, running;
Peter Zijlstraabf48682009-11-20 22:19:49 +01001983
Peter Zijlstra6f105812009-11-20 22:19:56 +01001984 mutex_lock(&ctx->mutex);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001985 count = perf_event_read_value(leader, &enabled, &running);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001986
1987 values[n++] = 1 + leader->nr_siblings;
Peter Zijlstra59ed4462009-11-20 22:19:55 +01001988 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1989 values[n++] = enabled;
1990 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1991 values[n++] = running;
Peter Zijlstraabf48682009-11-20 22:19:49 +01001992 values[n++] = count;
1993 if (read_format & PERF_FORMAT_ID)
1994 values[n++] = primary_event_id(leader);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001995
1996 size = n * sizeof(u64);
1997
1998 if (copy_to_user(buf, values, size))
Peter Zijlstra6f105812009-11-20 22:19:56 +01001999 goto unlock;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002000
Peter Zijlstra6f105812009-11-20 22:19:56 +01002001 ret = size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002002
2003 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
Peter Zijlstraabf48682009-11-20 22:19:49 +01002004 n = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002005
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002006 values[n++] = perf_event_read_value(sub, &enabled, &running);
Peter Zijlstraabf48682009-11-20 22:19:49 +01002007 if (read_format & PERF_FORMAT_ID)
2008 values[n++] = primary_event_id(sub);
2009
2010 size = n * sizeof(u64);
2011
Stephane Eranian184d3da2009-11-23 21:40:49 -08002012 if (copy_to_user(buf + ret, values, size)) {
Peter Zijlstra6f105812009-11-20 22:19:56 +01002013 ret = -EFAULT;
2014 goto unlock;
2015 }
Peter Zijlstraabf48682009-11-20 22:19:49 +01002016
2017 ret += size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002018 }
Peter Zijlstra6f105812009-11-20 22:19:56 +01002019unlock:
2020 mutex_unlock(&ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002021
Peter Zijlstraabf48682009-11-20 22:19:49 +01002022 return ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002023}
2024
2025static int perf_event_read_one(struct perf_event *event,
2026 u64 read_format, char __user *buf)
2027{
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002028 u64 enabled, running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002029 u64 values[4];
2030 int n = 0;
2031
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002032 values[n++] = perf_event_read_value(event, &enabled, &running);
2033 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2034 values[n++] = enabled;
2035 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2036 values[n++] = running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002037 if (read_format & PERF_FORMAT_ID)
2038 values[n++] = primary_event_id(event);
2039
2040 if (copy_to_user(buf, values, n * sizeof(u64)))
2041 return -EFAULT;
2042
2043 return n * sizeof(u64);
2044}
2045
2046/*
2047 * Read the performance event - simple non blocking version for now
2048 */
2049static ssize_t
2050perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2051{
2052 u64 read_format = event->attr.read_format;
2053 int ret;
2054
2055 /*
2056 * Return end-of-file for a read on a event that is in
2057 * error state (i.e. because it was pinned but it couldn't be
2058 * scheduled on to the CPU at some point).
2059 */
2060 if (event->state == PERF_EVENT_STATE_ERROR)
2061 return 0;
2062
2063 if (count < perf_event_read_size(event))
2064 return -ENOSPC;
2065
2066 WARN_ON_ONCE(event->ctx->parent_ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002067 if (read_format & PERF_FORMAT_GROUP)
2068 ret = perf_event_read_group(event, read_format, buf);
2069 else
2070 ret = perf_event_read_one(event, read_format, buf);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002071
2072 return ret;
2073}
2074
2075static ssize_t
2076perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
2077{
2078 struct perf_event *event = file->private_data;
2079
2080 return perf_read_hw(event, buf, count);
2081}
2082
2083static unsigned int perf_poll(struct file *file, poll_table *wait)
2084{
2085 struct perf_event *event = file->private_data;
2086 struct perf_mmap_data *data;
2087 unsigned int events = POLL_HUP;
2088
2089 rcu_read_lock();
2090 data = rcu_dereference(event->data);
2091 if (data)
2092 events = atomic_xchg(&data->poll, 0);
2093 rcu_read_unlock();
2094
2095 poll_wait(file, &event->waitq, wait);
2096
2097 return events;
2098}
2099
2100static void perf_event_reset(struct perf_event *event)
2101{
2102 (void)perf_event_read(event);
2103 atomic64_set(&event->count, 0);
2104 perf_event_update_userpage(event);
2105}
2106
2107/*
2108 * Holding the top-level event's child_mutex means that any
2109 * descendant process that has inherited this event will block
2110 * in sync_child_event if it goes to exit, thus satisfying the
2111 * task existence requirements of perf_event_enable/disable.
2112 */
2113static void perf_event_for_each_child(struct perf_event *event,
2114 void (*func)(struct perf_event *))
2115{
2116 struct perf_event *child;
2117
2118 WARN_ON_ONCE(event->ctx->parent_ctx);
2119 mutex_lock(&event->child_mutex);
2120 func(event);
2121 list_for_each_entry(child, &event->child_list, child_list)
2122 func(child);
2123 mutex_unlock(&event->child_mutex);
2124}
2125
2126static void perf_event_for_each(struct perf_event *event,
2127 void (*func)(struct perf_event *))
2128{
2129 struct perf_event_context *ctx = event->ctx;
2130 struct perf_event *sibling;
2131
2132 WARN_ON_ONCE(ctx->parent_ctx);
2133 mutex_lock(&ctx->mutex);
2134 event = event->group_leader;
2135
2136 perf_event_for_each_child(event, func);
2137 func(event);
2138 list_for_each_entry(sibling, &event->sibling_list, group_entry)
2139 perf_event_for_each_child(event, func);
2140 mutex_unlock(&ctx->mutex);
2141}
2142
2143static int perf_event_period(struct perf_event *event, u64 __user *arg)
2144{
2145 struct perf_event_context *ctx = event->ctx;
2146 unsigned long size;
2147 int ret = 0;
2148 u64 value;
2149
2150 if (!event->attr.sample_period)
2151 return -EINVAL;
2152
2153 size = copy_from_user(&value, arg, sizeof(value));
2154 if (size != sizeof(value))
2155 return -EFAULT;
2156
2157 if (!value)
2158 return -EINVAL;
2159
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002160 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002161 if (event->attr.freq) {
2162 if (value > sysctl_perf_event_sample_rate) {
2163 ret = -EINVAL;
2164 goto unlock;
2165 }
2166
2167 event->attr.sample_freq = value;
2168 } else {
2169 event->attr.sample_period = value;
2170 event->hw.sample_period = value;
2171 }
2172unlock:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002173 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002174
2175 return ret;
2176}
2177
Li Zefan6fb29152009-10-15 11:21:42 +08002178static int perf_event_set_output(struct perf_event *event, int output_fd);
2179static int perf_event_set_filter(struct perf_event *event, void __user *arg);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002180
2181static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2182{
2183 struct perf_event *event = file->private_data;
2184 void (*func)(struct perf_event *);
2185 u32 flags = arg;
2186
2187 switch (cmd) {
2188 case PERF_EVENT_IOC_ENABLE:
2189 func = perf_event_enable;
2190 break;
2191 case PERF_EVENT_IOC_DISABLE:
2192 func = perf_event_disable;
2193 break;
2194 case PERF_EVENT_IOC_RESET:
2195 func = perf_event_reset;
2196 break;
2197
2198 case PERF_EVENT_IOC_REFRESH:
2199 return perf_event_refresh(event, arg);
2200
2201 case PERF_EVENT_IOC_PERIOD:
2202 return perf_event_period(event, (u64 __user *)arg);
2203
2204 case PERF_EVENT_IOC_SET_OUTPUT:
2205 return perf_event_set_output(event, arg);
2206
Li Zefan6fb29152009-10-15 11:21:42 +08002207 case PERF_EVENT_IOC_SET_FILTER:
2208 return perf_event_set_filter(event, (void __user *)arg);
2209
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002210 default:
2211 return -ENOTTY;
2212 }
2213
2214 if (flags & PERF_IOC_FLAG_GROUP)
2215 perf_event_for_each(event, func);
2216 else
2217 perf_event_for_each_child(event, func);
2218
2219 return 0;
2220}
2221
2222int perf_event_task_enable(void)
2223{
2224 struct perf_event *event;
2225
2226 mutex_lock(&current->perf_event_mutex);
2227 list_for_each_entry(event, &current->perf_event_list, owner_entry)
2228 perf_event_for_each_child(event, perf_event_enable);
2229 mutex_unlock(&current->perf_event_mutex);
2230
2231 return 0;
2232}
2233
2234int perf_event_task_disable(void)
2235{
2236 struct perf_event *event;
2237
2238 mutex_lock(&current->perf_event_mutex);
2239 list_for_each_entry(event, &current->perf_event_list, owner_entry)
2240 perf_event_for_each_child(event, perf_event_disable);
2241 mutex_unlock(&current->perf_event_mutex);
2242
2243 return 0;
2244}
2245
2246#ifndef PERF_EVENT_INDEX_OFFSET
2247# define PERF_EVENT_INDEX_OFFSET 0
2248#endif
2249
2250static int perf_event_index(struct perf_event *event)
2251{
2252 if (event->state != PERF_EVENT_STATE_ACTIVE)
2253 return 0;
2254
2255 return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2256}
2257
2258/*
2259 * Callers need to ensure there can be no nesting of this function, otherwise
2260 * the seqlock logic goes bad. We can not serialize this because the arch
2261 * code calls this from NMI context.
2262 */
2263void perf_event_update_userpage(struct perf_event *event)
2264{
2265 struct perf_event_mmap_page *userpg;
2266 struct perf_mmap_data *data;
2267
2268 rcu_read_lock();
2269 data = rcu_dereference(event->data);
2270 if (!data)
2271 goto unlock;
2272
2273 userpg = data->user_page;
2274
2275 /*
2276 * Disable preemption so as to not let the corresponding user-space
2277 * spin too long if we get preempted.
2278 */
2279 preempt_disable();
2280 ++userpg->lock;
2281 barrier();
2282 userpg->index = perf_event_index(event);
2283 userpg->offset = atomic64_read(&event->count);
2284 if (event->state == PERF_EVENT_STATE_ACTIVE)
2285 userpg->offset -= atomic64_read(&event->hw.prev_count);
2286
2287 userpg->time_enabled = event->total_time_enabled +
2288 atomic64_read(&event->child_total_time_enabled);
2289
2290 userpg->time_running = event->total_time_running +
2291 atomic64_read(&event->child_total_time_running);
2292
2293 barrier();
2294 ++userpg->lock;
2295 preempt_enable();
2296unlock:
2297 rcu_read_unlock();
2298}
2299
Peter Zijlstra906010b2009-09-21 16:08:49 +02002300static unsigned long perf_data_size(struct perf_mmap_data *data)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002301{
Peter Zijlstra906010b2009-09-21 16:08:49 +02002302 return data->nr_pages << (PAGE_SHIFT + data->data_order);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002303}
2304
Peter Zijlstra906010b2009-09-21 16:08:49 +02002305#ifndef CONFIG_PERF_USE_VMALLOC
2306
2307/*
2308 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
2309 */
2310
2311static struct page *
2312perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
2313{
2314 if (pgoff > data->nr_pages)
2315 return NULL;
2316
2317 if (pgoff == 0)
2318 return virt_to_page(data->user_page);
2319
2320 return virt_to_page(data->data_pages[pgoff - 1]);
2321}
2322
2323static struct perf_mmap_data *
2324perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002325{
2326 struct perf_mmap_data *data;
2327 unsigned long size;
2328 int i;
2329
2330 WARN_ON(atomic_read(&event->mmap_count));
2331
2332 size = sizeof(struct perf_mmap_data);
2333 size += nr_pages * sizeof(void *);
2334
2335 data = kzalloc(size, GFP_KERNEL);
2336 if (!data)
2337 goto fail;
2338
2339 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
2340 if (!data->user_page)
2341 goto fail_user_page;
2342
2343 for (i = 0; i < nr_pages; i++) {
2344 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
2345 if (!data->data_pages[i])
2346 goto fail_data_pages;
2347 }
2348
Peter Zijlstra906010b2009-09-21 16:08:49 +02002349 data->data_order = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002350 data->nr_pages = nr_pages;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002351
Peter Zijlstra906010b2009-09-21 16:08:49 +02002352 return data;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002353
2354fail_data_pages:
2355 for (i--; i >= 0; i--)
2356 free_page((unsigned long)data->data_pages[i]);
2357
2358 free_page((unsigned long)data->user_page);
2359
2360fail_user_page:
2361 kfree(data);
2362
2363fail:
Peter Zijlstra906010b2009-09-21 16:08:49 +02002364 return NULL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002365}
2366
2367static void perf_mmap_free_page(unsigned long addr)
2368{
2369 struct page *page = virt_to_page((void *)addr);
2370
2371 page->mapping = NULL;
2372 __free_page(page);
2373}
2374
Peter Zijlstra906010b2009-09-21 16:08:49 +02002375static void perf_mmap_data_free(struct perf_mmap_data *data)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002376{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002377 int i;
2378
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002379 perf_mmap_free_page((unsigned long)data->user_page);
2380 for (i = 0; i < data->nr_pages; i++)
2381 perf_mmap_free_page((unsigned long)data->data_pages[i]);
Kristian Høgsbergec70ccd2009-12-01 15:05:01 -05002382 kfree(data);
Peter Zijlstra906010b2009-09-21 16:08:49 +02002383}
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002384
Peter Zijlstra906010b2009-09-21 16:08:49 +02002385#else
2386
2387/*
2388 * Back perf_mmap() with vmalloc memory.
2389 *
2390 * Required for architectures that have d-cache aliasing issues.
2391 */
2392
2393static struct page *
2394perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
2395{
2396 if (pgoff > (1UL << data->data_order))
2397 return NULL;
2398
2399 return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
2400}
2401
2402static void perf_mmap_unmark_page(void *addr)
2403{
2404 struct page *page = vmalloc_to_page(addr);
2405
2406 page->mapping = NULL;
2407}
2408
2409static void perf_mmap_data_free_work(struct work_struct *work)
2410{
2411 struct perf_mmap_data *data;
2412 void *base;
2413 int i, nr;
2414
2415 data = container_of(work, struct perf_mmap_data, work);
2416 nr = 1 << data->data_order;
2417
2418 base = data->user_page;
2419 for (i = 0; i < nr + 1; i++)
2420 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
2421
2422 vfree(base);
Kristian Høgsbergec70ccd2009-12-01 15:05:01 -05002423 kfree(data);
Peter Zijlstra906010b2009-09-21 16:08:49 +02002424}
2425
2426static void perf_mmap_data_free(struct perf_mmap_data *data)
2427{
2428 schedule_work(&data->work);
2429}
2430
2431static struct perf_mmap_data *
2432perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2433{
2434 struct perf_mmap_data *data;
2435 unsigned long size;
2436 void *all_buf;
2437
2438 WARN_ON(atomic_read(&event->mmap_count));
2439
2440 size = sizeof(struct perf_mmap_data);
2441 size += sizeof(void *);
2442
2443 data = kzalloc(size, GFP_KERNEL);
2444 if (!data)
2445 goto fail;
2446
2447 INIT_WORK(&data->work, perf_mmap_data_free_work);
2448
2449 all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
2450 if (!all_buf)
2451 goto fail_all_buf;
2452
2453 data->user_page = all_buf;
2454 data->data_pages[0] = all_buf + PAGE_SIZE;
2455 data->data_order = ilog2(nr_pages);
2456 data->nr_pages = 1;
2457
2458 return data;
2459
2460fail_all_buf:
2461 kfree(data);
2462
2463fail:
2464 return NULL;
2465}
2466
2467#endif
2468
2469static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2470{
2471 struct perf_event *event = vma->vm_file->private_data;
2472 struct perf_mmap_data *data;
2473 int ret = VM_FAULT_SIGBUS;
2474
2475 if (vmf->flags & FAULT_FLAG_MKWRITE) {
2476 if (vmf->pgoff == 0)
2477 ret = 0;
2478 return ret;
2479 }
2480
2481 rcu_read_lock();
2482 data = rcu_dereference(event->data);
2483 if (!data)
2484 goto unlock;
2485
2486 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
2487 goto unlock;
2488
2489 vmf->page = perf_mmap_to_page(data, vmf->pgoff);
2490 if (!vmf->page)
2491 goto unlock;
2492
2493 get_page(vmf->page);
2494 vmf->page->mapping = vma->vm_file->f_mapping;
2495 vmf->page->index = vmf->pgoff;
2496
2497 ret = 0;
2498unlock:
2499 rcu_read_unlock();
2500
2501 return ret;
2502}
2503
2504static void
2505perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
2506{
2507 long max_size = perf_data_size(data);
2508
2509 atomic_set(&data->lock, -1);
2510
2511 if (event->attr.watermark) {
2512 data->watermark = min_t(long, max_size,
2513 event->attr.wakeup_watermark);
2514 }
2515
2516 if (!data->watermark)
Stephane Eranian8904b182009-11-20 22:19:57 +01002517 data->watermark = max_size / 2;
Peter Zijlstra906010b2009-09-21 16:08:49 +02002518
2519
2520 rcu_assign_pointer(event->data, data);
2521}
2522
2523static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
2524{
2525 struct perf_mmap_data *data;
2526
2527 data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
2528 perf_mmap_data_free(data);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002529}
2530
Peter Zijlstra906010b2009-09-21 16:08:49 +02002531static void perf_mmap_data_release(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002532{
2533 struct perf_mmap_data *data = event->data;
2534
2535 WARN_ON(atomic_read(&event->mmap_count));
2536
2537 rcu_assign_pointer(event->data, NULL);
Peter Zijlstra906010b2009-09-21 16:08:49 +02002538 call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002539}
2540
2541static void perf_mmap_open(struct vm_area_struct *vma)
2542{
2543 struct perf_event *event = vma->vm_file->private_data;
2544
2545 atomic_inc(&event->mmap_count);
2546}
2547
2548static void perf_mmap_close(struct vm_area_struct *vma)
2549{
2550 struct perf_event *event = vma->vm_file->private_data;
2551
2552 WARN_ON_ONCE(event->ctx->parent_ctx);
2553 if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
Peter Zijlstra906010b2009-09-21 16:08:49 +02002554 unsigned long size = perf_data_size(event->data);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002555 struct user_struct *user = current_user();
2556
Peter Zijlstra906010b2009-09-21 16:08:49 +02002557 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002558 vma->vm_mm->locked_vm -= event->data->nr_locked;
Peter Zijlstra906010b2009-09-21 16:08:49 +02002559 perf_mmap_data_release(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002560 mutex_unlock(&event->mmap_mutex);
2561 }
2562}
2563
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04002564static const struct vm_operations_struct perf_mmap_vmops = {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002565 .open = perf_mmap_open,
2566 .close = perf_mmap_close,
2567 .fault = perf_mmap_fault,
2568 .page_mkwrite = perf_mmap_fault,
2569};
2570
2571static int perf_mmap(struct file *file, struct vm_area_struct *vma)
2572{
2573 struct perf_event *event = file->private_data;
2574 unsigned long user_locked, user_lock_limit;
2575 struct user_struct *user = current_user();
2576 unsigned long locked, lock_limit;
Peter Zijlstra906010b2009-09-21 16:08:49 +02002577 struct perf_mmap_data *data;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002578 unsigned long vma_size;
2579 unsigned long nr_pages;
2580 long user_extra, extra;
2581 int ret = 0;
2582
2583 if (!(vma->vm_flags & VM_SHARED))
2584 return -EINVAL;
2585
2586 vma_size = vma->vm_end - vma->vm_start;
2587 nr_pages = (vma_size / PAGE_SIZE) - 1;
2588
2589 /*
2590 * If we have data pages ensure they're a power-of-two number, so we
2591 * can do bitmasks instead of modulo.
2592 */
2593 if (nr_pages != 0 && !is_power_of_2(nr_pages))
2594 return -EINVAL;
2595
2596 if (vma_size != PAGE_SIZE * (1 + nr_pages))
2597 return -EINVAL;
2598
2599 if (vma->vm_pgoff != 0)
2600 return -EINVAL;
2601
2602 WARN_ON_ONCE(event->ctx->parent_ctx);
2603 mutex_lock(&event->mmap_mutex);
2604 if (event->output) {
2605 ret = -EINVAL;
2606 goto unlock;
2607 }
2608
2609 if (atomic_inc_not_zero(&event->mmap_count)) {
2610 if (nr_pages != event->data->nr_pages)
2611 ret = -EINVAL;
2612 goto unlock;
2613 }
2614
2615 user_extra = nr_pages + 1;
2616 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
2617
2618 /*
2619 * Increase the limit linearly with more CPUs:
2620 */
2621 user_lock_limit *= num_online_cpus();
2622
2623 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2624
2625 extra = 0;
2626 if (user_locked > user_lock_limit)
2627 extra = user_locked - user_lock_limit;
2628
Jiri Slaby78d7d402010-03-05 13:42:54 -08002629 lock_limit = rlimit(RLIMIT_MEMLOCK);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002630 lock_limit >>= PAGE_SHIFT;
2631 locked = vma->vm_mm->locked_vm + extra;
2632
2633 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
2634 !capable(CAP_IPC_LOCK)) {
2635 ret = -EPERM;
2636 goto unlock;
2637 }
2638
2639 WARN_ON(event->data);
Peter Zijlstra906010b2009-09-21 16:08:49 +02002640
2641 data = perf_mmap_data_alloc(event, nr_pages);
2642 ret = -ENOMEM;
2643 if (!data)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002644 goto unlock;
2645
Peter Zijlstra906010b2009-09-21 16:08:49 +02002646 ret = 0;
2647 perf_mmap_data_init(event, data);
2648
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002649 atomic_set(&event->mmap_count, 1);
2650 atomic_long_add(user_extra, &user->locked_vm);
2651 vma->vm_mm->locked_vm += extra;
2652 event->data->nr_locked = extra;
2653 if (vma->vm_flags & VM_WRITE)
2654 event->data->writable = 1;
2655
2656unlock:
2657 mutex_unlock(&event->mmap_mutex);
2658
2659 vma->vm_flags |= VM_RESERVED;
2660 vma->vm_ops = &perf_mmap_vmops;
2661
2662 return ret;
2663}
2664
2665static int perf_fasync(int fd, struct file *filp, int on)
2666{
2667 struct inode *inode = filp->f_path.dentry->d_inode;
2668 struct perf_event *event = filp->private_data;
2669 int retval;
2670
2671 mutex_lock(&inode->i_mutex);
2672 retval = fasync_helper(fd, filp, on, &event->fasync);
2673 mutex_unlock(&inode->i_mutex);
2674
2675 if (retval < 0)
2676 return retval;
2677
2678 return 0;
2679}
2680
2681static const struct file_operations perf_fops = {
Arnd Bergmann3326c1c2010-03-23 19:09:33 +01002682 .llseek = no_llseek,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002683 .release = perf_release,
2684 .read = perf_read,
2685 .poll = perf_poll,
2686 .unlocked_ioctl = perf_ioctl,
2687 .compat_ioctl = perf_ioctl,
2688 .mmap = perf_mmap,
2689 .fasync = perf_fasync,
2690};
2691
2692/*
2693 * Perf event wakeup
2694 *
2695 * If there's data, ensure we set the poll() state and publish everything
2696 * to user-space before waking everybody up.
2697 */
2698
2699void perf_event_wakeup(struct perf_event *event)
2700{
2701 wake_up_all(&event->waitq);
2702
2703 if (event->pending_kill) {
2704 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
2705 event->pending_kill = 0;
2706 }
2707}
2708
2709/*
2710 * Pending wakeups
2711 *
2712 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2713 *
2714 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2715 * single linked list and use cmpxchg() to add entries lockless.
2716 */
2717
2718static void perf_pending_event(struct perf_pending_entry *entry)
2719{
2720 struct perf_event *event = container_of(entry,
2721 struct perf_event, pending);
2722
2723 if (event->pending_disable) {
2724 event->pending_disable = 0;
2725 __perf_event_disable(event);
2726 }
2727
2728 if (event->pending_wakeup) {
2729 event->pending_wakeup = 0;
2730 perf_event_wakeup(event);
2731 }
2732}
2733
2734#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2735
2736static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2737 PENDING_TAIL,
2738};
2739
2740static void perf_pending_queue(struct perf_pending_entry *entry,
2741 void (*func)(struct perf_pending_entry *))
2742{
2743 struct perf_pending_entry **head;
2744
2745 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2746 return;
2747
2748 entry->func = func;
2749
2750 head = &get_cpu_var(perf_pending_head);
2751
2752 do {
2753 entry->next = *head;
2754 } while (cmpxchg(head, entry->next, entry) != entry->next);
2755
2756 set_perf_event_pending();
2757
2758 put_cpu_var(perf_pending_head);
2759}
2760
2761static int __perf_pending_run(void)
2762{
2763 struct perf_pending_entry *list;
2764 int nr = 0;
2765
2766 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2767 while (list != PENDING_TAIL) {
2768 void (*func)(struct perf_pending_entry *);
2769 struct perf_pending_entry *entry = list;
2770
2771 list = list->next;
2772
2773 func = entry->func;
2774 entry->next = NULL;
2775 /*
2776 * Ensure we observe the unqueue before we issue the wakeup,
2777 * so that we won't be waiting forever.
2778 * -- see perf_not_pending().
2779 */
2780 smp_wmb();
2781
2782 func(entry);
2783 nr++;
2784 }
2785
2786 return nr;
2787}
2788
2789static inline int perf_not_pending(struct perf_event *event)
2790{
2791 /*
2792 * If we flush on whatever cpu we run, there is a chance we don't
2793 * need to wait.
2794 */
2795 get_cpu();
2796 __perf_pending_run();
2797 put_cpu();
2798
2799 /*
2800 * Ensure we see the proper queue state before going to sleep
2801 * so that we do not miss the wakeup. -- see perf_pending_handle()
2802 */
2803 smp_rmb();
2804 return event->pending.next == NULL;
2805}
2806
2807static void perf_pending_sync(struct perf_event *event)
2808{
2809 wait_event(event->waitq, perf_not_pending(event));
2810}
2811
2812void perf_event_do_pending(void)
2813{
2814 __perf_pending_run();
2815}
2816
2817/*
2818 * Callchain support -- arch specific
2819 */
2820
2821__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2822{
2823 return NULL;
2824}
2825
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +01002826__weak
2827void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip)
2828{
2829}
Frederic Weisbecker26d80aa2010-04-03 12:22:05 +02002830
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +01002831
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002832/*
Zhang, Yanmin39447b32010-04-19 13:32:41 +08002833 * We assume there is only KVM supporting the callbacks.
2834 * Later on, we might change it to a list if there is
2835 * another virtualization implementation supporting the callbacks.
2836 */
2837struct perf_guest_info_callbacks *perf_guest_cbs;
2838
2839int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
2840{
2841 perf_guest_cbs = cbs;
2842 return 0;
2843}
2844EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
2845
2846int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
2847{
2848 perf_guest_cbs = NULL;
2849 return 0;
2850}
2851EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
2852
2853/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002854 * Output
2855 */
2856static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
2857 unsigned long offset, unsigned long head)
2858{
2859 unsigned long mask;
2860
2861 if (!data->writable)
2862 return true;
2863
Peter Zijlstra906010b2009-09-21 16:08:49 +02002864 mask = perf_data_size(data) - 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002865
2866 offset = (offset - tail) & mask;
2867 head = (head - tail) & mask;
2868
2869 if ((int)(head - offset) < 0)
2870 return false;
2871
2872 return true;
2873}
2874
2875static void perf_output_wakeup(struct perf_output_handle *handle)
2876{
2877 atomic_set(&handle->data->poll, POLL_IN);
2878
2879 if (handle->nmi) {
2880 handle->event->pending_wakeup = 1;
2881 perf_pending_queue(&handle->event->pending,
2882 perf_pending_event);
2883 } else
2884 perf_event_wakeup(handle->event);
2885}
2886
2887/*
2888 * Curious locking construct.
2889 *
2890 * We need to ensure a later event_id doesn't publish a head when a former
2891 * event_id isn't done writing. However since we need to deal with NMIs we
2892 * cannot fully serialize things.
2893 *
2894 * What we do is serialize between CPUs so we only have to deal with NMI
2895 * nesting on a single CPU.
2896 *
2897 * We only publish the head (and generate a wakeup) when the outer-most
2898 * event_id completes.
2899 */
2900static void perf_output_lock(struct perf_output_handle *handle)
2901{
2902 struct perf_mmap_data *data = handle->data;
Peter Zijlstra559fdc32009-11-16 12:45:14 +01002903 int cur, cpu = get_cpu();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002904
2905 handle->locked = 0;
2906
Peter Zijlstra559fdc32009-11-16 12:45:14 +01002907 for (;;) {
2908 cur = atomic_cmpxchg(&data->lock, -1, cpu);
2909 if (cur == -1) {
2910 handle->locked = 1;
2911 break;
2912 }
2913 if (cur == cpu)
2914 break;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002915
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002916 cpu_relax();
Peter Zijlstra559fdc32009-11-16 12:45:14 +01002917 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002918}
2919
2920static void perf_output_unlock(struct perf_output_handle *handle)
2921{
2922 struct perf_mmap_data *data = handle->data;
2923 unsigned long head;
2924 int cpu;
2925
2926 data->done_head = data->head;
2927
2928 if (!handle->locked)
2929 goto out;
2930
2931again:
2932 /*
2933 * The xchg implies a full barrier that ensures all writes are done
2934 * before we publish the new head, matched by a rmb() in userspace when
2935 * reading this position.
2936 */
2937 while ((head = atomic_long_xchg(&data->done_head, 0)))
2938 data->user_page->data_head = head;
2939
2940 /*
2941 * NMI can happen here, which means we can miss a done_head update.
2942 */
2943
2944 cpu = atomic_xchg(&data->lock, -1);
2945 WARN_ON_ONCE(cpu != smp_processor_id());
2946
2947 /*
2948 * Therefore we have to validate we did not indeed do so.
2949 */
2950 if (unlikely(atomic_long_read(&data->done_head))) {
2951 /*
2952 * Since we had it locked, we can lock it again.
2953 */
2954 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2955 cpu_relax();
2956
2957 goto again;
2958 }
2959
2960 if (atomic_xchg(&data->wakeup, 0))
2961 perf_output_wakeup(handle);
2962out:
Peter Zijlstra559fdc32009-11-16 12:45:14 +01002963 put_cpu();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002964}
2965
2966void perf_output_copy(struct perf_output_handle *handle,
2967 const void *buf, unsigned int len)
2968{
2969 unsigned int pages_mask;
Peter Zijlstra906010b2009-09-21 16:08:49 +02002970 unsigned long offset;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002971 unsigned int size;
2972 void **pages;
2973
2974 offset = handle->offset;
2975 pages_mask = handle->data->nr_pages - 1;
2976 pages = handle->data->data_pages;
2977
2978 do {
Peter Zijlstra906010b2009-09-21 16:08:49 +02002979 unsigned long page_offset;
2980 unsigned long page_size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002981 int nr;
2982
2983 nr = (offset >> PAGE_SHIFT) & pages_mask;
Peter Zijlstra906010b2009-09-21 16:08:49 +02002984 page_size = 1UL << (handle->data->data_order + PAGE_SHIFT);
2985 page_offset = offset & (page_size - 1);
2986 size = min_t(unsigned int, page_size - page_offset, len);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002987
2988 memcpy(pages[nr] + page_offset, buf, size);
2989
2990 len -= size;
2991 buf += size;
2992 offset += size;
2993 } while (len);
2994
2995 handle->offset = offset;
2996
2997 /*
2998 * Check we didn't copy past our reservation window, taking the
2999 * possible unsigned int wrap into account.
3000 */
3001 WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
3002}
3003
3004int perf_output_begin(struct perf_output_handle *handle,
3005 struct perf_event *event, unsigned int size,
3006 int nmi, int sample)
3007{
3008 struct perf_event *output_event;
3009 struct perf_mmap_data *data;
3010 unsigned long tail, offset, head;
3011 int have_lost;
3012 struct {
3013 struct perf_event_header header;
3014 u64 id;
3015 u64 lost;
3016 } lost_event;
3017
3018 rcu_read_lock();
3019 /*
3020 * For inherited events we send all the output towards the parent.
3021 */
3022 if (event->parent)
3023 event = event->parent;
3024
3025 output_event = rcu_dereference(event->output);
3026 if (output_event)
3027 event = output_event;
3028
3029 data = rcu_dereference(event->data);
3030 if (!data)
3031 goto out;
3032
3033 handle->data = data;
3034 handle->event = event;
3035 handle->nmi = nmi;
3036 handle->sample = sample;
3037
3038 if (!data->nr_pages)
3039 goto fail;
3040
3041 have_lost = atomic_read(&data->lost);
3042 if (have_lost)
3043 size += sizeof(lost_event);
3044
3045 perf_output_lock(handle);
3046
3047 do {
3048 /*
3049 * Userspace could choose to issue a mb() before updating the
3050 * tail pointer. So that all reads will be completed before the
3051 * write is issued.
3052 */
3053 tail = ACCESS_ONCE(data->user_page->data_tail);
3054 smp_rmb();
3055 offset = head = atomic_long_read(&data->head);
3056 head += size;
3057 if (unlikely(!perf_output_space(data, tail, offset, head)))
3058 goto fail;
3059 } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
3060
3061 handle->offset = offset;
3062 handle->head = head;
3063
3064 if (head - tail > data->watermark)
3065 atomic_set(&data->wakeup, 1);
3066
3067 if (have_lost) {
3068 lost_event.header.type = PERF_RECORD_LOST;
3069 lost_event.header.misc = 0;
3070 lost_event.header.size = sizeof(lost_event);
3071 lost_event.id = event->id;
3072 lost_event.lost = atomic_xchg(&data->lost, 0);
3073
3074 perf_output_put(handle, lost_event);
3075 }
3076
3077 return 0;
3078
3079fail:
3080 atomic_inc(&data->lost);
3081 perf_output_unlock(handle);
3082out:
3083 rcu_read_unlock();
3084
3085 return -ENOSPC;
3086}
3087
3088void perf_output_end(struct perf_output_handle *handle)
3089{
3090 struct perf_event *event = handle->event;
3091 struct perf_mmap_data *data = handle->data;
3092
3093 int wakeup_events = event->attr.wakeup_events;
3094
3095 if (handle->sample && wakeup_events) {
3096 int events = atomic_inc_return(&data->events);
3097 if (events >= wakeup_events) {
3098 atomic_sub(wakeup_events, &data->events);
3099 atomic_set(&data->wakeup, 1);
3100 }
3101 }
3102
3103 perf_output_unlock(handle);
3104 rcu_read_unlock();
3105}
3106
3107static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3108{
3109 /*
3110 * only top level events have the pid namespace they were created in
3111 */
3112 if (event->parent)
3113 event = event->parent;
3114
3115 return task_tgid_nr_ns(p, event->ns);
3116}
3117
3118static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3119{
3120 /*
3121 * only top level events have the pid namespace they were created in
3122 */
3123 if (event->parent)
3124 event = event->parent;
3125
3126 return task_pid_nr_ns(p, event->ns);
3127}
3128
3129static void perf_output_read_one(struct perf_output_handle *handle,
3130 struct perf_event *event)
3131{
3132 u64 read_format = event->attr.read_format;
3133 u64 values[4];
3134 int n = 0;
3135
3136 values[n++] = atomic64_read(&event->count);
3137 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3138 values[n++] = event->total_time_enabled +
3139 atomic64_read(&event->child_total_time_enabled);
3140 }
3141 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3142 values[n++] = event->total_time_running +
3143 atomic64_read(&event->child_total_time_running);
3144 }
3145 if (read_format & PERF_FORMAT_ID)
3146 values[n++] = primary_event_id(event);
3147
3148 perf_output_copy(handle, values, n * sizeof(u64));
3149}
3150
3151/*
3152 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3153 */
3154static void perf_output_read_group(struct perf_output_handle *handle,
3155 struct perf_event *event)
3156{
3157 struct perf_event *leader = event->group_leader, *sub;
3158 u64 read_format = event->attr.read_format;
3159 u64 values[5];
3160 int n = 0;
3161
3162 values[n++] = 1 + leader->nr_siblings;
3163
3164 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3165 values[n++] = leader->total_time_enabled;
3166
3167 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3168 values[n++] = leader->total_time_running;
3169
3170 if (leader != event)
3171 leader->pmu->read(leader);
3172
3173 values[n++] = atomic64_read(&leader->count);
3174 if (read_format & PERF_FORMAT_ID)
3175 values[n++] = primary_event_id(leader);
3176
3177 perf_output_copy(handle, values, n * sizeof(u64));
3178
3179 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3180 n = 0;
3181
3182 if (sub != event)
3183 sub->pmu->read(sub);
3184
3185 values[n++] = atomic64_read(&sub->count);
3186 if (read_format & PERF_FORMAT_ID)
3187 values[n++] = primary_event_id(sub);
3188
3189 perf_output_copy(handle, values, n * sizeof(u64));
3190 }
3191}
3192
3193static void perf_output_read(struct perf_output_handle *handle,
3194 struct perf_event *event)
3195{
3196 if (event->attr.read_format & PERF_FORMAT_GROUP)
3197 perf_output_read_group(handle, event);
3198 else
3199 perf_output_read_one(handle, event);
3200}
3201
3202void perf_output_sample(struct perf_output_handle *handle,
3203 struct perf_event_header *header,
3204 struct perf_sample_data *data,
3205 struct perf_event *event)
3206{
3207 u64 sample_type = data->type;
3208
3209 perf_output_put(handle, *header);
3210
3211 if (sample_type & PERF_SAMPLE_IP)
3212 perf_output_put(handle, data->ip);
3213
3214 if (sample_type & PERF_SAMPLE_TID)
3215 perf_output_put(handle, data->tid_entry);
3216
3217 if (sample_type & PERF_SAMPLE_TIME)
3218 perf_output_put(handle, data->time);
3219
3220 if (sample_type & PERF_SAMPLE_ADDR)
3221 perf_output_put(handle, data->addr);
3222
3223 if (sample_type & PERF_SAMPLE_ID)
3224 perf_output_put(handle, data->id);
3225
3226 if (sample_type & PERF_SAMPLE_STREAM_ID)
3227 perf_output_put(handle, data->stream_id);
3228
3229 if (sample_type & PERF_SAMPLE_CPU)
3230 perf_output_put(handle, data->cpu_entry);
3231
3232 if (sample_type & PERF_SAMPLE_PERIOD)
3233 perf_output_put(handle, data->period);
3234
3235 if (sample_type & PERF_SAMPLE_READ)
3236 perf_output_read(handle, event);
3237
3238 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3239 if (data->callchain) {
3240 int size = 1;
3241
3242 if (data->callchain)
3243 size += data->callchain->nr;
3244
3245 size *= sizeof(u64);
3246
3247 perf_output_copy(handle, data->callchain, size);
3248 } else {
3249 u64 nr = 0;
3250 perf_output_put(handle, nr);
3251 }
3252 }
3253
3254 if (sample_type & PERF_SAMPLE_RAW) {
3255 if (data->raw) {
3256 perf_output_put(handle, data->raw->size);
3257 perf_output_copy(handle, data->raw->data,
3258 data->raw->size);
3259 } else {
3260 struct {
3261 u32 size;
3262 u32 data;
3263 } raw = {
3264 .size = sizeof(u32),
3265 .data = 0,
3266 };
3267 perf_output_put(handle, raw);
3268 }
3269 }
3270}
3271
3272void perf_prepare_sample(struct perf_event_header *header,
3273 struct perf_sample_data *data,
3274 struct perf_event *event,
3275 struct pt_regs *regs)
3276{
3277 u64 sample_type = event->attr.sample_type;
3278
3279 data->type = sample_type;
3280
3281 header->type = PERF_RECORD_SAMPLE;
3282 header->size = sizeof(*header);
3283
3284 header->misc = 0;
3285 header->misc |= perf_misc_flags(regs);
3286
3287 if (sample_type & PERF_SAMPLE_IP) {
3288 data->ip = perf_instruction_pointer(regs);
3289
3290 header->size += sizeof(data->ip);
3291 }
3292
3293 if (sample_type & PERF_SAMPLE_TID) {
3294 /* namespace issues */
3295 data->tid_entry.pid = perf_event_pid(event, current);
3296 data->tid_entry.tid = perf_event_tid(event, current);
3297
3298 header->size += sizeof(data->tid_entry);
3299 }
3300
3301 if (sample_type & PERF_SAMPLE_TIME) {
3302 data->time = perf_clock();
3303
3304 header->size += sizeof(data->time);
3305 }
3306
3307 if (sample_type & PERF_SAMPLE_ADDR)
3308 header->size += sizeof(data->addr);
3309
3310 if (sample_type & PERF_SAMPLE_ID) {
3311 data->id = primary_event_id(event);
3312
3313 header->size += sizeof(data->id);
3314 }
3315
3316 if (sample_type & PERF_SAMPLE_STREAM_ID) {
3317 data->stream_id = event->id;
3318
3319 header->size += sizeof(data->stream_id);
3320 }
3321
3322 if (sample_type & PERF_SAMPLE_CPU) {
3323 data->cpu_entry.cpu = raw_smp_processor_id();
3324 data->cpu_entry.reserved = 0;
3325
3326 header->size += sizeof(data->cpu_entry);
3327 }
3328
3329 if (sample_type & PERF_SAMPLE_PERIOD)
3330 header->size += sizeof(data->period);
3331
3332 if (sample_type & PERF_SAMPLE_READ)
3333 header->size += perf_event_read_size(event);
3334
3335 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3336 int size = 1;
3337
3338 data->callchain = perf_callchain(regs);
3339
3340 if (data->callchain)
3341 size += data->callchain->nr;
3342
3343 header->size += size * sizeof(u64);
3344 }
3345
3346 if (sample_type & PERF_SAMPLE_RAW) {
3347 int size = sizeof(u32);
3348
3349 if (data->raw)
3350 size += data->raw->size;
3351 else
3352 size += sizeof(u32);
3353
3354 WARN_ON_ONCE(size & (sizeof(u64)-1));
3355 header->size += size;
3356 }
3357}
3358
3359static void perf_event_output(struct perf_event *event, int nmi,
3360 struct perf_sample_data *data,
3361 struct pt_regs *regs)
3362{
3363 struct perf_output_handle handle;
3364 struct perf_event_header header;
3365
3366 perf_prepare_sample(&header, data, event, regs);
3367
3368 if (perf_output_begin(&handle, event, header.size, nmi, 1))
3369 return;
3370
3371 perf_output_sample(&handle, &header, data, event);
3372
3373 perf_output_end(&handle);
3374}
3375
3376/*
3377 * read event_id
3378 */
3379
3380struct perf_read_event {
3381 struct perf_event_header header;
3382
3383 u32 pid;
3384 u32 tid;
3385};
3386
3387static void
3388perf_event_read_event(struct perf_event *event,
3389 struct task_struct *task)
3390{
3391 struct perf_output_handle handle;
3392 struct perf_read_event read_event = {
3393 .header = {
3394 .type = PERF_RECORD_READ,
3395 .misc = 0,
3396 .size = sizeof(read_event) + perf_event_read_size(event),
3397 },
3398 .pid = perf_event_pid(event, task),
3399 .tid = perf_event_tid(event, task),
3400 };
3401 int ret;
3402
3403 ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3404 if (ret)
3405 return;
3406
3407 perf_output_put(&handle, read_event);
3408 perf_output_read(&handle, event);
3409
3410 perf_output_end(&handle);
3411}
3412
3413/*
3414 * task tracking -- fork/exit
3415 *
3416 * enabled by: attr.comm | attr.mmap | attr.task
3417 */
3418
3419struct perf_task_event {
3420 struct task_struct *task;
3421 struct perf_event_context *task_ctx;
3422
3423 struct {
3424 struct perf_event_header header;
3425
3426 u32 pid;
3427 u32 ppid;
3428 u32 tid;
3429 u32 ptid;
3430 u64 time;
3431 } event_id;
3432};
3433
3434static void perf_event_task_output(struct perf_event *event,
3435 struct perf_task_event *task_event)
3436{
3437 struct perf_output_handle handle;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003438 struct task_struct *task = task_event->task;
Mike Galbraith8bb39f92010-03-26 11:11:33 +01003439 unsigned long flags;
3440 int size, ret;
3441
3442 /*
3443 * If this CPU attempts to acquire an rq lock held by a CPU spinning
3444 * in perf_output_lock() from interrupt context, it's game over.
3445 */
3446 local_irq_save(flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003447
3448 size = task_event->event_id.header.size;
3449 ret = perf_output_begin(&handle, event, size, 0, 0);
3450
Mike Galbraith8bb39f92010-03-26 11:11:33 +01003451 if (ret) {
3452 local_irq_restore(flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003453 return;
Mike Galbraith8bb39f92010-03-26 11:11:33 +01003454 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003455
3456 task_event->event_id.pid = perf_event_pid(event, task);
3457 task_event->event_id.ppid = perf_event_pid(event, current);
3458
3459 task_event->event_id.tid = perf_event_tid(event, task);
3460 task_event->event_id.ptid = perf_event_tid(event, current);
3461
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003462 perf_output_put(&handle, task_event->event_id);
3463
3464 perf_output_end(&handle);
Mike Galbraith8bb39f92010-03-26 11:11:33 +01003465 local_irq_restore(flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003466}
3467
3468static int perf_event_task_match(struct perf_event *event)
3469{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01003470 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01003471 return 0;
3472
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003473 if (event->cpu != -1 && event->cpu != smp_processor_id())
3474 return 0;
3475
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003476 if (event->attr.comm || event->attr.mmap || event->attr.task)
3477 return 1;
3478
3479 return 0;
3480}
3481
3482static void perf_event_task_ctx(struct perf_event_context *ctx,
3483 struct perf_task_event *task_event)
3484{
3485 struct perf_event *event;
3486
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003487 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3488 if (perf_event_task_match(event))
3489 perf_event_task_output(event, task_event);
3490 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003491}
3492
3493static void perf_event_task_event(struct perf_task_event *task_event)
3494{
3495 struct perf_cpu_context *cpuctx;
3496 struct perf_event_context *ctx = task_event->task_ctx;
3497
Peter Zijlstrad6ff86c2009-11-20 22:19:46 +01003498 rcu_read_lock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003499 cpuctx = &get_cpu_var(perf_cpu_context);
3500 perf_event_task_ctx(&cpuctx->ctx, task_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003501 if (!ctx)
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01003502 ctx = rcu_dereference(current->perf_event_ctxp);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003503 if (ctx)
3504 perf_event_task_ctx(ctx, task_event);
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003505 put_cpu_var(perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003506 rcu_read_unlock();
3507}
3508
3509static void perf_event_task(struct task_struct *task,
3510 struct perf_event_context *task_ctx,
3511 int new)
3512{
3513 struct perf_task_event task_event;
3514
3515 if (!atomic_read(&nr_comm_events) &&
3516 !atomic_read(&nr_mmap_events) &&
3517 !atomic_read(&nr_task_events))
3518 return;
3519
3520 task_event = (struct perf_task_event){
3521 .task = task,
3522 .task_ctx = task_ctx,
3523 .event_id = {
3524 .header = {
3525 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3526 .misc = 0,
3527 .size = sizeof(task_event.event_id),
3528 },
3529 /* .pid */
3530 /* .ppid */
3531 /* .tid */
3532 /* .ptid */
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01003533 .time = perf_clock(),
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003534 },
3535 };
3536
3537 perf_event_task_event(&task_event);
3538}
3539
3540void perf_event_fork(struct task_struct *task)
3541{
3542 perf_event_task(task, NULL, 1);
3543}
3544
3545/*
3546 * comm tracking
3547 */
3548
3549struct perf_comm_event {
3550 struct task_struct *task;
3551 char *comm;
3552 int comm_size;
3553
3554 struct {
3555 struct perf_event_header header;
3556
3557 u32 pid;
3558 u32 tid;
3559 } event_id;
3560};
3561
3562static void perf_event_comm_output(struct perf_event *event,
3563 struct perf_comm_event *comm_event)
3564{
3565 struct perf_output_handle handle;
3566 int size = comm_event->event_id.header.size;
3567 int ret = perf_output_begin(&handle, event, size, 0, 0);
3568
3569 if (ret)
3570 return;
3571
3572 comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
3573 comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3574
3575 perf_output_put(&handle, comm_event->event_id);
3576 perf_output_copy(&handle, comm_event->comm,
3577 comm_event->comm_size);
3578 perf_output_end(&handle);
3579}
3580
3581static int perf_event_comm_match(struct perf_event *event)
3582{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01003583 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01003584 return 0;
3585
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003586 if (event->cpu != -1 && event->cpu != smp_processor_id())
3587 return 0;
3588
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003589 if (event->attr.comm)
3590 return 1;
3591
3592 return 0;
3593}
3594
3595static void perf_event_comm_ctx(struct perf_event_context *ctx,
3596 struct perf_comm_event *comm_event)
3597{
3598 struct perf_event *event;
3599
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003600 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3601 if (perf_event_comm_match(event))
3602 perf_event_comm_output(event, comm_event);
3603 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003604}
3605
3606static void perf_event_comm_event(struct perf_comm_event *comm_event)
3607{
3608 struct perf_cpu_context *cpuctx;
3609 struct perf_event_context *ctx;
3610 unsigned int size;
3611 char comm[TASK_COMM_LEN];
3612
3613 memset(comm, 0, sizeof(comm));
Márton Németh96b02d72009-11-21 23:10:15 +01003614 strlcpy(comm, comm_event->task->comm, sizeof(comm));
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003615 size = ALIGN(strlen(comm)+1, sizeof(u64));
3616
3617 comm_event->comm = comm;
3618 comm_event->comm_size = size;
3619
3620 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3621
Peter Zijlstraf6595f32009-11-20 22:19:47 +01003622 rcu_read_lock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003623 cpuctx = &get_cpu_var(perf_cpu_context);
3624 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003625 ctx = rcu_dereference(current->perf_event_ctxp);
3626 if (ctx)
3627 perf_event_comm_ctx(ctx, comm_event);
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003628 put_cpu_var(perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003629 rcu_read_unlock();
3630}
3631
3632void perf_event_comm(struct task_struct *task)
3633{
3634 struct perf_comm_event comm_event;
3635
3636 if (task->perf_event_ctxp)
3637 perf_event_enable_on_exec(task);
3638
3639 if (!atomic_read(&nr_comm_events))
3640 return;
3641
3642 comm_event = (struct perf_comm_event){
3643 .task = task,
3644 /* .comm */
3645 /* .comm_size */
3646 .event_id = {
3647 .header = {
3648 .type = PERF_RECORD_COMM,
3649 .misc = 0,
3650 /* .size */
3651 },
3652 /* .pid */
3653 /* .tid */
3654 },
3655 };
3656
3657 perf_event_comm_event(&comm_event);
3658}
3659
3660/*
3661 * mmap tracking
3662 */
3663
3664struct perf_mmap_event {
3665 struct vm_area_struct *vma;
3666
3667 const char *file_name;
3668 int file_size;
3669
3670 struct {
3671 struct perf_event_header header;
3672
3673 u32 pid;
3674 u32 tid;
3675 u64 start;
3676 u64 len;
3677 u64 pgoff;
3678 } event_id;
3679};
3680
3681static void perf_event_mmap_output(struct perf_event *event,
3682 struct perf_mmap_event *mmap_event)
3683{
3684 struct perf_output_handle handle;
3685 int size = mmap_event->event_id.header.size;
3686 int ret = perf_output_begin(&handle, event, size, 0, 0);
3687
3688 if (ret)
3689 return;
3690
3691 mmap_event->event_id.pid = perf_event_pid(event, current);
3692 mmap_event->event_id.tid = perf_event_tid(event, current);
3693
3694 perf_output_put(&handle, mmap_event->event_id);
3695 perf_output_copy(&handle, mmap_event->file_name,
3696 mmap_event->file_size);
3697 perf_output_end(&handle);
3698}
3699
3700static int perf_event_mmap_match(struct perf_event *event,
3701 struct perf_mmap_event *mmap_event)
3702{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01003703 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01003704 return 0;
3705
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003706 if (event->cpu != -1 && event->cpu != smp_processor_id())
3707 return 0;
3708
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003709 if (event->attr.mmap)
3710 return 1;
3711
3712 return 0;
3713}
3714
3715static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3716 struct perf_mmap_event *mmap_event)
3717{
3718 struct perf_event *event;
3719
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003720 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3721 if (perf_event_mmap_match(event, mmap_event))
3722 perf_event_mmap_output(event, mmap_event);
3723 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003724}
3725
3726static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3727{
3728 struct perf_cpu_context *cpuctx;
3729 struct perf_event_context *ctx;
3730 struct vm_area_struct *vma = mmap_event->vma;
3731 struct file *file = vma->vm_file;
3732 unsigned int size;
3733 char tmp[16];
3734 char *buf = NULL;
3735 const char *name;
3736
3737 memset(tmp, 0, sizeof(tmp));
3738
3739 if (file) {
3740 /*
3741 * d_path works from the end of the buffer backwards, so we
3742 * need to add enough zero bytes after the string to handle
3743 * the 64bit alignment we do later.
3744 */
3745 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3746 if (!buf) {
3747 name = strncpy(tmp, "//enomem", sizeof(tmp));
3748 goto got_name;
3749 }
3750 name = d_path(&file->f_path, buf, PATH_MAX);
3751 if (IS_ERR(name)) {
3752 name = strncpy(tmp, "//toolong", sizeof(tmp));
3753 goto got_name;
3754 }
3755 } else {
3756 if (arch_vma_name(mmap_event->vma)) {
3757 name = strncpy(tmp, arch_vma_name(mmap_event->vma),
3758 sizeof(tmp));
3759 goto got_name;
3760 }
3761
3762 if (!vma->vm_mm) {
3763 name = strncpy(tmp, "[vdso]", sizeof(tmp));
3764 goto got_name;
3765 }
3766
3767 name = strncpy(tmp, "//anon", sizeof(tmp));
3768 goto got_name;
3769 }
3770
3771got_name:
3772 size = ALIGN(strlen(name)+1, sizeof(u64));
3773
3774 mmap_event->file_name = name;
3775 mmap_event->file_size = size;
3776
3777 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3778
Peter Zijlstraf6d9dd22009-11-20 22:19:48 +01003779 rcu_read_lock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003780 cpuctx = &get_cpu_var(perf_cpu_context);
3781 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003782 ctx = rcu_dereference(current->perf_event_ctxp);
3783 if (ctx)
3784 perf_event_mmap_ctx(ctx, mmap_event);
Peter Zijlstra5d27c232009-12-17 13:16:32 +01003785 put_cpu_var(perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003786 rcu_read_unlock();
3787
3788 kfree(buf);
3789}
3790
3791void __perf_event_mmap(struct vm_area_struct *vma)
3792{
3793 struct perf_mmap_event mmap_event;
3794
3795 if (!atomic_read(&nr_mmap_events))
3796 return;
3797
3798 mmap_event = (struct perf_mmap_event){
3799 .vma = vma,
3800 /* .file_name */
3801 /* .file_size */
3802 .event_id = {
3803 .header = {
3804 .type = PERF_RECORD_MMAP,
Zhang, Yanmin39447b32010-04-19 13:32:41 +08003805 .misc = PERF_RECORD_MISC_USER,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003806 /* .size */
3807 },
3808 /* .pid */
3809 /* .tid */
3810 .start = vma->vm_start,
3811 .len = vma->vm_end - vma->vm_start,
Peter Zijlstra3a0304e2010-02-26 10:33:41 +01003812 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003813 },
3814 };
3815
3816 perf_event_mmap_event(&mmap_event);
3817}
3818
3819/*
3820 * IRQ throttle logging
3821 */
3822
3823static void perf_log_throttle(struct perf_event *event, int enable)
3824{
3825 struct perf_output_handle handle;
3826 int ret;
3827
3828 struct {
3829 struct perf_event_header header;
3830 u64 time;
3831 u64 id;
3832 u64 stream_id;
3833 } throttle_event = {
3834 .header = {
3835 .type = PERF_RECORD_THROTTLE,
3836 .misc = 0,
3837 .size = sizeof(throttle_event),
3838 },
3839 .time = perf_clock(),
3840 .id = primary_event_id(event),
3841 .stream_id = event->id,
3842 };
3843
3844 if (enable)
3845 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3846
3847 ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3848 if (ret)
3849 return;
3850
3851 perf_output_put(&handle, throttle_event);
3852 perf_output_end(&handle);
3853}
3854
3855/*
3856 * Generic event overflow handling, sampling.
3857 */
3858
3859static int __perf_event_overflow(struct perf_event *event, int nmi,
3860 int throttle, struct perf_sample_data *data,
3861 struct pt_regs *regs)
3862{
3863 int events = atomic_read(&event->event_limit);
3864 struct hw_perf_event *hwc = &event->hw;
3865 int ret = 0;
3866
3867 throttle = (throttle && event->pmu->unthrottle != NULL);
3868
3869 if (!throttle) {
3870 hwc->interrupts++;
3871 } else {
3872 if (hwc->interrupts != MAX_INTERRUPTS) {
3873 hwc->interrupts++;
3874 if (HZ * hwc->interrupts >
3875 (u64)sysctl_perf_event_sample_rate) {
3876 hwc->interrupts = MAX_INTERRUPTS;
3877 perf_log_throttle(event, 0);
3878 ret = 1;
3879 }
3880 } else {
3881 /*
3882 * Keep re-disabling events even though on the previous
3883 * pass we disabled it - just in case we raced with a
3884 * sched-in and the event got enabled again:
3885 */
3886 ret = 1;
3887 }
3888 }
3889
3890 if (event->attr.freq) {
3891 u64 now = perf_clock();
Peter Zijlstraabd50712010-01-26 18:50:16 +01003892 s64 delta = now - hwc->freq_time_stamp;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003893
Peter Zijlstraabd50712010-01-26 18:50:16 +01003894 hwc->freq_time_stamp = now;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003895
Peter Zijlstraabd50712010-01-26 18:50:16 +01003896 if (delta > 0 && delta < 2*TICK_NSEC)
3897 perf_adjust_period(event, delta, hwc->last_period);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003898 }
3899
3900 /*
3901 * XXX event_limit might not quite work as expected on inherited
3902 * events
3903 */
3904
3905 event->pending_kill = POLL_IN;
3906 if (events && atomic_dec_and_test(&event->event_limit)) {
3907 ret = 1;
3908 event->pending_kill = POLL_HUP;
3909 if (nmi) {
3910 event->pending_disable = 1;
3911 perf_pending_queue(&event->pending,
3912 perf_pending_event);
3913 } else
3914 perf_event_disable(event);
3915 }
3916
Peter Zijlstra453f19e2009-11-20 22:19:43 +01003917 if (event->overflow_handler)
3918 event->overflow_handler(event, nmi, data, regs);
3919 else
3920 perf_event_output(event, nmi, data, regs);
3921
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003922 return ret;
3923}
3924
3925int perf_event_overflow(struct perf_event *event, int nmi,
3926 struct perf_sample_data *data,
3927 struct pt_regs *regs)
3928{
3929 return __perf_event_overflow(event, nmi, 1, data, regs);
3930}
3931
3932/*
3933 * Generic software event infrastructure
3934 */
3935
3936/*
3937 * We directly increment event->count and keep a second value in
3938 * event->hw.period_left to count intervals. This period event
3939 * is kept in the range [-sample_period, 0] so that we can use the
3940 * sign as trigger.
3941 */
3942
3943static u64 perf_swevent_set_period(struct perf_event *event)
3944{
3945 struct hw_perf_event *hwc = &event->hw;
3946 u64 period = hwc->last_period;
3947 u64 nr, offset;
3948 s64 old, val;
3949
3950 hwc->last_period = hwc->sample_period;
3951
3952again:
3953 old = val = atomic64_read(&hwc->period_left);
3954 if (val < 0)
3955 return 0;
3956
3957 nr = div64_u64(period + val, period);
3958 offset = nr * period;
3959 val -= offset;
3960 if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
3961 goto again;
3962
3963 return nr;
3964}
3965
Peter Zijlstra0cff7842009-11-20 22:19:44 +01003966static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003967 int nmi, struct perf_sample_data *data,
3968 struct pt_regs *regs)
3969{
3970 struct hw_perf_event *hwc = &event->hw;
3971 int throttle = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003972
3973 data->period = event->hw.last_period;
Peter Zijlstra0cff7842009-11-20 22:19:44 +01003974 if (!overflow)
3975 overflow = perf_swevent_set_period(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003976
3977 if (hwc->interrupts == MAX_INTERRUPTS)
3978 return;
3979
3980 for (; overflow; overflow--) {
3981 if (__perf_event_overflow(event, nmi, throttle,
3982 data, regs)) {
3983 /*
3984 * We inhibit the overflow from happening when
3985 * hwc->interrupts == MAX_INTERRUPTS.
3986 */
3987 break;
3988 }
3989 throttle = 1;
3990 }
3991}
3992
3993static void perf_swevent_unthrottle(struct perf_event *event)
3994{
3995 /*
3996 * Nothing to do, we already reset hwc->interrupts.
3997 */
3998}
3999
4000static void perf_swevent_add(struct perf_event *event, u64 nr,
4001 int nmi, struct perf_sample_data *data,
4002 struct pt_regs *regs)
4003{
4004 struct hw_perf_event *hwc = &event->hw;
4005
4006 atomic64_add(nr, &event->count);
4007
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004008 if (!regs)
4009 return;
4010
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004011 if (!hwc->sample_period)
4012 return;
4013
4014 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4015 return perf_swevent_overflow(event, 1, nmi, data, regs);
4016
4017 if (atomic64_add_negative(nr, &hwc->period_left))
4018 return;
4019
4020 perf_swevent_overflow(event, 0, nmi, data, regs);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004021}
4022
Li Zefan6fb29152009-10-15 11:21:42 +08004023static int perf_tp_event_match(struct perf_event *event,
4024 struct perf_sample_data *data);
4025
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004026static int perf_exclude_event(struct perf_event *event,
4027 struct pt_regs *regs)
4028{
4029 if (regs) {
4030 if (event->attr.exclude_user && user_mode(regs))
4031 return 1;
4032
4033 if (event->attr.exclude_kernel && !user_mode(regs))
4034 return 1;
4035 }
4036
4037 return 0;
4038}
4039
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004040static int perf_swevent_match(struct perf_event *event,
4041 enum perf_type_id type,
Li Zefan6fb29152009-10-15 11:21:42 +08004042 u32 event_id,
4043 struct perf_sample_data *data,
4044 struct pt_regs *regs)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004045{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004046 if (event->attr.type != type)
4047 return 0;
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004048
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004049 if (event->attr.config != event_id)
4050 return 0;
4051
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004052 if (perf_exclude_event(event, regs))
4053 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004054
Li Zefan6fb29152009-10-15 11:21:42 +08004055 if (event->attr.type == PERF_TYPE_TRACEPOINT &&
4056 !perf_tp_event_match(event, data))
4057 return 0;
4058
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004059 return 1;
4060}
4061
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004062static inline u64 swevent_hash(u64 type, u32 event_id)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004063{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004064 u64 val = event_id | (type << 32);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004065
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004066 return hash_64(val, SWEVENT_HLIST_BITS);
4067}
4068
4069static struct hlist_head *
4070find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id)
4071{
4072 u64 hash;
4073 struct swevent_hlist *hlist;
4074
4075 hash = swevent_hash(type, event_id);
4076
4077 hlist = rcu_dereference(ctx->swevent_hlist);
4078 if (!hlist)
4079 return NULL;
4080
4081 return &hlist->heads[hash];
4082}
4083
4084static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4085 u64 nr, int nmi,
4086 struct perf_sample_data *data,
4087 struct pt_regs *regs)
4088{
4089 struct perf_cpu_context *cpuctx;
4090 struct perf_event *event;
4091 struct hlist_node *node;
4092 struct hlist_head *head;
4093
4094 cpuctx = &__get_cpu_var(perf_cpu_context);
4095
4096 rcu_read_lock();
4097
4098 head = find_swevent_head(cpuctx, type, event_id);
4099
4100 if (!head)
4101 goto end;
4102
4103 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
Li Zefan6fb29152009-10-15 11:21:42 +08004104 if (perf_swevent_match(event, type, event_id, data, regs))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004105 perf_swevent_add(event, nr, nmi, data, regs);
4106 }
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004107end:
4108 rcu_read_unlock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004109}
4110
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004111int perf_swevent_get_recursion_context(void)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004112{
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004113 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
4114 int rctx;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004115
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004116 if (in_nmi())
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004117 rctx = 3;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004118 else if (in_irq())
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004119 rctx = 2;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004120 else if (in_softirq())
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004121 rctx = 1;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004122 else
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004123 rctx = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004124
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004125 if (cpuctx->recursion[rctx]) {
4126 put_cpu_var(perf_cpu_context);
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004127 return -1;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004128 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004129
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004130 cpuctx->recursion[rctx]++;
4131 barrier();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004132
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004133 return rctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004134}
Ingo Molnar645e8cc2009-11-22 12:20:19 +01004135EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004136
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004137void perf_swevent_put_recursion_context(int rctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004138{
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004139 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4140 barrier();
Frederic Weisbeckerfe612672009-11-24 20:38:22 +01004141 cpuctx->recursion[rctx]--;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004142 put_cpu_var(perf_cpu_context);
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004143}
Ingo Molnar645e8cc2009-11-22 12:20:19 +01004144EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004145
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004146
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004147void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4148 struct pt_regs *regs, u64 addr)
4149{
Ingo Molnara4234bf2009-11-23 10:57:59 +01004150 struct perf_sample_data data;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004151 int rctx;
4152
4153 rctx = perf_swevent_get_recursion_context();
4154 if (rctx < 0)
4155 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004156
Peter Zijlstradc1d6282010-03-03 15:55:04 +01004157 perf_sample_data_init(&data, addr);
Ingo Molnara4234bf2009-11-23 10:57:59 +01004158
4159 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004160
4161 perf_swevent_put_recursion_context(rctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004162}
4163
4164static void perf_swevent_read(struct perf_event *event)
4165{
4166}
4167
4168static int perf_swevent_enable(struct perf_event *event)
4169{
4170 struct hw_perf_event *hwc = &event->hw;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004171 struct perf_cpu_context *cpuctx;
4172 struct hlist_head *head;
4173
4174 cpuctx = &__get_cpu_var(perf_cpu_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004175
4176 if (hwc->sample_period) {
4177 hwc->last_period = hwc->sample_period;
4178 perf_swevent_set_period(event);
4179 }
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004180
4181 head = find_swevent_head(cpuctx, event->attr.type, event->attr.config);
4182 if (WARN_ON_ONCE(!head))
4183 return -EINVAL;
4184
4185 hlist_add_head_rcu(&event->hlist_entry, head);
4186
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004187 return 0;
4188}
4189
4190static void perf_swevent_disable(struct perf_event *event)
4191{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004192 hlist_del_rcu(&event->hlist_entry);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004193}
4194
4195static const struct pmu perf_ops_generic = {
4196 .enable = perf_swevent_enable,
4197 .disable = perf_swevent_disable,
4198 .read = perf_swevent_read,
4199 .unthrottle = perf_swevent_unthrottle,
4200};
4201
4202/*
4203 * hrtimer based swevent callback
4204 */
4205
4206static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4207{
4208 enum hrtimer_restart ret = HRTIMER_RESTART;
4209 struct perf_sample_data data;
4210 struct pt_regs *regs;
4211 struct perf_event *event;
4212 u64 period;
4213
Peter Zijlstradc1d6282010-03-03 15:55:04 +01004214 event = container_of(hrtimer, struct perf_event, hw.hrtimer);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004215 event->pmu->read(event);
4216
Peter Zijlstradc1d6282010-03-03 15:55:04 +01004217 perf_sample_data_init(&data, 0);
Xiao Guangrong59d069e2009-12-01 17:30:08 +08004218 data.period = event->hw.last_period;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004219 regs = get_irq_regs();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004220
Frederic Weisbeckerdf8290b2010-04-09 00:28:14 +02004221 if (regs && !perf_exclude_event(event, regs)) {
Soeren Sandmann54f44072009-10-22 18:34:08 +02004222 if (!(event->attr.exclude_idle && current->pid == 0))
4223 if (perf_event_overflow(event, 0, &data, regs))
4224 ret = HRTIMER_NORESTART;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004225 }
4226
4227 period = max_t(u64, 10000, event->hw.sample_period);
4228 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
4229
4230 return ret;
4231}
4232
Soeren Sandmann721a6692009-09-15 14:33:08 +02004233static void perf_swevent_start_hrtimer(struct perf_event *event)
4234{
4235 struct hw_perf_event *hwc = &event->hw;
4236
4237 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
4238 hwc->hrtimer.function = perf_swevent_hrtimer;
4239 if (hwc->sample_period) {
4240 u64 period;
4241
4242 if (hwc->remaining) {
4243 if (hwc->remaining < 0)
4244 period = 10000;
4245 else
4246 period = hwc->remaining;
4247 hwc->remaining = 0;
4248 } else {
4249 period = max_t(u64, 10000, hwc->sample_period);
4250 }
4251 __hrtimer_start_range_ns(&hwc->hrtimer,
4252 ns_to_ktime(period), 0,
4253 HRTIMER_MODE_REL, 0);
4254 }
4255}
4256
4257static void perf_swevent_cancel_hrtimer(struct perf_event *event)
4258{
4259 struct hw_perf_event *hwc = &event->hw;
4260
4261 if (hwc->sample_period) {
4262 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
4263 hwc->remaining = ktime_to_ns(remaining);
4264
4265 hrtimer_cancel(&hwc->hrtimer);
4266 }
4267}
4268
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004269/*
4270 * Software event: cpu wall time clock
4271 */
4272
4273static void cpu_clock_perf_event_update(struct perf_event *event)
4274{
4275 int cpu = raw_smp_processor_id();
4276 s64 prev;
4277 u64 now;
4278
4279 now = cpu_clock(cpu);
Xiao Guangrongec89a06f2009-12-09 11:30:36 +08004280 prev = atomic64_xchg(&event->hw.prev_count, now);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004281 atomic64_add(now - prev, &event->count);
4282}
4283
4284static int cpu_clock_perf_event_enable(struct perf_event *event)
4285{
4286 struct hw_perf_event *hwc = &event->hw;
4287 int cpu = raw_smp_processor_id();
4288
4289 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
Soeren Sandmann721a6692009-09-15 14:33:08 +02004290 perf_swevent_start_hrtimer(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004291
4292 return 0;
4293}
4294
4295static void cpu_clock_perf_event_disable(struct perf_event *event)
4296{
Soeren Sandmann721a6692009-09-15 14:33:08 +02004297 perf_swevent_cancel_hrtimer(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004298 cpu_clock_perf_event_update(event);
4299}
4300
4301static void cpu_clock_perf_event_read(struct perf_event *event)
4302{
4303 cpu_clock_perf_event_update(event);
4304}
4305
4306static const struct pmu perf_ops_cpu_clock = {
4307 .enable = cpu_clock_perf_event_enable,
4308 .disable = cpu_clock_perf_event_disable,
4309 .read = cpu_clock_perf_event_read,
4310};
4311
4312/*
4313 * Software event: task time clock
4314 */
4315
4316static void task_clock_perf_event_update(struct perf_event *event, u64 now)
4317{
4318 u64 prev;
4319 s64 delta;
4320
4321 prev = atomic64_xchg(&event->hw.prev_count, now);
4322 delta = now - prev;
4323 atomic64_add(delta, &event->count);
4324}
4325
4326static int task_clock_perf_event_enable(struct perf_event *event)
4327{
4328 struct hw_perf_event *hwc = &event->hw;
4329 u64 now;
4330
4331 now = event->ctx->time;
4332
4333 atomic64_set(&hwc->prev_count, now);
Soeren Sandmann721a6692009-09-15 14:33:08 +02004334
4335 perf_swevent_start_hrtimer(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004336
4337 return 0;
4338}
4339
4340static void task_clock_perf_event_disable(struct perf_event *event)
4341{
Soeren Sandmann721a6692009-09-15 14:33:08 +02004342 perf_swevent_cancel_hrtimer(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004343 task_clock_perf_event_update(event, event->ctx->time);
4344
4345}
4346
4347static void task_clock_perf_event_read(struct perf_event *event)
4348{
4349 u64 time;
4350
4351 if (!in_nmi()) {
4352 update_context_time(event->ctx);
4353 time = event->ctx->time;
4354 } else {
4355 u64 now = perf_clock();
4356 u64 delta = now - event->ctx->timestamp;
4357 time = event->ctx->time + delta;
4358 }
4359
4360 task_clock_perf_event_update(event, time);
4361}
4362
4363static const struct pmu perf_ops_task_clock = {
4364 .enable = task_clock_perf_event_enable,
4365 .disable = task_clock_perf_event_disable,
4366 .read = task_clock_perf_event_read,
4367};
4368
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004369static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4370{
4371 struct swevent_hlist *hlist;
4372
4373 hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
4374 kfree(hlist);
4375}
4376
4377static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
4378{
4379 struct swevent_hlist *hlist;
4380
4381 if (!cpuctx->swevent_hlist)
4382 return;
4383
4384 hlist = cpuctx->swevent_hlist;
4385 rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
4386 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4387}
4388
4389static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4390{
4391 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4392
4393 mutex_lock(&cpuctx->hlist_mutex);
4394
4395 if (!--cpuctx->hlist_refcount)
4396 swevent_hlist_release(cpuctx);
4397
4398 mutex_unlock(&cpuctx->hlist_mutex);
4399}
4400
4401static void swevent_hlist_put(struct perf_event *event)
4402{
4403 int cpu;
4404
4405 if (event->cpu != -1) {
4406 swevent_hlist_put_cpu(event, event->cpu);
4407 return;
4408 }
4409
4410 for_each_possible_cpu(cpu)
4411 swevent_hlist_put_cpu(event, cpu);
4412}
4413
4414static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4415{
4416 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4417 int err = 0;
4418
4419 mutex_lock(&cpuctx->hlist_mutex);
4420
4421 if (!cpuctx->swevent_hlist && cpu_online(cpu)) {
4422 struct swevent_hlist *hlist;
4423
4424 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4425 if (!hlist) {
4426 err = -ENOMEM;
4427 goto exit;
4428 }
4429 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
4430 }
4431 cpuctx->hlist_refcount++;
4432 exit:
4433 mutex_unlock(&cpuctx->hlist_mutex);
4434
4435 return err;
4436}
4437
4438static int swevent_hlist_get(struct perf_event *event)
4439{
4440 int err;
4441 int cpu, failed_cpu;
4442
4443 if (event->cpu != -1)
4444 return swevent_hlist_get_cpu(event, event->cpu);
4445
4446 get_online_cpus();
4447 for_each_possible_cpu(cpu) {
4448 err = swevent_hlist_get_cpu(event, cpu);
4449 if (err) {
4450 failed_cpu = cpu;
4451 goto fail;
4452 }
4453 }
4454 put_online_cpus();
4455
4456 return 0;
4457 fail:
4458 for_each_possible_cpu(cpu) {
4459 if (cpu == failed_cpu)
4460 break;
4461 swevent_hlist_put_cpu(event, cpu);
4462 }
4463
4464 put_online_cpus();
4465 return err;
4466}
4467
Frederic Weisbecker95476b62010-04-14 23:42:18 +02004468#ifdef CONFIG_EVENT_TRACING
4469
4470void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4471 int entry_size, struct pt_regs *regs)
4472{
4473 struct perf_sample_data data;
4474 struct perf_raw_record raw = {
4475 .size = entry_size,
4476 .data = record,
4477 };
4478
4479 perf_sample_data_init(&data, addr);
4480 data.raw = &raw;
4481
4482 /* Trace events already protected against recursion */
4483 do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4484 &data, regs);
4485}
4486EXPORT_SYMBOL_GPL(perf_tp_event);
4487
4488static int perf_tp_event_match(struct perf_event *event,
4489 struct perf_sample_data *data)
4490{
4491 void *record = data->raw->data;
4492
4493 if (likely(!event->filter) || filter_match_preds(event->filter, record))
4494 return 1;
4495 return 0;
4496}
4497
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004498static void tp_perf_event_destroy(struct perf_event *event)
4499{
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01004500 perf_trace_disable(event->attr.config);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004501 swevent_hlist_put(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004502}
4503
4504static const struct pmu *tp_perf_event_init(struct perf_event *event)
4505{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004506 int err;
4507
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004508 /*
4509 * Raw tracepoint data is a severe data leak, only allow root to
4510 * have these.
4511 */
4512 if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4513 perf_paranoid_tracepoint_raw() &&
4514 !capable(CAP_SYS_ADMIN))
4515 return ERR_PTR(-EPERM);
4516
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01004517 if (perf_trace_enable(event->attr.config))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004518 return NULL;
4519
4520 event->destroy = tp_perf_event_destroy;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004521 err = swevent_hlist_get(event);
4522 if (err) {
4523 perf_trace_disable(event->attr.config);
4524 return ERR_PTR(err);
4525 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004526
4527 return &perf_ops_generic;
4528}
Li Zefan6fb29152009-10-15 11:21:42 +08004529
4530static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4531{
4532 char *filter_str;
4533 int ret;
4534
4535 if (event->attr.type != PERF_TYPE_TRACEPOINT)
4536 return -EINVAL;
4537
4538 filter_str = strndup_user(arg, PAGE_SIZE);
4539 if (IS_ERR(filter_str))
4540 return PTR_ERR(filter_str);
4541
4542 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
4543
4544 kfree(filter_str);
4545 return ret;
4546}
4547
4548static void perf_event_free_filter(struct perf_event *event)
4549{
4550 ftrace_profile_free_filter(event);
4551}
4552
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004553#else
Li Zefan6fb29152009-10-15 11:21:42 +08004554
4555static int perf_tp_event_match(struct perf_event *event,
4556 struct perf_sample_data *data)
4557{
4558 return 1;
4559}
4560
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004561static const struct pmu *tp_perf_event_init(struct perf_event *event)
4562{
4563 return NULL;
4564}
Li Zefan6fb29152009-10-15 11:21:42 +08004565
4566static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4567{
4568 return -ENOENT;
4569}
4570
4571static void perf_event_free_filter(struct perf_event *event)
4572{
4573}
4574
Li Zefan07b139c2009-12-21 14:27:35 +08004575#endif /* CONFIG_EVENT_TRACING */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004576
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004577#ifdef CONFIG_HAVE_HW_BREAKPOINT
4578static void bp_perf_event_destroy(struct perf_event *event)
4579{
4580 release_bp_slot(event);
4581}
4582
4583static const struct pmu *bp_perf_event_init(struct perf_event *bp)
4584{
4585 int err;
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01004586
4587 err = register_perf_hw_breakpoint(bp);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004588 if (err)
4589 return ERR_PTR(err);
4590
4591 bp->destroy = bp_perf_event_destroy;
4592
4593 return &perf_ops_bp;
4594}
4595
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004596void perf_bp_event(struct perf_event *bp, void *data)
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004597{
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004598 struct perf_sample_data sample;
4599 struct pt_regs *regs = data;
4600
Peter Zijlstradc1d6282010-03-03 15:55:04 +01004601 perf_sample_data_init(&sample, bp->attr.bp_addr);
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004602
4603 if (!perf_exclude_event(bp, regs))
4604 perf_swevent_add(bp, 1, 1, &sample, regs);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004605}
4606#else
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004607static const struct pmu *bp_perf_event_init(struct perf_event *bp)
4608{
4609 return NULL;
4610}
4611
4612void perf_bp_event(struct perf_event *bp, void *regs)
4613{
4614}
4615#endif
4616
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004617atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4618
4619static void sw_perf_event_destroy(struct perf_event *event)
4620{
4621 u64 event_id = event->attr.config;
4622
4623 WARN_ON(event->parent);
4624
4625 atomic_dec(&perf_swevent_enabled[event_id]);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004626 swevent_hlist_put(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004627}
4628
4629static const struct pmu *sw_perf_event_init(struct perf_event *event)
4630{
4631 const struct pmu *pmu = NULL;
4632 u64 event_id = event->attr.config;
4633
4634 /*
4635 * Software events (currently) can't in general distinguish
4636 * between user, kernel and hypervisor events.
4637 * However, context switches and cpu migrations are considered
4638 * to be kernel events, and page faults are never hypervisor
4639 * events.
4640 */
4641 switch (event_id) {
4642 case PERF_COUNT_SW_CPU_CLOCK:
4643 pmu = &perf_ops_cpu_clock;
4644
4645 break;
4646 case PERF_COUNT_SW_TASK_CLOCK:
4647 /*
4648 * If the user instantiates this as a per-cpu event,
4649 * use the cpu_clock event instead.
4650 */
4651 if (event->ctx->task)
4652 pmu = &perf_ops_task_clock;
4653 else
4654 pmu = &perf_ops_cpu_clock;
4655
4656 break;
4657 case PERF_COUNT_SW_PAGE_FAULTS:
4658 case PERF_COUNT_SW_PAGE_FAULTS_MIN:
4659 case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
4660 case PERF_COUNT_SW_CONTEXT_SWITCHES:
4661 case PERF_COUNT_SW_CPU_MIGRATIONS:
Anton Blanchardf7d79862009-10-18 01:09:29 +00004662 case PERF_COUNT_SW_ALIGNMENT_FAULTS:
4663 case PERF_COUNT_SW_EMULATION_FAULTS:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004664 if (!event->parent) {
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004665 int err;
4666
4667 err = swevent_hlist_get(event);
4668 if (err)
4669 return ERR_PTR(err);
4670
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004671 atomic_inc(&perf_swevent_enabled[event_id]);
4672 event->destroy = sw_perf_event_destroy;
4673 }
4674 pmu = &perf_ops_generic;
4675 break;
4676 }
4677
4678 return pmu;
4679}
4680
4681/*
4682 * Allocate and initialize a event structure
4683 */
4684static struct perf_event *
4685perf_event_alloc(struct perf_event_attr *attr,
4686 int cpu,
4687 struct perf_event_context *ctx,
4688 struct perf_event *group_leader,
4689 struct perf_event *parent_event,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01004690 perf_overflow_handler_t overflow_handler,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004691 gfp_t gfpflags)
4692{
4693 const struct pmu *pmu;
4694 struct perf_event *event;
4695 struct hw_perf_event *hwc;
4696 long err;
4697
4698 event = kzalloc(sizeof(*event), gfpflags);
4699 if (!event)
4700 return ERR_PTR(-ENOMEM);
4701
4702 /*
4703 * Single events are their own group leaders, with an
4704 * empty sibling list:
4705 */
4706 if (!group_leader)
4707 group_leader = event;
4708
4709 mutex_init(&event->child_mutex);
4710 INIT_LIST_HEAD(&event->child_list);
4711
4712 INIT_LIST_HEAD(&event->group_entry);
4713 INIT_LIST_HEAD(&event->event_entry);
4714 INIT_LIST_HEAD(&event->sibling_list);
4715 init_waitqueue_head(&event->waitq);
4716
4717 mutex_init(&event->mmap_mutex);
4718
4719 event->cpu = cpu;
4720 event->attr = *attr;
4721 event->group_leader = group_leader;
4722 event->pmu = NULL;
4723 event->ctx = ctx;
4724 event->oncpu = -1;
4725
4726 event->parent = parent_event;
4727
4728 event->ns = get_pid_ns(current->nsproxy->pid_ns);
4729 event->id = atomic64_inc_return(&perf_event_id);
4730
4731 event->state = PERF_EVENT_STATE_INACTIVE;
4732
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01004733 if (!overflow_handler && parent_event)
4734 overflow_handler = parent_event->overflow_handler;
Frederic Weisbecker97eaf532009-10-18 15:33:50 +02004735
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01004736 event->overflow_handler = overflow_handler;
Frederic Weisbecker97eaf532009-10-18 15:33:50 +02004737
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004738 if (attr->disabled)
4739 event->state = PERF_EVENT_STATE_OFF;
4740
4741 pmu = NULL;
4742
4743 hwc = &event->hw;
4744 hwc->sample_period = attr->sample_period;
4745 if (attr->freq && attr->sample_freq)
4746 hwc->sample_period = 1;
4747 hwc->last_period = hwc->sample_period;
4748
4749 atomic64_set(&hwc->period_left, hwc->sample_period);
4750
4751 /*
4752 * we currently do not support PERF_FORMAT_GROUP on inherited events
4753 */
4754 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4755 goto done;
4756
4757 switch (attr->type) {
4758 case PERF_TYPE_RAW:
4759 case PERF_TYPE_HARDWARE:
4760 case PERF_TYPE_HW_CACHE:
4761 pmu = hw_perf_event_init(event);
4762 break;
4763
4764 case PERF_TYPE_SOFTWARE:
4765 pmu = sw_perf_event_init(event);
4766 break;
4767
4768 case PERF_TYPE_TRACEPOINT:
4769 pmu = tp_perf_event_init(event);
4770 break;
4771
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02004772 case PERF_TYPE_BREAKPOINT:
4773 pmu = bp_perf_event_init(event);
4774 break;
4775
4776
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004777 default:
4778 break;
4779 }
4780done:
4781 err = 0;
4782 if (!pmu)
4783 err = -EINVAL;
4784 else if (IS_ERR(pmu))
4785 err = PTR_ERR(pmu);
4786
4787 if (err) {
4788 if (event->ns)
4789 put_pid_ns(event->ns);
4790 kfree(event);
4791 return ERR_PTR(err);
4792 }
4793
4794 event->pmu = pmu;
4795
4796 if (!event->parent) {
4797 atomic_inc(&nr_events);
4798 if (event->attr.mmap)
4799 atomic_inc(&nr_mmap_events);
4800 if (event->attr.comm)
4801 atomic_inc(&nr_comm_events);
4802 if (event->attr.task)
4803 atomic_inc(&nr_task_events);
4804 }
4805
4806 return event;
4807}
4808
4809static int perf_copy_attr(struct perf_event_attr __user *uattr,
4810 struct perf_event_attr *attr)
4811{
4812 u32 size;
4813 int ret;
4814
4815 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
4816 return -EFAULT;
4817
4818 /*
4819 * zero the full structure, so that a short copy will be nice.
4820 */
4821 memset(attr, 0, sizeof(*attr));
4822
4823 ret = get_user(size, &uattr->size);
4824 if (ret)
4825 return ret;
4826
4827 if (size > PAGE_SIZE) /* silly large */
4828 goto err_size;
4829
4830 if (!size) /* abi compat */
4831 size = PERF_ATTR_SIZE_VER0;
4832
4833 if (size < PERF_ATTR_SIZE_VER0)
4834 goto err_size;
4835
4836 /*
4837 * If we're handed a bigger struct than we know of,
4838 * ensure all the unknown bits are 0 - i.e. new
4839 * user-space does not rely on any kernel feature
4840 * extensions we dont know about yet.
4841 */
4842 if (size > sizeof(*attr)) {
4843 unsigned char __user *addr;
4844 unsigned char __user *end;
4845 unsigned char val;
4846
4847 addr = (void __user *)uattr + sizeof(*attr);
4848 end = (void __user *)uattr + size;
4849
4850 for (; addr < end; addr++) {
4851 ret = get_user(val, addr);
4852 if (ret)
4853 return ret;
4854 if (val)
4855 goto err_size;
4856 }
4857 size = sizeof(*attr);
4858 }
4859
4860 ret = copy_from_user(attr, uattr, size);
4861 if (ret)
4862 return -EFAULT;
4863
4864 /*
4865 * If the type exists, the corresponding creation will verify
4866 * the attr->config.
4867 */
4868 if (attr->type >= PERF_TYPE_MAX)
4869 return -EINVAL;
4870
Mahesh Salgaonkarcd757642010-01-30 10:25:18 +05304871 if (attr->__reserved_1)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004872 return -EINVAL;
4873
4874 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
4875 return -EINVAL;
4876
4877 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
4878 return -EINVAL;
4879
4880out:
4881 return ret;
4882
4883err_size:
4884 put_user(sizeof(*attr), &uattr->size);
4885 ret = -E2BIG;
4886 goto out;
4887}
4888
Li Zefan6fb29152009-10-15 11:21:42 +08004889static int perf_event_set_output(struct perf_event *event, int output_fd)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004890{
4891 struct perf_event *output_event = NULL;
4892 struct file *output_file = NULL;
4893 struct perf_event *old_output;
4894 int fput_needed = 0;
4895 int ret = -EINVAL;
4896
4897 if (!output_fd)
4898 goto set;
4899
4900 output_file = fget_light(output_fd, &fput_needed);
4901 if (!output_file)
4902 return -EBADF;
4903
4904 if (output_file->f_op != &perf_fops)
4905 goto out;
4906
4907 output_event = output_file->private_data;
4908
4909 /* Don't chain output fds */
4910 if (output_event->output)
4911 goto out;
4912
4913 /* Don't set an output fd when we already have an output channel */
4914 if (event->data)
4915 goto out;
4916
4917 atomic_long_inc(&output_file->f_count);
4918
4919set:
4920 mutex_lock(&event->mmap_mutex);
4921 old_output = event->output;
4922 rcu_assign_pointer(event->output, output_event);
4923 mutex_unlock(&event->mmap_mutex);
4924
4925 if (old_output) {
4926 /*
4927 * we need to make sure no existing perf_output_*()
4928 * is still referencing this event.
4929 */
4930 synchronize_rcu();
4931 fput(old_output->filp);
4932 }
4933
4934 ret = 0;
4935out:
4936 fput_light(output_file, fput_needed);
4937 return ret;
4938}
4939
4940/**
4941 * sys_perf_event_open - open a performance event, associate it to a task/cpu
4942 *
4943 * @attr_uptr: event_id type attributes for monitoring/sampling
4944 * @pid: target pid
4945 * @cpu: target cpu
4946 * @group_fd: group leader event fd
4947 */
4948SYSCALL_DEFINE5(perf_event_open,
4949 struct perf_event_attr __user *, attr_uptr,
4950 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
4951{
4952 struct perf_event *event, *group_leader;
4953 struct perf_event_attr attr;
4954 struct perf_event_context *ctx;
4955 struct file *event_file = NULL;
4956 struct file *group_file = NULL;
4957 int fput_needed = 0;
4958 int fput_needed2 = 0;
4959 int err;
4960
4961 /* for future expandability... */
4962 if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4963 return -EINVAL;
4964
4965 err = perf_copy_attr(attr_uptr, &attr);
4966 if (err)
4967 return err;
4968
4969 if (!attr.exclude_kernel) {
4970 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
4971 return -EACCES;
4972 }
4973
4974 if (attr.freq) {
4975 if (attr.sample_freq > sysctl_perf_event_sample_rate)
4976 return -EINVAL;
4977 }
4978
4979 /*
4980 * Get the target context (task or percpu):
4981 */
4982 ctx = find_get_context(pid, cpu);
4983 if (IS_ERR(ctx))
4984 return PTR_ERR(ctx);
4985
4986 /*
4987 * Look up the group leader (we will attach this event to it):
4988 */
4989 group_leader = NULL;
4990 if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4991 err = -EINVAL;
4992 group_file = fget_light(group_fd, &fput_needed);
4993 if (!group_file)
4994 goto err_put_context;
4995 if (group_file->f_op != &perf_fops)
4996 goto err_put_context;
4997
4998 group_leader = group_file->private_data;
4999 /*
5000 * Do not allow a recursive hierarchy (this new sibling
5001 * becoming part of another group-sibling):
5002 */
5003 if (group_leader->group_leader != group_leader)
5004 goto err_put_context;
5005 /*
5006 * Do not allow to attach to a group in a different
5007 * task or CPU context:
5008 */
5009 if (group_leader->ctx != ctx)
5010 goto err_put_context;
5011 /*
5012 * Only a group leader can be exclusive or pinned
5013 */
5014 if (attr.exclusive || attr.pinned)
5015 goto err_put_context;
5016 }
5017
5018 event = perf_event_alloc(&attr, cpu, ctx, group_leader,
Frederic Weisbecker97eaf532009-10-18 15:33:50 +02005019 NULL, NULL, GFP_KERNEL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005020 err = PTR_ERR(event);
5021 if (IS_ERR(event))
5022 goto err_put_context;
5023
Roland Dreier628ff7c2009-12-18 09:41:24 -08005024 err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005025 if (err < 0)
5026 goto err_free_put_context;
5027
5028 event_file = fget_light(err, &fput_needed2);
5029 if (!event_file)
5030 goto err_free_put_context;
5031
5032 if (flags & PERF_FLAG_FD_OUTPUT) {
5033 err = perf_event_set_output(event, group_fd);
5034 if (err)
5035 goto err_fput_free_put_context;
5036 }
5037
5038 event->filp = event_file;
5039 WARN_ON_ONCE(ctx->parent_ctx);
5040 mutex_lock(&ctx->mutex);
5041 perf_install_in_context(ctx, event, cpu);
5042 ++ctx->generation;
5043 mutex_unlock(&ctx->mutex);
5044
5045 event->owner = current;
5046 get_task_struct(current);
5047 mutex_lock(&current->perf_event_mutex);
5048 list_add_tail(&event->owner_entry, &current->perf_event_list);
5049 mutex_unlock(&current->perf_event_mutex);
5050
5051err_fput_free_put_context:
5052 fput_light(event_file, fput_needed2);
5053
5054err_free_put_context:
5055 if (err < 0)
Tejun Heo048c8522010-05-01 10:11:35 +02005056 free_event(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005057
5058err_put_context:
5059 if (err < 0)
5060 put_ctx(ctx);
5061
5062 fput_light(group_file, fput_needed);
5063
5064 return err;
5065}
5066
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005067/**
5068 * perf_event_create_kernel_counter
5069 *
5070 * @attr: attributes of the counter to create
5071 * @cpu: cpu in which the counter is bound
5072 * @pid: task to profile
5073 */
5074struct perf_event *
5075perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01005076 pid_t pid,
5077 perf_overflow_handler_t overflow_handler)
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005078{
5079 struct perf_event *event;
5080 struct perf_event_context *ctx;
5081 int err;
5082
5083 /*
5084 * Get the target context (task or percpu):
5085 */
5086
5087 ctx = find_get_context(pid, cpu);
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01005088 if (IS_ERR(ctx)) {
5089 err = PTR_ERR(ctx);
5090 goto err_exit;
5091 }
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005092
5093 event = perf_event_alloc(attr, cpu, ctx, NULL,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01005094 NULL, overflow_handler, GFP_KERNEL);
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01005095 if (IS_ERR(event)) {
5096 err = PTR_ERR(event);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005097 goto err_put_context;
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01005098 }
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005099
5100 event->filp = NULL;
5101 WARN_ON_ONCE(ctx->parent_ctx);
5102 mutex_lock(&ctx->mutex);
5103 perf_install_in_context(ctx, event, cpu);
5104 ++ctx->generation;
5105 mutex_unlock(&ctx->mutex);
5106
5107 event->owner = current;
5108 get_task_struct(current);
5109 mutex_lock(&current->perf_event_mutex);
5110 list_add_tail(&event->owner_entry, &current->perf_event_list);
5111 mutex_unlock(&current->perf_event_mutex);
5112
5113 return event;
5114
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01005115 err_put_context:
5116 put_ctx(ctx);
5117 err_exit:
5118 return ERR_PTR(err);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02005119}
5120EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
5121
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005122/*
5123 * inherit a event from parent task to child task:
5124 */
5125static struct perf_event *
5126inherit_event(struct perf_event *parent_event,
5127 struct task_struct *parent,
5128 struct perf_event_context *parent_ctx,
5129 struct task_struct *child,
5130 struct perf_event *group_leader,
5131 struct perf_event_context *child_ctx)
5132{
5133 struct perf_event *child_event;
5134
5135 /*
5136 * Instead of creating recursive hierarchies of events,
5137 * we link inherited events back to the original parent,
5138 * which has a filp for sure, which we use as the reference
5139 * count:
5140 */
5141 if (parent_event->parent)
5142 parent_event = parent_event->parent;
5143
5144 child_event = perf_event_alloc(&parent_event->attr,
5145 parent_event->cpu, child_ctx,
5146 group_leader, parent_event,
Frederic Weisbecker97eaf532009-10-18 15:33:50 +02005147 NULL, GFP_KERNEL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005148 if (IS_ERR(child_event))
5149 return child_event;
5150 get_ctx(child_ctx);
5151
5152 /*
5153 * Make the child state follow the state of the parent event,
5154 * not its attr.disabled bit. We hold the parent's mutex,
5155 * so we won't race with perf_event_{en, dis}able_family.
5156 */
5157 if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
5158 child_event->state = PERF_EVENT_STATE_INACTIVE;
5159 else
5160 child_event->state = PERF_EVENT_STATE_OFF;
5161
Peter Zijlstra75c9f322010-01-29 09:04:26 +01005162 if (parent_event->attr.freq) {
5163 u64 sample_period = parent_event->hw.sample_period;
5164 struct hw_perf_event *hwc = &child_event->hw;
5165
5166 hwc->sample_period = sample_period;
5167 hwc->last_period = sample_period;
5168
5169 atomic64_set(&hwc->period_left, sample_period);
5170 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005171
Peter Zijlstra453f19e2009-11-20 22:19:43 +01005172 child_event->overflow_handler = parent_event->overflow_handler;
5173
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005174 /*
5175 * Link it up in the child's context:
5176 */
5177 add_event_to_ctx(child_event, child_ctx);
5178
5179 /*
5180 * Get a reference to the parent filp - we will fput it
5181 * when the child event exits. This is safe to do because
5182 * we are in the parent and we know that the filp still
5183 * exists and has a nonzero count:
5184 */
5185 atomic_long_inc(&parent_event->filp->f_count);
5186
5187 /*
5188 * Link this into the parent event's child list
5189 */
5190 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5191 mutex_lock(&parent_event->child_mutex);
5192 list_add_tail(&child_event->child_list, &parent_event->child_list);
5193 mutex_unlock(&parent_event->child_mutex);
5194
5195 return child_event;
5196}
5197
5198static int inherit_group(struct perf_event *parent_event,
5199 struct task_struct *parent,
5200 struct perf_event_context *parent_ctx,
5201 struct task_struct *child,
5202 struct perf_event_context *child_ctx)
5203{
5204 struct perf_event *leader;
5205 struct perf_event *sub;
5206 struct perf_event *child_ctr;
5207
5208 leader = inherit_event(parent_event, parent, parent_ctx,
5209 child, NULL, child_ctx);
5210 if (IS_ERR(leader))
5211 return PTR_ERR(leader);
5212 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
5213 child_ctr = inherit_event(sub, parent, parent_ctx,
5214 child, leader, child_ctx);
5215 if (IS_ERR(child_ctr))
5216 return PTR_ERR(child_ctr);
5217 }
5218 return 0;
5219}
5220
5221static void sync_child_event(struct perf_event *child_event,
5222 struct task_struct *child)
5223{
5224 struct perf_event *parent_event = child_event->parent;
5225 u64 child_val;
5226
5227 if (child_event->attr.inherit_stat)
5228 perf_event_read_event(child_event, child);
5229
5230 child_val = atomic64_read(&child_event->count);
5231
5232 /*
5233 * Add back the child's count to the parent's count:
5234 */
5235 atomic64_add(child_val, &parent_event->count);
5236 atomic64_add(child_event->total_time_enabled,
5237 &parent_event->child_total_time_enabled);
5238 atomic64_add(child_event->total_time_running,
5239 &parent_event->child_total_time_running);
5240
5241 /*
5242 * Remove this event from the parent's list
5243 */
5244 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5245 mutex_lock(&parent_event->child_mutex);
5246 list_del_init(&child_event->child_list);
5247 mutex_unlock(&parent_event->child_mutex);
5248
5249 /*
5250 * Release the parent event, if this was the last
5251 * reference to it.
5252 */
5253 fput(parent_event->filp);
5254}
5255
5256static void
5257__perf_event_exit_task(struct perf_event *child_event,
5258 struct perf_event_context *child_ctx,
5259 struct task_struct *child)
5260{
5261 struct perf_event *parent_event;
5262
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005263 perf_event_remove_from_context(child_event);
5264
5265 parent_event = child_event->parent;
5266 /*
5267 * It can happen that parent exits first, and has events
5268 * that are still around due to the child reference. These
5269 * events need to be zapped - but otherwise linger.
5270 */
5271 if (parent_event) {
5272 sync_child_event(child_event, child);
5273 free_event(child_event);
5274 }
5275}
5276
5277/*
5278 * When a child task exits, feed back event values to parent events.
5279 */
5280void perf_event_exit_task(struct task_struct *child)
5281{
5282 struct perf_event *child_event, *tmp;
5283 struct perf_event_context *child_ctx;
5284 unsigned long flags;
5285
5286 if (likely(!child->perf_event_ctxp)) {
5287 perf_event_task(child, NULL, 0);
5288 return;
5289 }
5290
5291 local_irq_save(flags);
5292 /*
5293 * We can't reschedule here because interrupts are disabled,
5294 * and either child is current or it is a task that can't be
5295 * scheduled, so we are now safe from rescheduling changing
5296 * our context.
5297 */
5298 child_ctx = child->perf_event_ctxp;
5299 __perf_event_task_sched_out(child_ctx);
5300
5301 /*
5302 * Take the context lock here so that if find_get_context is
5303 * reading child->perf_event_ctxp, we wait until it has
5304 * incremented the context's refcount before we do put_ctx below.
5305 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +01005306 raw_spin_lock(&child_ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005307 child->perf_event_ctxp = NULL;
5308 /*
5309 * If this context is a clone; unclone it so it can't get
5310 * swapped to another process while we're removing all
5311 * the events from it.
5312 */
5313 unclone_ctx(child_ctx);
Peter Zijlstra5e942bb2009-11-23 11:37:26 +01005314 update_context_time(child_ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01005315 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005316
5317 /*
5318 * Report the task dead after unscheduling the events so that we
5319 * won't get any samples after PERF_RECORD_EXIT. We can however still
5320 * get a few PERF_RECORD_READ events.
5321 */
5322 perf_event_task(child, child_ctx, 0);
5323
5324 /*
5325 * We can recurse on the same lock type through:
5326 *
5327 * __perf_event_exit_task()
5328 * sync_child_event()
5329 * fput(parent_event->filp)
5330 * perf_release()
5331 * mutex_lock(&ctx->mutex)
5332 *
5333 * But since its the parent context it won't be the same instance.
5334 */
Peter Zijlstraa0507c82010-05-06 15:42:53 +02005335 mutex_lock(&child_ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005336
5337again:
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005338 list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
5339 group_entry)
5340 __perf_event_exit_task(child_event, child_ctx, child);
5341
5342 list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005343 group_entry)
5344 __perf_event_exit_task(child_event, child_ctx, child);
5345
5346 /*
5347 * If the last event was a group event, it will have appended all
5348 * its siblings to the list, but we obtained 'tmp' before that which
5349 * will still point to the list head terminating the iteration.
5350 */
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005351 if (!list_empty(&child_ctx->pinned_groups) ||
5352 !list_empty(&child_ctx->flexible_groups))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005353 goto again;
5354
5355 mutex_unlock(&child_ctx->mutex);
5356
5357 put_ctx(child_ctx);
5358}
5359
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005360static void perf_free_event(struct perf_event *event,
5361 struct perf_event_context *ctx)
5362{
5363 struct perf_event *parent = event->parent;
5364
5365 if (WARN_ON_ONCE(!parent))
5366 return;
5367
5368 mutex_lock(&parent->child_mutex);
5369 list_del_init(&event->child_list);
5370 mutex_unlock(&parent->child_mutex);
5371
5372 fput(parent->filp);
5373
5374 list_del_event(event, ctx);
5375 free_event(event);
5376}
5377
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005378/*
5379 * free an unexposed, unused context as created by inheritance by
5380 * init_task below, used by fork() in case of fail.
5381 */
5382void perf_event_free_task(struct task_struct *task)
5383{
5384 struct perf_event_context *ctx = task->perf_event_ctxp;
5385 struct perf_event *event, *tmp;
5386
5387 if (!ctx)
5388 return;
5389
5390 mutex_lock(&ctx->mutex);
5391again:
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005392 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5393 perf_free_event(event, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005394
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005395 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
5396 group_entry)
5397 perf_free_event(event, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005398
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005399 if (!list_empty(&ctx->pinned_groups) ||
5400 !list_empty(&ctx->flexible_groups))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005401 goto again;
5402
5403 mutex_unlock(&ctx->mutex);
5404
5405 put_ctx(ctx);
5406}
5407
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005408static int
5409inherit_task_group(struct perf_event *event, struct task_struct *parent,
5410 struct perf_event_context *parent_ctx,
5411 struct task_struct *child,
5412 int *inherited_all)
5413{
5414 int ret;
5415 struct perf_event_context *child_ctx = child->perf_event_ctxp;
5416
5417 if (!event->attr.inherit) {
5418 *inherited_all = 0;
5419 return 0;
5420 }
5421
5422 if (!child_ctx) {
5423 /*
5424 * This is executed from the parent task context, so
5425 * inherit events that have been marked for cloning.
5426 * First allocate and initialize a context for the
5427 * child.
5428 */
5429
5430 child_ctx = kzalloc(sizeof(struct perf_event_context),
5431 GFP_KERNEL);
5432 if (!child_ctx)
5433 return -ENOMEM;
5434
5435 __perf_event_init_context(child_ctx, child);
5436 child->perf_event_ctxp = child_ctx;
5437 get_task_struct(child);
5438 }
5439
5440 ret = inherit_group(event, parent, parent_ctx,
5441 child, child_ctx);
5442
5443 if (ret)
5444 *inherited_all = 0;
5445
5446 return ret;
5447}
5448
5449
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005450/*
5451 * Initialize the perf_event context in task_struct
5452 */
5453int perf_event_init_task(struct task_struct *child)
5454{
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005455 struct perf_event_context *child_ctx, *parent_ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005456 struct perf_event_context *cloned_ctx;
5457 struct perf_event *event;
5458 struct task_struct *parent = current;
5459 int inherited_all = 1;
5460 int ret = 0;
5461
5462 child->perf_event_ctxp = NULL;
5463
5464 mutex_init(&child->perf_event_mutex);
5465 INIT_LIST_HEAD(&child->perf_event_list);
5466
5467 if (likely(!parent->perf_event_ctxp))
5468 return 0;
5469
5470 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005471 * If the parent's context is a clone, pin it so it won't get
5472 * swapped under us.
5473 */
5474 parent_ctx = perf_pin_task_context(parent);
5475
5476 /*
5477 * No need to check if parent_ctx != NULL here; since we saw
5478 * it non-NULL earlier, the only reason for it to become NULL
5479 * is if we exit, and since we're currently in the middle of
5480 * a fork we can't be exiting at the same time.
5481 */
5482
5483 /*
5484 * Lock the parent list. No need to lock the child - not PID
5485 * hashed yet and not running, so nobody can access it.
5486 */
5487 mutex_lock(&parent_ctx->mutex);
5488
5489 /*
5490 * We dont have to disable NMIs - we are only looking at
5491 * the list, not manipulating it:
5492 */
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005493 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
5494 ret = inherit_task_group(event, parent, parent_ctx, child,
5495 &inherited_all);
5496 if (ret)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005497 break;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005498 }
5499
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005500 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
5501 ret = inherit_task_group(event, parent, parent_ctx, child,
5502 &inherited_all);
5503 if (ret)
5504 break;
5505 }
5506
5507 child_ctx = child->perf_event_ctxp;
5508
Peter Zijlstra05cbaa22009-12-30 16:00:35 +01005509 if (child_ctx && inherited_all) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005510 /*
5511 * Mark the child context as a clone of the parent
5512 * context, or of whatever the parent is a clone of.
5513 * Note that if the parent is a clone, it could get
5514 * uncloned at any point, but that doesn't matter
5515 * because the list of events and the generation
5516 * count can't have changed since we took the mutex.
5517 */
5518 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
5519 if (cloned_ctx) {
5520 child_ctx->parent_ctx = cloned_ctx;
5521 child_ctx->parent_gen = parent_ctx->parent_gen;
5522 } else {
5523 child_ctx->parent_ctx = parent_ctx;
5524 child_ctx->parent_gen = parent_ctx->generation;
5525 }
5526 get_ctx(child_ctx->parent_ctx);
5527 }
5528
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005529 mutex_unlock(&parent_ctx->mutex);
5530
5531 perf_unpin_context(parent_ctx);
5532
5533 return ret;
5534}
5535
Paul Mackerras220b1402010-03-10 20:45:52 +11005536static void __init perf_event_init_all_cpus(void)
5537{
5538 int cpu;
5539 struct perf_cpu_context *cpuctx;
5540
5541 for_each_possible_cpu(cpu) {
5542 cpuctx = &per_cpu(perf_cpu_context, cpu);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005543 mutex_init(&cpuctx->hlist_mutex);
Paul Mackerras220b1402010-03-10 20:45:52 +11005544 __perf_event_init_context(&cpuctx->ctx, NULL);
5545 }
5546}
5547
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005548static void __cpuinit perf_event_init_cpu(int cpu)
5549{
5550 struct perf_cpu_context *cpuctx;
5551
5552 cpuctx = &per_cpu(perf_cpu_context, cpu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005553
5554 spin_lock(&perf_resource_lock);
5555 cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5556 spin_unlock(&perf_resource_lock);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005557
5558 mutex_lock(&cpuctx->hlist_mutex);
5559 if (cpuctx->hlist_refcount > 0) {
5560 struct swevent_hlist *hlist;
5561
5562 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5563 WARN_ON_ONCE(!hlist);
5564 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
5565 }
5566 mutex_unlock(&cpuctx->hlist_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005567}
5568
5569#ifdef CONFIG_HOTPLUG_CPU
5570static void __perf_event_exit_cpu(void *info)
5571{
5572 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5573 struct perf_event_context *ctx = &cpuctx->ctx;
5574 struct perf_event *event, *tmp;
5575
Frederic Weisbecker889ff012010-01-09 20:04:47 +01005576 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5577 __perf_event_remove_from_context(event);
5578 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005579 __perf_event_remove_from_context(event);
5580}
5581static void perf_event_exit_cpu(int cpu)
5582{
5583 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5584 struct perf_event_context *ctx = &cpuctx->ctx;
5585
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005586 mutex_lock(&cpuctx->hlist_mutex);
5587 swevent_hlist_release(cpuctx);
5588 mutex_unlock(&cpuctx->hlist_mutex);
5589
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005590 mutex_lock(&ctx->mutex);
5591 smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5592 mutex_unlock(&ctx->mutex);
5593}
5594#else
5595static inline void perf_event_exit_cpu(int cpu) { }
5596#endif
5597
5598static int __cpuinit
5599perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
5600{
5601 unsigned int cpu = (long)hcpu;
5602
5603 switch (action) {
5604
5605 case CPU_UP_PREPARE:
5606 case CPU_UP_PREPARE_FROZEN:
5607 perf_event_init_cpu(cpu);
5608 break;
5609
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005610 case CPU_DOWN_PREPARE:
5611 case CPU_DOWN_PREPARE_FROZEN:
5612 perf_event_exit_cpu(cpu);
5613 break;
5614
5615 default:
5616 break;
5617 }
5618
5619 return NOTIFY_OK;
5620}
5621
5622/*
5623 * This has to have a higher priority than migration_notifier in sched.c.
5624 */
5625static struct notifier_block __cpuinitdata perf_cpu_nb = {
5626 .notifier_call = perf_cpu_notify,
5627 .priority = 20,
5628};
5629
5630void __init perf_event_init(void)
5631{
Paul Mackerras220b1402010-03-10 20:45:52 +11005632 perf_event_init_all_cpus();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005633 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
5634 (void *)(long)smp_processor_id());
5635 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
5636 (void *)(long)smp_processor_id());
5637 register_cpu_notifier(&perf_cpu_nb);
5638}
5639
Andi Kleenc9be0a32010-01-05 12:47:58 +01005640static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
5641 struct sysdev_class_attribute *attr,
5642 char *buf)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005643{
5644 return sprintf(buf, "%d\n", perf_reserved_percpu);
5645}
5646
5647static ssize_t
5648perf_set_reserve_percpu(struct sysdev_class *class,
Andi Kleenc9be0a32010-01-05 12:47:58 +01005649 struct sysdev_class_attribute *attr,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005650 const char *buf,
5651 size_t count)
5652{
5653 struct perf_cpu_context *cpuctx;
5654 unsigned long val;
5655 int err, cpu, mpt;
5656
5657 err = strict_strtoul(buf, 10, &val);
5658 if (err)
5659 return err;
5660 if (val > perf_max_events)
5661 return -EINVAL;
5662
5663 spin_lock(&perf_resource_lock);
5664 perf_reserved_percpu = val;
5665 for_each_online_cpu(cpu) {
5666 cpuctx = &per_cpu(perf_cpu_context, cpu);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01005667 raw_spin_lock_irq(&cpuctx->ctx.lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005668 mpt = min(perf_max_events - cpuctx->ctx.nr_events,
5669 perf_max_events - perf_reserved_percpu);
5670 cpuctx->max_pertask = mpt;
Thomas Gleixnere625cce2009-11-17 18:02:06 +01005671 raw_spin_unlock_irq(&cpuctx->ctx.lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005672 }
5673 spin_unlock(&perf_resource_lock);
5674
5675 return count;
5676}
5677
Andi Kleenc9be0a32010-01-05 12:47:58 +01005678static ssize_t perf_show_overcommit(struct sysdev_class *class,
5679 struct sysdev_class_attribute *attr,
5680 char *buf)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005681{
5682 return sprintf(buf, "%d\n", perf_overcommit);
5683}
5684
5685static ssize_t
Andi Kleenc9be0a32010-01-05 12:47:58 +01005686perf_set_overcommit(struct sysdev_class *class,
5687 struct sysdev_class_attribute *attr,
5688 const char *buf, size_t count)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005689{
5690 unsigned long val;
5691 int err;
5692
5693 err = strict_strtoul(buf, 10, &val);
5694 if (err)
5695 return err;
5696 if (val > 1)
5697 return -EINVAL;
5698
5699 spin_lock(&perf_resource_lock);
5700 perf_overcommit = val;
5701 spin_unlock(&perf_resource_lock);
5702
5703 return count;
5704}
5705
5706static SYSDEV_CLASS_ATTR(
5707 reserve_percpu,
5708 0644,
5709 perf_show_reserve_percpu,
5710 perf_set_reserve_percpu
5711 );
5712
5713static SYSDEV_CLASS_ATTR(
5714 overcommit,
5715 0644,
5716 perf_show_overcommit,
5717 perf_set_overcommit
5718 );
5719
5720static struct attribute *perfclass_attrs[] = {
5721 &attr_reserve_percpu.attr,
5722 &attr_overcommit.attr,
5723 NULL
5724};
5725
5726static struct attribute_group perfclass_attr_group = {
5727 .attrs = perfclass_attrs,
5728 .name = "perf_events",
5729};
5730
5731static int __init perf_event_sysfs_init(void)
5732{
5733 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
5734 &perfclass_attr_group);
5735}
5736device_initcall(perf_event_sysfs_init);