blob: 1b5c081d8b9f9c8ea05f1a8ecaf861f80ab7ba1c [file] [log] [blame]
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001/*
Ingo Molnar57c0c152009-09-21 12:20:38 +02002 * Performance events core code:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
Ingo Molnare7e7ee22011-05-04 08:42:29 +02005 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
Al Virod36b6912011-12-29 17:09:01 -05007 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
Ingo Molnarcdd6c482009-09-21 12:02:48 +02008 *
Ingo Molnar57c0c152009-09-21 12:20:38 +02009 * For licensing details see kernel-base/COPYING
Ingo Molnarcdd6c482009-09-21 12:02:48 +020010 */
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
Peter Zijlstra2e80a822010-11-17 23:17:36 +010016#include <linux/idr.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020017#include <linux/file.h>
18#include <linux/poll.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Frederic Weisbecker76e1d902010-04-05 15:35:57 +020020#include <linux/hash.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020021#include <linux/sysfs.h>
22#include <linux/dcache.h>
23#include <linux/percpu.h>
24#include <linux/ptrace.h>
Peter Zijlstrac2774432010-12-08 15:29:02 +010025#include <linux/reboot.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020026#include <linux/vmstat.h>
Peter Zijlstraabe43402010-11-17 23:17:37 +010027#include <linux/device.h>
Paul Gortmaker6e5fdee2011-05-26 16:00:52 -040028#include <linux/export.h>
Peter Zijlstra906010b2009-09-21 16:08:49 +020029#include <linux/vmalloc.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020030#include <linux/hardirq.h>
31#include <linux/rculist.h>
32#include <linux/uaccess.h>
33#include <linux/syscalls.h>
34#include <linux/anon_inodes.h>
35#include <linux/kernel_stat.h>
36#include <linux/perf_event.h>
Li Zefan6fb29152009-10-15 11:21:42 +080037#include <linux/ftrace_event.h>
Jason Wessel3c502e72010-11-04 17:33:01 -050038#include <linux/hw_breakpoint.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020039
Frederic Weisbecker76369132011-05-19 19:55:04 +020040#include "internal.h"
41
Ingo Molnarcdd6c482009-09-21 12:02:48 +020042#include <asm/irq_regs.h>
43
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +010044struct remote_function_call {
Ingo Molnare7e7ee22011-05-04 08:42:29 +020045 struct task_struct *p;
46 int (*func)(void *info);
47 void *info;
48 int ret;
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +010049};
50
51static void remote_function(void *data)
52{
53 struct remote_function_call *tfc = data;
54 struct task_struct *p = tfc->p;
55
56 if (p) {
57 tfc->ret = -EAGAIN;
58 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
59 return;
60 }
61
62 tfc->ret = tfc->func(tfc->info);
63}
64
65/**
66 * task_function_call - call a function on the cpu on which a task runs
67 * @p: the task to evaluate
68 * @func: the function to be called
69 * @info: the function call argument
70 *
71 * Calls the function @func when the task is currently running. This might
72 * be on the current CPU, which just calls the function directly
73 *
74 * returns: @func return value, or
75 * -ESRCH - when the process isn't running
76 * -EAGAIN - when the process moved away
77 */
78static int
79task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
80{
81 struct remote_function_call data = {
Ingo Molnare7e7ee22011-05-04 08:42:29 +020082 .p = p,
83 .func = func,
84 .info = info,
85 .ret = -ESRCH, /* No such (running) process */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +010086 };
87
88 if (task_curr(p))
89 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
90
91 return data.ret;
92}
93
94/**
95 * cpu_function_call - call a function on the cpu
96 * @func: the function to be called
97 * @info: the function call argument
98 *
99 * Calls the function @func on the remote cpu.
100 *
101 * returns: @func return value or -ENXIO when the cpu is offline
102 */
103static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
104{
105 struct remote_function_call data = {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200106 .p = NULL,
107 .func = func,
108 .info = info,
109 .ret = -ENXIO, /* No such CPU */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +0100110 };
111
112 smp_call_function_single(cpu, remote_function, &data, 1);
113
114 return data.ret;
115}
116
Stephane Eraniane5d13672011-02-14 11:20:01 +0200117#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
118 PERF_FLAG_FD_OUTPUT |\
119 PERF_FLAG_PID_CGROUP)
120
Stephane Eranian0b3fcf12011-01-03 18:20:01 +0200121enum event_type_t {
122 EVENT_FLEXIBLE = 0x1,
123 EVENT_PINNED = 0x2,
124 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
125};
126
Stephane Eraniane5d13672011-02-14 11:20:01 +0200127/*
128 * perf_sched_events : >0 events exist
129 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
130 */
Gleb Natapovb2029522011-11-27 17:59:09 +0200131struct jump_label_key_deferred perf_sched_events __read_mostly;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200132static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
133
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200134static atomic_t nr_mmap_events __read_mostly;
135static atomic_t nr_comm_events __read_mostly;
136static atomic_t nr_task_events __read_mostly;
137
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200138static LIST_HEAD(pmus);
139static DEFINE_MUTEX(pmus_lock);
140static struct srcu_struct pmus_srcu;
141
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200142/*
143 * perf event paranoia level:
144 * -1 - not paranoid at all
145 * 0 - disallow raw tracepoint access for unpriv
146 * 1 - disallow cpu events for unpriv
147 * 2 - disallow kernel profiling for unpriv
148 */
149int sysctl_perf_event_paranoid __read_mostly = 1;
150
Frederic Weisbecker20443382011-03-31 03:33:29 +0200151/* Minimum for 512 kiB + 1 user control page */
152int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200153
154/*
155 * max perf event sample rate
156 */
Peter Zijlstra163ec432011-02-16 11:22:34 +0100157#define DEFAULT_MAX_SAMPLE_RATE 100000
158int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
159static int max_samples_per_tick __read_mostly =
160 DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
161
162int perf_proc_update_handler(struct ctl_table *table, int write,
163 void __user *buffer, size_t *lenp,
164 loff_t *ppos)
165{
166 int ret = proc_dointvec(table, write, buffer, lenp, ppos);
167
168 if (ret || !write)
169 return ret;
170
171 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
172
173 return 0;
174}
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200175
176static atomic64_t perf_event_id;
177
Stephane Eranian0b3fcf12011-01-03 18:20:01 +0200178static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
179 enum event_type_t event_type);
180
181static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
Stephane Eraniane5d13672011-02-14 11:20:01 +0200182 enum event_type_t event_type,
183 struct task_struct *task);
184
185static void update_context_time(struct perf_event_context *ctx);
186static u64 perf_event_time(struct perf_event *event);
Stephane Eranian0b3fcf12011-01-03 18:20:01 +0200187
Peter Zijlstra10c6db12011-11-26 02:47:31 +0100188static void ring_buffer_attach(struct perf_event *event,
189 struct ring_buffer *rb);
190
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200191void __weak perf_event_print_debug(void) { }
192
Matt Fleming84c79912010-10-03 21:41:13 +0100193extern __weak const char *perf_pmu_name(void)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200194{
Matt Fleming84c79912010-10-03 21:41:13 +0100195 return "pmu";
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200196}
197
Stephane Eranian0b3fcf12011-01-03 18:20:01 +0200198static inline u64 perf_clock(void)
199{
200 return local_clock();
201}
202
Stephane Eraniane5d13672011-02-14 11:20:01 +0200203static inline struct perf_cpu_context *
204__get_cpu_context(struct perf_event_context *ctx)
205{
206 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
207}
208
Peter Zijlstrafacc4302011-04-09 21:17:42 +0200209static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
210 struct perf_event_context *ctx)
211{
212 raw_spin_lock(&cpuctx->ctx.lock);
213 if (ctx)
214 raw_spin_lock(&ctx->lock);
215}
216
217static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
218 struct perf_event_context *ctx)
219{
220 if (ctx)
221 raw_spin_unlock(&ctx->lock);
222 raw_spin_unlock(&cpuctx->ctx.lock);
223}
224
Stephane Eraniane5d13672011-02-14 11:20:01 +0200225#ifdef CONFIG_CGROUP_PERF
226
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200227/*
228 * Must ensure cgroup is pinned (css_get) before calling
229 * this function. In other words, we cannot call this function
230 * if there is no cgroup event for the current CPU context.
231 */
Stephane Eraniane5d13672011-02-14 11:20:01 +0200232static inline struct perf_cgroup *
233perf_cgroup_from_task(struct task_struct *task)
234{
235 return container_of(task_subsys_state(task, perf_subsys_id),
236 struct perf_cgroup, css);
237}
238
239static inline bool
240perf_cgroup_match(struct perf_event *event)
241{
242 struct perf_event_context *ctx = event->ctx;
243 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
244
245 return !event->cgrp || event->cgrp == cpuctx->cgrp;
246}
247
248static inline void perf_get_cgroup(struct perf_event *event)
249{
250 css_get(&event->cgrp->css);
251}
252
253static inline void perf_put_cgroup(struct perf_event *event)
254{
255 css_put(&event->cgrp->css);
256}
257
258static inline void perf_detach_cgroup(struct perf_event *event)
259{
260 perf_put_cgroup(event);
261 event->cgrp = NULL;
262}
263
264static inline int is_cgroup_event(struct perf_event *event)
265{
266 return event->cgrp != NULL;
267}
268
269static inline u64 perf_cgroup_event_time(struct perf_event *event)
270{
271 struct perf_cgroup_info *t;
272
273 t = per_cpu_ptr(event->cgrp->info, event->cpu);
274 return t->time;
275}
276
277static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
278{
279 struct perf_cgroup_info *info;
280 u64 now;
281
282 now = perf_clock();
283
284 info = this_cpu_ptr(cgrp->info);
285
286 info->time += now - info->timestamp;
287 info->timestamp = now;
288}
289
290static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
291{
292 struct perf_cgroup *cgrp_out = cpuctx->cgrp;
293 if (cgrp_out)
294 __update_cgrp_time(cgrp_out);
295}
296
297static inline void update_cgrp_time_from_event(struct perf_event *event)
298{
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200299 struct perf_cgroup *cgrp;
300
Stephane Eraniane5d13672011-02-14 11:20:01 +0200301 /*
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200302 * ensure we access cgroup data only when needed and
303 * when we know the cgroup is pinned (css_get)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200304 */
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200305 if (!is_cgroup_event(event))
Stephane Eraniane5d13672011-02-14 11:20:01 +0200306 return;
307
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200308 cgrp = perf_cgroup_from_task(current);
309 /*
310 * Do not update time when cgroup is not active
311 */
312 if (cgrp == event->cgrp)
313 __update_cgrp_time(event->cgrp);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200314}
315
316static inline void
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200317perf_cgroup_set_timestamp(struct task_struct *task,
318 struct perf_event_context *ctx)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200319{
320 struct perf_cgroup *cgrp;
321 struct perf_cgroup_info *info;
322
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200323 /*
324 * ctx->lock held by caller
325 * ensure we do not access cgroup data
326 * unless we have the cgroup pinned (css_get)
327 */
328 if (!task || !ctx->nr_cgroups)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200329 return;
330
331 cgrp = perf_cgroup_from_task(task);
332 info = this_cpu_ptr(cgrp->info);
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200333 info->timestamp = ctx->timestamp;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200334}
335
336#define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
337#define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
338
339/*
340 * reschedule events based on the cgroup constraint of task.
341 *
342 * mode SWOUT : schedule out everything
343 * mode SWIN : schedule in based on cgroup for next
344 */
345void perf_cgroup_switch(struct task_struct *task, int mode)
346{
347 struct perf_cpu_context *cpuctx;
348 struct pmu *pmu;
349 unsigned long flags;
350
351 /*
352 * disable interrupts to avoid geting nr_cgroup
353 * changes via __perf_event_disable(). Also
354 * avoids preemption.
355 */
356 local_irq_save(flags);
357
358 /*
359 * we reschedule only in the presence of cgroup
360 * constrained events.
361 */
362 rcu_read_lock();
363
364 list_for_each_entry_rcu(pmu, &pmus, entry) {
Stephane Eraniane5d13672011-02-14 11:20:01 +0200365 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
366
Stephane Eraniane5d13672011-02-14 11:20:01 +0200367 /*
368 * perf_cgroup_events says at least one
369 * context on this CPU has cgroup events.
370 *
371 * ctx->nr_cgroups reports the number of cgroup
372 * events for a context.
373 */
374 if (cpuctx->ctx.nr_cgroups > 0) {
Peter Zijlstrafacc4302011-04-09 21:17:42 +0200375 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
376 perf_pmu_disable(cpuctx->ctx.pmu);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200377
378 if (mode & PERF_CGROUP_SWOUT) {
379 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
380 /*
381 * must not be done before ctxswout due
382 * to event_filter_match() in event_sched_out()
383 */
384 cpuctx->cgrp = NULL;
385 }
386
387 if (mode & PERF_CGROUP_SWIN) {
Stephane Eraniane566b762011-04-06 02:54:54 +0200388 WARN_ON_ONCE(cpuctx->cgrp);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200389 /* set cgrp before ctxsw in to
390 * allow event_filter_match() to not
391 * have to pass task around
392 */
393 cpuctx->cgrp = perf_cgroup_from_task(task);
394 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
395 }
Peter Zijlstrafacc4302011-04-09 21:17:42 +0200396 perf_pmu_enable(cpuctx->ctx.pmu);
397 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200398 }
Stephane Eraniane5d13672011-02-14 11:20:01 +0200399 }
400
401 rcu_read_unlock();
402
403 local_irq_restore(flags);
404}
405
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200406static inline void perf_cgroup_sched_out(struct task_struct *task,
407 struct task_struct *next)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200408{
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200409 struct perf_cgroup *cgrp1;
410 struct perf_cgroup *cgrp2 = NULL;
411
412 /*
413 * we come here when we know perf_cgroup_events > 0
414 */
415 cgrp1 = perf_cgroup_from_task(task);
416
417 /*
418 * next is NULL when called from perf_event_enable_on_exec()
419 * that will systematically cause a cgroup_switch()
420 */
421 if (next)
422 cgrp2 = perf_cgroup_from_task(next);
423
424 /*
425 * only schedule out current cgroup events if we know
426 * that we are switching to a different cgroup. Otherwise,
427 * do no touch the cgroup events.
428 */
429 if (cgrp1 != cgrp2)
430 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200431}
432
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200433static inline void perf_cgroup_sched_in(struct task_struct *prev,
434 struct task_struct *task)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200435{
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200436 struct perf_cgroup *cgrp1;
437 struct perf_cgroup *cgrp2 = NULL;
438
439 /*
440 * we come here when we know perf_cgroup_events > 0
441 */
442 cgrp1 = perf_cgroup_from_task(task);
443
444 /* prev can never be NULL */
445 cgrp2 = perf_cgroup_from_task(prev);
446
447 /*
448 * only need to schedule in cgroup events if we are changing
449 * cgroup during ctxsw. Cgroup events were not scheduled
450 * out of ctxsw out if that was not the case.
451 */
452 if (cgrp1 != cgrp2)
453 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200454}
455
456static inline int perf_cgroup_connect(int fd, struct perf_event *event,
457 struct perf_event_attr *attr,
458 struct perf_event *group_leader)
459{
460 struct perf_cgroup *cgrp;
461 struct cgroup_subsys_state *css;
462 struct file *file;
463 int ret = 0, fput_needed;
464
465 file = fget_light(fd, &fput_needed);
466 if (!file)
467 return -EBADF;
468
469 css = cgroup_css_from_dir(file, perf_subsys_id);
Li Zefan3db272c2011-03-03 14:25:37 +0800470 if (IS_ERR(css)) {
471 ret = PTR_ERR(css);
472 goto out;
473 }
Stephane Eraniane5d13672011-02-14 11:20:01 +0200474
475 cgrp = container_of(css, struct perf_cgroup, css);
476 event->cgrp = cgrp;
477
Li Zefanf75e18c2011-03-03 14:25:50 +0800478 /* must be done before we fput() the file */
479 perf_get_cgroup(event);
480
Stephane Eraniane5d13672011-02-14 11:20:01 +0200481 /*
482 * all events in a group must monitor
483 * the same cgroup because a task belongs
484 * to only one perf cgroup at a time
485 */
486 if (group_leader && group_leader->cgrp != cgrp) {
487 perf_detach_cgroup(event);
488 ret = -EINVAL;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200489 }
Li Zefan3db272c2011-03-03 14:25:37 +0800490out:
Stephane Eraniane5d13672011-02-14 11:20:01 +0200491 fput_light(file, fput_needed);
492 return ret;
493}
494
495static inline void
496perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
497{
498 struct perf_cgroup_info *t;
499 t = per_cpu_ptr(event->cgrp->info, event->cpu);
500 event->shadow_ctx_time = now - t->timestamp;
501}
502
503static inline void
504perf_cgroup_defer_enabled(struct perf_event *event)
505{
506 /*
507 * when the current task's perf cgroup does not match
508 * the event's, we need to remember to call the
509 * perf_mark_enable() function the first time a task with
510 * a matching perf cgroup is scheduled in.
511 */
512 if (is_cgroup_event(event) && !perf_cgroup_match(event))
513 event->cgrp_defer_enabled = 1;
514}
515
516static inline void
517perf_cgroup_mark_enabled(struct perf_event *event,
518 struct perf_event_context *ctx)
519{
520 struct perf_event *sub;
521 u64 tstamp = perf_event_time(event);
522
523 if (!event->cgrp_defer_enabled)
524 return;
525
526 event->cgrp_defer_enabled = 0;
527
528 event->tstamp_enabled = tstamp - event->total_time_enabled;
529 list_for_each_entry(sub, &event->sibling_list, group_entry) {
530 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
531 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
532 sub->cgrp_defer_enabled = 0;
533 }
534 }
535}
536#else /* !CONFIG_CGROUP_PERF */
537
538static inline bool
539perf_cgroup_match(struct perf_event *event)
540{
541 return true;
542}
543
544static inline void perf_detach_cgroup(struct perf_event *event)
545{}
546
547static inline int is_cgroup_event(struct perf_event *event)
548{
549 return 0;
550}
551
552static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
553{
554 return 0;
555}
556
557static inline void update_cgrp_time_from_event(struct perf_event *event)
558{
559}
560
561static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
562{
563}
564
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200565static inline void perf_cgroup_sched_out(struct task_struct *task,
566 struct task_struct *next)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200567{
568}
569
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200570static inline void perf_cgroup_sched_in(struct task_struct *prev,
571 struct task_struct *task)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200572{
573}
574
575static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
576 struct perf_event_attr *attr,
577 struct perf_event *group_leader)
578{
579 return -EINVAL;
580}
581
582static inline void
Stephane Eranian3f7cce32011-02-18 14:40:01 +0200583perf_cgroup_set_timestamp(struct task_struct *task,
584 struct perf_event_context *ctx)
Stephane Eraniane5d13672011-02-14 11:20:01 +0200585{
586}
587
588void
589perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
590{
591}
592
593static inline void
594perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
595{
596}
597
598static inline u64 perf_cgroup_event_time(struct perf_event *event)
599{
600 return 0;
601}
602
603static inline void
604perf_cgroup_defer_enabled(struct perf_event *event)
605{
606}
607
608static inline void
609perf_cgroup_mark_enabled(struct perf_event *event,
610 struct perf_event_context *ctx)
611{
612}
613#endif
614
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200615void perf_pmu_disable(struct pmu *pmu)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200616{
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200617 int *count = this_cpu_ptr(pmu->pmu_disable_count);
618 if (!(*count)++)
619 pmu->pmu_disable(pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200620}
621
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200622void perf_pmu_enable(struct pmu *pmu)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200623{
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200624 int *count = this_cpu_ptr(pmu->pmu_disable_count);
625 if (!--(*count))
626 pmu->pmu_enable(pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200627}
628
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200629static DEFINE_PER_CPU(struct list_head, rotation_list);
630
631/*
632 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
633 * because they're strictly cpu affine and rotate_start is called with IRQs
634 * disabled, while rotate_context is called from IRQ context.
635 */
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200636static void perf_pmu_rotate_start(struct pmu *pmu)
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +0200637{
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200638 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200639 struct list_head *head = &__get_cpu_var(rotation_list);
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +0200640
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200641 WARN_ON(!irqs_disabled());
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +0200642
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200643 if (list_empty(&cpuctx->rotation_list))
644 list_add(&cpuctx->rotation_list, head);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200645}
646
647static void get_ctx(struct perf_event_context *ctx)
648{
649 WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
650}
651
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200652static void put_ctx(struct perf_event_context *ctx)
653{
654 if (atomic_dec_and_test(&ctx->refcount)) {
655 if (ctx->parent_ctx)
656 put_ctx(ctx->parent_ctx);
657 if (ctx->task)
658 put_task_struct(ctx->task);
Lai Jiangshancb796ff2011-03-18 12:07:41 +0800659 kfree_rcu(ctx, rcu_head);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200660 }
661}
662
663static void unclone_ctx(struct perf_event_context *ctx)
664{
665 if (ctx->parent_ctx) {
666 put_ctx(ctx->parent_ctx);
667 ctx->parent_ctx = NULL;
668 }
669}
670
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200671static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
672{
673 /*
674 * only top level events have the pid namespace they were created in
675 */
676 if (event->parent)
677 event = event->parent;
678
679 return task_tgid_nr_ns(p, event->ns);
680}
681
682static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
683{
684 /*
685 * only top level events have the pid namespace they were created in
686 */
687 if (event->parent)
688 event = event->parent;
689
690 return task_pid_nr_ns(p, event->ns);
691}
692
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200693/*
694 * If we inherit events we want to return the parent event id
695 * to userspace.
696 */
697static u64 primary_event_id(struct perf_event *event)
698{
699 u64 id = event->id;
700
701 if (event->parent)
702 id = event->parent->id;
703
704 return id;
705}
706
707/*
708 * Get the perf_event_context for a task and lock it.
709 * This has to cope with with the fact that until it is locked,
710 * the context could get moved to another task.
711 */
712static struct perf_event_context *
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200713perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200714{
715 struct perf_event_context *ctx;
716
717 rcu_read_lock();
Peter Zijlstra9ed60602010-06-11 17:36:35 +0200718retry:
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200719 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200720 if (ctx) {
721 /*
722 * If this context is a clone of another, it might
723 * get swapped for another underneath us by
724 * perf_event_task_sched_out, though the
725 * rcu_read_lock() protects us from any context
726 * getting freed. Lock the context and check if it
727 * got swapped before we could get the lock, and retry
728 * if so. If we locked the right context, then it
729 * can't get swapped on us any more.
730 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100731 raw_spin_lock_irqsave(&ctx->lock, *flags);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200732 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100733 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200734 goto retry;
735 }
736
737 if (!atomic_inc_not_zero(&ctx->refcount)) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100738 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200739 ctx = NULL;
740 }
741 }
742 rcu_read_unlock();
743 return ctx;
744}
745
746/*
747 * Get the context for a task and increment its pin_count so it
748 * can't get swapped to another task. This also increments its
749 * reference count so that the context can't get freed.
750 */
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200751static struct perf_event_context *
752perf_pin_task_context(struct task_struct *task, int ctxn)
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200753{
754 struct perf_event_context *ctx;
755 unsigned long flags;
756
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200757 ctx = perf_lock_task_context(task, ctxn, &flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200758 if (ctx) {
759 ++ctx->pin_count;
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100760 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200761 }
762 return ctx;
763}
764
765static void perf_unpin_context(struct perf_event_context *ctx)
766{
767 unsigned long flags;
768
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100769 raw_spin_lock_irqsave(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200770 --ctx->pin_count;
Thomas Gleixnere625cce2009-11-17 18:02:06 +0100771 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200772}
773
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100774/*
775 * Update the record of the current time in a context.
776 */
777static void update_context_time(struct perf_event_context *ctx)
778{
779 u64 now = perf_clock();
780
781 ctx->time += now - ctx->timestamp;
782 ctx->timestamp = now;
783}
784
Stephane Eranian41587552011-01-03 18:20:01 +0200785static u64 perf_event_time(struct perf_event *event)
786{
787 struct perf_event_context *ctx = event->ctx;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200788
789 if (is_cgroup_event(event))
790 return perf_cgroup_event_time(event);
791
Stephane Eranian41587552011-01-03 18:20:01 +0200792 return ctx ? ctx->time : 0;
793}
794
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100795/*
796 * Update the total_time_enabled and total_time_running fields for a event.
Eric B Munsonb7526f02011-06-23 16:34:37 -0400797 * The caller of this function needs to hold the ctx->lock.
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100798 */
799static void update_event_times(struct perf_event *event)
800{
801 struct perf_event_context *ctx = event->ctx;
802 u64 run_end;
803
804 if (event->state < PERF_EVENT_STATE_INACTIVE ||
805 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
806 return;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200807 /*
808 * in cgroup mode, time_enabled represents
809 * the time the event was enabled AND active
810 * tasks were in the monitored cgroup. This is
811 * independent of the activity of the context as
812 * there may be a mix of cgroup and non-cgroup events.
813 *
814 * That is why we treat cgroup events differently
815 * here.
816 */
817 if (is_cgroup_event(event))
Namhyung Kim46cd6a7f2012-01-20 10:12:46 +0900818 run_end = perf_cgroup_event_time(event);
Stephane Eraniane5d13672011-02-14 11:20:01 +0200819 else if (ctx->is_active)
820 run_end = ctx->time;
Peter Zijlstraacd1d7c2009-11-23 15:00:36 +0100821 else
822 run_end = event->tstamp_stopped;
823
824 event->total_time_enabled = run_end - event->tstamp_enabled;
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100825
826 if (event->state == PERF_EVENT_STATE_INACTIVE)
827 run_end = event->tstamp_stopped;
828 else
Stephane Eranian41587552011-01-03 18:20:01 +0200829 run_end = perf_event_time(event);
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100830
831 event->total_time_running = run_end - event->tstamp_running;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200832
Peter Zijlstraf67218c2009-11-23 11:37:27 +0100833}
834
Peter Zijlstra96c21a42010-05-11 16:19:10 +0200835/*
836 * Update total_time_enabled and total_time_running for all events in a group.
837 */
838static void update_group_times(struct perf_event *leader)
839{
840 struct perf_event *event;
841
842 update_event_times(leader);
843 list_for_each_entry(event, &leader->sibling_list, group_entry)
844 update_event_times(event);
845}
846
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100847static struct list_head *
848ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
849{
850 if (event->attr.pinned)
851 return &ctx->pinned_groups;
852 else
853 return &ctx->flexible_groups;
854}
855
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200856/*
857 * Add a event from the lists for its context.
858 * Must be called with ctx->mutex and ctx->lock held.
859 */
860static void
861list_add_event(struct perf_event *event, struct perf_event_context *ctx)
862{
Peter Zijlstra8a495422010-05-27 15:47:49 +0200863 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
864 event->attach_state |= PERF_ATTACH_CONTEXT;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200865
866 /*
Peter Zijlstra8a495422010-05-27 15:47:49 +0200867 * If we're a stand alone event or group leader, we go to the context
868 * list, group events are kept attached to the group so that
869 * perf_group_detach can, at all times, locate all siblings.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200870 */
Peter Zijlstra8a495422010-05-27 15:47:49 +0200871 if (event->group_leader == event) {
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100872 struct list_head *list;
873
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100874 if (is_software_event(event))
875 event->group_flags |= PERF_GROUP_SOFTWARE;
876
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100877 list = ctx_group_list(event, ctx);
878 list_add_tail(&event->group_entry, list);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200879 }
880
Peter Zijlstra08309372011-03-03 11:31:20 +0100881 if (is_cgroup_event(event))
Stephane Eraniane5d13672011-02-14 11:20:01 +0200882 ctx->nr_cgroups++;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200883
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200884 list_add_rcu(&event->event_entry, &ctx->event_list);
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +0200885 if (!ctx->nr_events)
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200886 perf_pmu_rotate_start(ctx->pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200887 ctx->nr_events++;
888 if (event->attr.inherit_stat)
889 ctx->nr_stat++;
890}
891
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200892/*
893 * Called at perf_event creation and when events are attached/detached from a
894 * group.
895 */
896static void perf_event__read_size(struct perf_event *event)
897{
898 int entry = sizeof(u64); /* value */
899 int size = 0;
900 int nr = 1;
901
902 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
903 size += sizeof(u64);
904
905 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
906 size += sizeof(u64);
907
908 if (event->attr.read_format & PERF_FORMAT_ID)
909 entry += sizeof(u64);
910
911 if (event->attr.read_format & PERF_FORMAT_GROUP) {
912 nr += event->group_leader->nr_siblings;
913 size += sizeof(u64);
914 }
915
916 size += entry * nr;
917 event->read_size = size;
918}
919
920static void perf_event__header_size(struct perf_event *event)
921{
922 struct perf_sample_data *data;
923 u64 sample_type = event->attr.sample_type;
924 u16 size = 0;
925
926 perf_event__read_size(event);
927
928 if (sample_type & PERF_SAMPLE_IP)
929 size += sizeof(data->ip);
930
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200931 if (sample_type & PERF_SAMPLE_ADDR)
932 size += sizeof(data->addr);
933
934 if (sample_type & PERF_SAMPLE_PERIOD)
935 size += sizeof(data->period);
936
937 if (sample_type & PERF_SAMPLE_READ)
938 size += event->read_size;
939
940 event->header_size = size;
941}
942
943static void perf_event__id_header_size(struct perf_event *event)
944{
945 struct perf_sample_data *data;
946 u64 sample_type = event->attr.sample_type;
947 u16 size = 0;
948
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200949 if (sample_type & PERF_SAMPLE_TID)
950 size += sizeof(data->tid_entry);
951
952 if (sample_type & PERF_SAMPLE_TIME)
953 size += sizeof(data->time);
954
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200955 if (sample_type & PERF_SAMPLE_ID)
956 size += sizeof(data->id);
957
958 if (sample_type & PERF_SAMPLE_STREAM_ID)
959 size += sizeof(data->stream_id);
960
961 if (sample_type & PERF_SAMPLE_CPU)
962 size += sizeof(data->cpu_entry);
963
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200964 event->id_header_size = size;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200965}
966
Peter Zijlstra8a495422010-05-27 15:47:49 +0200967static void perf_group_attach(struct perf_event *event)
968{
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200969 struct perf_event *group_leader = event->group_leader, *pos;
Peter Zijlstra8a495422010-05-27 15:47:49 +0200970
Peter Zijlstra74c33372010-10-15 11:40:29 +0200971 /*
972 * We can have double attach due to group movement in perf_event_open.
973 */
974 if (event->attach_state & PERF_ATTACH_GROUP)
975 return;
976
Peter Zijlstra8a495422010-05-27 15:47:49 +0200977 event->attach_state |= PERF_ATTACH_GROUP;
978
979 if (group_leader == event)
980 return;
981
982 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
983 !is_software_event(event))
984 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
985
986 list_add_tail(&event->group_entry, &group_leader->sibling_list);
987 group_leader->nr_siblings++;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200988
989 perf_event__header_size(group_leader);
990
991 list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
992 perf_event__header_size(pos);
Peter Zijlstra8a495422010-05-27 15:47:49 +0200993}
994
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200995/*
996 * Remove a event from the lists for its context.
997 * Must be called with ctx->mutex and ctx->lock held.
998 */
999static void
1000list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1001{
Stephane Eranian68cacd22011-03-23 16:03:06 +01001002 struct perf_cpu_context *cpuctx;
Peter Zijlstra8a495422010-05-27 15:47:49 +02001003 /*
1004 * We can have double detach due to exit/hot-unplug + close.
1005 */
1006 if (!(event->attach_state & PERF_ATTACH_CONTEXT))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001007 return;
Peter Zijlstra8a495422010-05-27 15:47:49 +02001008
1009 event->attach_state &= ~PERF_ATTACH_CONTEXT;
1010
Stephane Eranian68cacd22011-03-23 16:03:06 +01001011 if (is_cgroup_event(event)) {
Stephane Eraniane5d13672011-02-14 11:20:01 +02001012 ctx->nr_cgroups--;
Stephane Eranian68cacd22011-03-23 16:03:06 +01001013 cpuctx = __get_cpu_context(ctx);
1014 /*
1015 * if there are no more cgroup events
1016 * then cler cgrp to avoid stale pointer
1017 * in update_cgrp_time_from_cpuctx()
1018 */
1019 if (!ctx->nr_cgroups)
1020 cpuctx->cgrp = NULL;
1021 }
Stephane Eraniane5d13672011-02-14 11:20:01 +02001022
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001023 ctx->nr_events--;
1024 if (event->attr.inherit_stat)
1025 ctx->nr_stat--;
1026
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001027 list_del_rcu(&event->event_entry);
1028
Peter Zijlstra8a495422010-05-27 15:47:49 +02001029 if (event->group_leader == event)
1030 list_del_init(&event->group_entry);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001031
Peter Zijlstra96c21a42010-05-11 16:19:10 +02001032 update_group_times(event);
Stephane Eranianb2e74a22009-11-26 09:24:30 -08001033
1034 /*
1035 * If event was in error state, then keep it
1036 * that way, otherwise bogus counts will be
1037 * returned on read(). The only way to get out
1038 * of error state is by explicit re-enabling
1039 * of the event
1040 */
1041 if (event->state > PERF_EVENT_STATE_OFF)
1042 event->state = PERF_EVENT_STATE_OFF;
Peter Zijlstra050735b2010-05-11 11:51:53 +02001043}
1044
Peter Zijlstra8a495422010-05-27 15:47:49 +02001045static void perf_group_detach(struct perf_event *event)
Peter Zijlstra050735b2010-05-11 11:51:53 +02001046{
1047 struct perf_event *sibling, *tmp;
Peter Zijlstra8a495422010-05-27 15:47:49 +02001048 struct list_head *list = NULL;
1049
1050 /*
1051 * We can have double detach due to exit/hot-unplug + close.
1052 */
1053 if (!(event->attach_state & PERF_ATTACH_GROUP))
1054 return;
1055
1056 event->attach_state &= ~PERF_ATTACH_GROUP;
1057
1058 /*
1059 * If this is a sibling, remove it from its group.
1060 */
1061 if (event->group_leader != event) {
1062 list_del_init(&event->group_entry);
1063 event->group_leader->nr_siblings--;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02001064 goto out;
Peter Zijlstra8a495422010-05-27 15:47:49 +02001065 }
1066
1067 if (!list_empty(&event->group_entry))
1068 list = &event->group_entry;
Peter Zijlstra2e2af502009-11-23 11:37:25 +01001069
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001070 /*
1071 * If this was a group event with sibling events then
1072 * upgrade the siblings to singleton events by adding them
Peter Zijlstra8a495422010-05-27 15:47:49 +02001073 * to whatever list we are on.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001074 */
1075 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
Peter Zijlstra8a495422010-05-27 15:47:49 +02001076 if (list)
1077 list_move_tail(&sibling->group_entry, list);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001078 sibling->group_leader = sibling;
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +01001079
1080 /* Inherit group flags from the previous leader */
1081 sibling->group_flags = event->group_flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001082 }
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02001083
1084out:
1085 perf_event__header_size(event->group_leader);
1086
1087 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1088 perf_event__header_size(tmp);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001089}
1090
Stephane Eranianfa66f072010-08-26 16:40:01 +02001091static inline int
1092event_filter_match(struct perf_event *event)
1093{
Stephane Eraniane5d13672011-02-14 11:20:01 +02001094 return (event->cpu == -1 || event->cpu == smp_processor_id())
1095 && perf_cgroup_match(event);
Stephane Eranianfa66f072010-08-26 16:40:01 +02001096}
1097
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001098static void
1099event_sched_out(struct perf_event *event,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001100 struct perf_cpu_context *cpuctx,
1101 struct perf_event_context *ctx)
1102{
Stephane Eranian41587552011-01-03 18:20:01 +02001103 u64 tstamp = perf_event_time(event);
Stephane Eranianfa66f072010-08-26 16:40:01 +02001104 u64 delta;
1105 /*
1106 * An event which could not be activated because of
1107 * filter mismatch still needs to have its timings
1108 * maintained, otherwise bogus information is return
1109 * via read() for time_enabled, time_running:
1110 */
1111 if (event->state == PERF_EVENT_STATE_INACTIVE
1112 && !event_filter_match(event)) {
Stephane Eraniane5d13672011-02-14 11:20:01 +02001113 delta = tstamp - event->tstamp_stopped;
Stephane Eranianfa66f072010-08-26 16:40:01 +02001114 event->tstamp_running += delta;
Stephane Eranian41587552011-01-03 18:20:01 +02001115 event->tstamp_stopped = tstamp;
Stephane Eranianfa66f072010-08-26 16:40:01 +02001116 }
1117
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001118 if (event->state != PERF_EVENT_STATE_ACTIVE)
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001119 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001120
1121 event->state = PERF_EVENT_STATE_INACTIVE;
1122 if (event->pending_disable) {
1123 event->pending_disable = 0;
1124 event->state = PERF_EVENT_STATE_OFF;
1125 }
Stephane Eranian41587552011-01-03 18:20:01 +02001126 event->tstamp_stopped = tstamp;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02001127 event->pmu->del(event, 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001128 event->oncpu = -1;
1129
1130 if (!is_software_event(event))
1131 cpuctx->active_oncpu--;
1132 ctx->nr_active--;
Peter Zijlstra0f5a2602011-11-16 14:38:16 +01001133 if (event->attr.freq && event->attr.sample_freq)
1134 ctx->nr_freq--;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001135 if (event->attr.exclusive || !cpuctx->active_oncpu)
1136 cpuctx->exclusive = 0;
1137}
1138
1139static void
1140group_sched_out(struct perf_event *group_event,
1141 struct perf_cpu_context *cpuctx,
1142 struct perf_event_context *ctx)
1143{
1144 struct perf_event *event;
Stephane Eranianfa66f072010-08-26 16:40:01 +02001145 int state = group_event->state;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001146
1147 event_sched_out(group_event, cpuctx, ctx);
1148
1149 /*
1150 * Schedule out siblings (if any):
1151 */
1152 list_for_each_entry(event, &group_event->sibling_list, group_entry)
1153 event_sched_out(event, cpuctx, ctx);
1154
Stephane Eranianfa66f072010-08-26 16:40:01 +02001155 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001156 cpuctx->exclusive = 0;
1157}
1158
1159/*
1160 * Cross CPU call to remove a performance event
1161 *
1162 * We disable the event on the hardware level first. After that we
1163 * remove it from the context list.
1164 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001165static int __perf_remove_from_context(void *info)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001166{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001167 struct perf_event *event = info;
1168 struct perf_event_context *ctx = event->ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001169 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001170
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001171 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001172 event_sched_out(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001173 list_del_event(event, ctx);
Peter Zijlstra64ce3122011-04-09 21:17:48 +02001174 if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1175 ctx->is_active = 0;
1176 cpuctx->task_ctx = NULL;
1177 }
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001178 raw_spin_unlock(&ctx->lock);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001179
1180 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001181}
1182
1183
1184/*
1185 * Remove the event from a task's (or a CPU's) list of events.
1186 *
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001187 * CPU events are removed with a smp call. For task events we only
1188 * call when the task is on a CPU.
1189 *
1190 * If event->ctx is a cloned context, callers must make sure that
1191 * every task struct that event->ctx->task could possibly point to
1192 * remains valid. This is OK when called from perf_release since
1193 * that only calls us on the top-level context, which can't be a clone.
1194 * When called from perf_event_exit_task, it's OK because the
1195 * context has been detached from its task.
1196 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001197static void perf_remove_from_context(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001198{
1199 struct perf_event_context *ctx = event->ctx;
1200 struct task_struct *task = ctx->task;
1201
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001202 lockdep_assert_held(&ctx->mutex);
1203
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001204 if (!task) {
1205 /*
1206 * Per cpu events are removed via an smp call and
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001207 * the removal is always successful.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001208 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001209 cpu_function_call(event->cpu, __perf_remove_from_context, event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001210 return;
1211 }
1212
1213retry:
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001214 if (!task_function_call(task, __perf_remove_from_context, event))
1215 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001216
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001217 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001218 /*
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001219 * If we failed to find a running task, but find the context active now
1220 * that we've acquired the ctx->lock, retry.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001221 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001222 if (ctx->is_active) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001223 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001224 goto retry;
1225 }
1226
1227 /*
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001228 * Since the task isn't running, its safe to remove the event, us
1229 * holding the ctx->lock ensures the task won't get scheduled in.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001230 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001231 list_del_event(event, ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001232 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001233}
1234
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001235/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001236 * Cross CPU call to disable a performance event
1237 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001238static int __perf_event_disable(void *info)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001239{
1240 struct perf_event *event = info;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001241 struct perf_event_context *ctx = event->ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001242 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001243
1244 /*
1245 * If this is a per-task event, need to check whether this
1246 * event's task is the current task on this cpu.
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001247 *
1248 * Can trigger due to concurrent perf_event_context_sched_out()
1249 * flipping contexts around.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001250 */
1251 if (ctx->task && cpuctx->task_ctx != ctx)
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001252 return -EINVAL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001253
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001254 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001255
1256 /*
1257 * If the event is on, turn it off.
1258 * If it is in error state, leave it in error state.
1259 */
1260 if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1261 update_context_time(ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02001262 update_cgrp_time_from_event(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001263 update_group_times(event);
1264 if (event == event->group_leader)
1265 group_sched_out(event, cpuctx, ctx);
1266 else
1267 event_sched_out(event, cpuctx, ctx);
1268 event->state = PERF_EVENT_STATE_OFF;
1269 }
1270
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001271 raw_spin_unlock(&ctx->lock);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001272
1273 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001274}
1275
1276/*
1277 * Disable a event.
1278 *
1279 * If event->ctx is a cloned context, callers must make sure that
1280 * every task struct that event->ctx->task could possibly point to
1281 * remains valid. This condition is satisifed when called through
1282 * perf_event_for_each_child or perf_event_for_each because they
1283 * hold the top-level event's child_mutex, so any descendant that
1284 * goes to exit will block in sync_child_event.
1285 * When called from perf_pending_event it's OK because event->ctx
1286 * is the current context on this CPU and preemption is disabled,
1287 * hence we can't get into perf_event_task_sched_out for this context.
1288 */
Frederic Weisbecker44234ad2009-12-09 09:25:48 +01001289void perf_event_disable(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001290{
1291 struct perf_event_context *ctx = event->ctx;
1292 struct task_struct *task = ctx->task;
1293
1294 if (!task) {
1295 /*
1296 * Disable the event on the cpu that it's on
1297 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001298 cpu_function_call(event->cpu, __perf_event_disable, event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001299 return;
1300 }
1301
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001302retry:
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001303 if (!task_function_call(task, __perf_event_disable, event))
1304 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001305
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001306 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001307 /*
1308 * If the event is still active, we need to retry the cross-call.
1309 */
1310 if (event->state == PERF_EVENT_STATE_ACTIVE) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001311 raw_spin_unlock_irq(&ctx->lock);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001312 /*
1313 * Reload the task pointer, it might have been changed by
1314 * a concurrent perf_event_context_sched_out().
1315 */
1316 task = ctx->task;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001317 goto retry;
1318 }
1319
1320 /*
1321 * Since we have the lock this context can't be scheduled
1322 * in, so we can change the state safely.
1323 */
1324 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1325 update_group_times(event);
1326 event->state = PERF_EVENT_STATE_OFF;
1327 }
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001328 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001329}
Robert Richterdcfce4a2011-10-11 17:11:08 +02001330EXPORT_SYMBOL_GPL(perf_event_disable);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001331
Stephane Eraniane5d13672011-02-14 11:20:01 +02001332static void perf_set_shadow_time(struct perf_event *event,
1333 struct perf_event_context *ctx,
1334 u64 tstamp)
1335{
1336 /*
1337 * use the correct time source for the time snapshot
1338 *
1339 * We could get by without this by leveraging the
1340 * fact that to get to this function, the caller
1341 * has most likely already called update_context_time()
1342 * and update_cgrp_time_xx() and thus both timestamp
1343 * are identical (or very close). Given that tstamp is,
1344 * already adjusted for cgroup, we could say that:
1345 * tstamp - ctx->timestamp
1346 * is equivalent to
1347 * tstamp - cgrp->timestamp.
1348 *
1349 * Then, in perf_output_read(), the calculation would
1350 * work with no changes because:
1351 * - event is guaranteed scheduled in
1352 * - no scheduled out in between
1353 * - thus the timestamp would be the same
1354 *
1355 * But this is a bit hairy.
1356 *
1357 * So instead, we have an explicit cgroup call to remain
1358 * within the time time source all along. We believe it
1359 * is cleaner and simpler to understand.
1360 */
1361 if (is_cgroup_event(event))
1362 perf_cgroup_set_shadow_time(event, tstamp);
1363 else
1364 event->shadow_ctx_time = tstamp - ctx->timestamp;
1365}
1366
Peter Zijlstra4fe757d2011-02-15 22:26:07 +01001367#define MAX_INTERRUPTS (~0ULL)
1368
1369static void perf_log_throttle(struct perf_event *event, int enable);
1370
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001371static int
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001372event_sched_in(struct perf_event *event,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001373 struct perf_cpu_context *cpuctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01001374 struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001375{
Stephane Eranian41587552011-01-03 18:20:01 +02001376 u64 tstamp = perf_event_time(event);
1377
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001378 if (event->state <= PERF_EVENT_STATE_OFF)
1379 return 0;
1380
1381 event->state = PERF_EVENT_STATE_ACTIVE;
Peter Zijlstra6e377382010-02-11 13:21:58 +01001382 event->oncpu = smp_processor_id();
Peter Zijlstra4fe757d2011-02-15 22:26:07 +01001383
1384 /*
1385 * Unthrottle events, since we scheduled we might have missed several
1386 * ticks already, also for a heavily scheduling task there is little
1387 * guarantee it'll get a tick in a timely manner.
1388 */
1389 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1390 perf_log_throttle(event, 1);
1391 event->hw.interrupts = 0;
1392 }
1393
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001394 /*
1395 * The new state must be visible before we turn it on in the hardware:
1396 */
1397 smp_wmb();
1398
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02001399 if (event->pmu->add(event, PERF_EF_START)) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001400 event->state = PERF_EVENT_STATE_INACTIVE;
1401 event->oncpu = -1;
1402 return -EAGAIN;
1403 }
1404
Stephane Eranian41587552011-01-03 18:20:01 +02001405 event->tstamp_running += tstamp - event->tstamp_stopped;
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001406
Stephane Eraniane5d13672011-02-14 11:20:01 +02001407 perf_set_shadow_time(event, ctx, tstamp);
Stephane Eranianeed01522010-10-26 16:08:01 +02001408
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001409 if (!is_software_event(event))
1410 cpuctx->active_oncpu++;
1411 ctx->nr_active++;
Peter Zijlstra0f5a2602011-11-16 14:38:16 +01001412 if (event->attr.freq && event->attr.sample_freq)
1413 ctx->nr_freq++;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001414
1415 if (event->attr.exclusive)
1416 cpuctx->exclusive = 1;
1417
1418 return 0;
1419}
1420
1421static int
1422group_sched_in(struct perf_event *group_event,
1423 struct perf_cpu_context *cpuctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01001424 struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001425{
Lin Ming6bde9b62010-04-23 13:56:00 +08001426 struct perf_event *event, *partial_group = NULL;
Peter Zijlstra51b0fe32010-06-11 13:35:57 +02001427 struct pmu *pmu = group_event->pmu;
Stephane Eraniand7842da2010-10-20 15:25:01 +02001428 u64 now = ctx->time;
1429 bool simulate = false;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001430
1431 if (group_event->state == PERF_EVENT_STATE_OFF)
1432 return 0;
1433
Peter Zijlstraad5133b2010-06-15 12:22:39 +02001434 pmu->start_txn(pmu);
Lin Ming6bde9b62010-04-23 13:56:00 +08001435
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001436 if (event_sched_in(group_event, cpuctx, ctx)) {
Peter Zijlstraad5133b2010-06-15 12:22:39 +02001437 pmu->cancel_txn(pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001438 return -EAGAIN;
Stephane Eranian90151c32010-05-25 16:23:10 +02001439 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001440
1441 /*
1442 * Schedule in siblings as one group (if any):
1443 */
1444 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001445 if (event_sched_in(event, cpuctx, ctx)) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001446 partial_group = event;
1447 goto group_error;
1448 }
1449 }
1450
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001451 if (!pmu->commit_txn(pmu))
Paul Mackerras6e851582010-05-08 20:58:00 +10001452 return 0;
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001453
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001454group_error:
1455 /*
1456 * Groups can be scheduled in as one unit only, so undo any
1457 * partial group before returning:
Stephane Eraniand7842da2010-10-20 15:25:01 +02001458 * The events up to the failed event are scheduled out normally,
1459 * tstamp_stopped will be updated.
1460 *
1461 * The failed events and the remaining siblings need to have
1462 * their timings updated as if they had gone thru event_sched_in()
1463 * and event_sched_out(). This is required to get consistent timings
1464 * across the group. This also takes care of the case where the group
1465 * could never be scheduled by ensuring tstamp_stopped is set to mark
1466 * the time the event was actually stopped, such that time delta
1467 * calculation in update_event_times() is correct.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001468 */
1469 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1470 if (event == partial_group)
Stephane Eraniand7842da2010-10-20 15:25:01 +02001471 simulate = true;
1472
1473 if (simulate) {
1474 event->tstamp_running += now - event->tstamp_stopped;
1475 event->tstamp_stopped = now;
1476 } else {
1477 event_sched_out(event, cpuctx, ctx);
1478 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001479 }
Stephane Eranian9ffcfa62010-10-20 15:25:01 +02001480 event_sched_out(group_event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001481
Peter Zijlstraad5133b2010-06-15 12:22:39 +02001482 pmu->cancel_txn(pmu);
Stephane Eranian90151c32010-05-25 16:23:10 +02001483
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001484 return -EAGAIN;
1485}
1486
1487/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001488 * Work out whether we can put this event group on the CPU now.
1489 */
1490static int group_can_go_on(struct perf_event *event,
1491 struct perf_cpu_context *cpuctx,
1492 int can_add_hw)
1493{
1494 /*
1495 * Groups consisting entirely of software events can always go on.
1496 */
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +01001497 if (event->group_flags & PERF_GROUP_SOFTWARE)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001498 return 1;
1499 /*
1500 * If an exclusive group is already on, no other hardware
1501 * events can go on.
1502 */
1503 if (cpuctx->exclusive)
1504 return 0;
1505 /*
1506 * If this group is exclusive and there are already
1507 * events on the CPU, it can't go on.
1508 */
1509 if (event->attr.exclusive && cpuctx->active_oncpu)
1510 return 0;
1511 /*
1512 * Otherwise, try to add it if all previous groups were able
1513 * to go on.
1514 */
1515 return can_add_hw;
1516}
1517
1518static void add_event_to_ctx(struct perf_event *event,
1519 struct perf_event_context *ctx)
1520{
Stephane Eranian41587552011-01-03 18:20:01 +02001521 u64 tstamp = perf_event_time(event);
1522
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001523 list_add_event(event, ctx);
Peter Zijlstra8a495422010-05-27 15:47:49 +02001524 perf_group_attach(event);
Stephane Eranian41587552011-01-03 18:20:01 +02001525 event->tstamp_enabled = tstamp;
1526 event->tstamp_running = tstamp;
1527 event->tstamp_stopped = tstamp;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001528}
1529
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001530static void task_ctx_sched_out(struct perf_event_context *ctx);
1531static void
1532ctx_sched_in(struct perf_event_context *ctx,
1533 struct perf_cpu_context *cpuctx,
1534 enum event_type_t event_type,
1535 struct task_struct *task);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001536
Peter Zijlstradce58552011-04-09 21:17:46 +02001537static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1538 struct perf_event_context *ctx,
1539 struct task_struct *task)
1540{
1541 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1542 if (ctx)
1543 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1544 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1545 if (ctx)
1546 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1547}
1548
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001549/*
1550 * Cross CPU call to install and enable a performance event
1551 *
1552 * Must be called with ctx->mutex held
1553 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001554static int __perf_install_in_context(void *info)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001555{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001556 struct perf_event *event = info;
1557 struct perf_event_context *ctx = event->ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001558 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001559 struct perf_event_context *task_ctx = cpuctx->task_ctx;
1560 struct task_struct *task = current;
1561
Peter Zijlstrab58f6b02011-06-07 00:23:28 +02001562 perf_ctx_lock(cpuctx, task_ctx);
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001563 perf_pmu_disable(cpuctx->ctx.pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001564
1565 /*
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001566 * If there was an active task_ctx schedule it out.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001567 */
Peter Zijlstrab58f6b02011-06-07 00:23:28 +02001568 if (task_ctx)
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001569 task_ctx_sched_out(task_ctx);
Peter Zijlstrab58f6b02011-06-07 00:23:28 +02001570
1571 /*
1572 * If the context we're installing events in is not the
1573 * active task_ctx, flip them.
1574 */
1575 if (ctx->task && task_ctx != ctx) {
1576 if (task_ctx)
1577 raw_spin_unlock(&task_ctx->lock);
1578 raw_spin_lock(&ctx->lock);
1579 task_ctx = ctx;
1580 }
1581
1582 if (task_ctx) {
1583 cpuctx->task_ctx = task_ctx;
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001584 task = task_ctx->task;
1585 }
Peter Zijlstrab58f6b02011-06-07 00:23:28 +02001586
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001587 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001588
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001589 update_context_time(ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02001590 /*
1591 * update cgrp time only if current cgrp
1592 * matches event->cgrp. Must be done before
1593 * calling add_event_to_ctx()
1594 */
1595 update_cgrp_time_from_event(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001596
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001597 add_event_to_ctx(event, ctx);
1598
1599 /*
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001600 * Schedule everything back in
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001601 */
Peter Zijlstradce58552011-04-09 21:17:46 +02001602 perf_event_sched_in(cpuctx, task_ctx, task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001603
Peter Zijlstra2c29ef02011-04-09 21:17:44 +02001604 perf_pmu_enable(cpuctx->ctx.pmu);
1605 perf_ctx_unlock(cpuctx, task_ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001606
1607 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001608}
1609
1610/*
1611 * Attach a performance event to a context
1612 *
1613 * First we add the event to the list with the hardware enable bit
1614 * in event->hw_config cleared.
1615 *
1616 * If the event is attached to a task which is on a CPU we use a smp
1617 * call to enable it in the task context. The task might have been
1618 * scheduled away, but we check this in the smp call again.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001619 */
1620static void
1621perf_install_in_context(struct perf_event_context *ctx,
1622 struct perf_event *event,
1623 int cpu)
1624{
1625 struct task_struct *task = ctx->task;
1626
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001627 lockdep_assert_held(&ctx->mutex);
1628
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02001629 event->ctx = ctx;
1630
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001631 if (!task) {
1632 /*
1633 * Per cpu events are installed via an smp call and
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001634 * the install is always successful.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001635 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001636 cpu_function_call(cpu, __perf_install_in_context, event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001637 return;
1638 }
1639
1640retry:
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001641 if (!task_function_call(task, __perf_install_in_context, event))
1642 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001643
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001644 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001645 /*
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001646 * If we failed to find a running task, but find the context active now
1647 * that we've acquired the ctx->lock, retry.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001648 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001649 if (ctx->is_active) {
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001650 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001651 goto retry;
1652 }
1653
1654 /*
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001655 * Since the task isn't running, its safe to add the event, us holding
1656 * the ctx->lock ensures the task won't get scheduled in.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001657 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001658 add_event_to_ctx(event, ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001659 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001660}
1661
1662/*
1663 * Put a event into inactive state and update time fields.
1664 * Enabling the leader of a group effectively enables all
1665 * the group members that aren't explicitly disabled, so we
1666 * have to update their ->tstamp_enabled also.
1667 * Note: this works for group members as well as group leaders
1668 * since the non-leader members' sibling_lists will be empty.
1669 */
Peter Zijlstra1d9b4822011-11-23 12:34:20 +01001670static void __perf_event_mark_enabled(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001671{
1672 struct perf_event *sub;
Stephane Eranian41587552011-01-03 18:20:01 +02001673 u64 tstamp = perf_event_time(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001674
1675 event->state = PERF_EVENT_STATE_INACTIVE;
Stephane Eranian41587552011-01-03 18:20:01 +02001676 event->tstamp_enabled = tstamp - event->total_time_enabled;
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001677 list_for_each_entry(sub, &event->sibling_list, group_entry) {
Stephane Eranian41587552011-01-03 18:20:01 +02001678 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1679 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001680 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001681}
1682
1683/*
1684 * Cross CPU call to enable a performance event
1685 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001686static int __perf_event_enable(void *info)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001687{
1688 struct perf_event *event = info;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001689 struct perf_event_context *ctx = event->ctx;
1690 struct perf_event *leader = event->group_leader;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001691 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001692 int err;
1693
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001694 if (WARN_ON_ONCE(!ctx->is_active))
1695 return -EINVAL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001696
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001697 raw_spin_lock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001698 update_context_time(ctx);
1699
1700 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1701 goto unlock;
Stephane Eraniane5d13672011-02-14 11:20:01 +02001702
1703 /*
1704 * set current task's cgroup time reference point
1705 */
Stephane Eranian3f7cce32011-02-18 14:40:01 +02001706 perf_cgroup_set_timestamp(current, ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02001707
Peter Zijlstra1d9b4822011-11-23 12:34:20 +01001708 __perf_event_mark_enabled(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001709
Stephane Eraniane5d13672011-02-14 11:20:01 +02001710 if (!event_filter_match(event)) {
1711 if (is_cgroup_event(event))
1712 perf_cgroup_defer_enabled(event);
Peter Zijlstraf4c41762009-12-16 17:55:54 +01001713 goto unlock;
Stephane Eraniane5d13672011-02-14 11:20:01 +02001714 }
Peter Zijlstraf4c41762009-12-16 17:55:54 +01001715
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001716 /*
1717 * If the event is in a group and isn't the group leader,
1718 * then don't put it on unless the group is on.
1719 */
1720 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1721 goto unlock;
1722
1723 if (!group_can_go_on(event, cpuctx, 1)) {
1724 err = -EEXIST;
1725 } else {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001726 if (event == leader)
Peter Zijlstra6e377382010-02-11 13:21:58 +01001727 err = group_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001728 else
Peter Zijlstra6e377382010-02-11 13:21:58 +01001729 err = event_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001730 }
1731
1732 if (err) {
1733 /*
1734 * If this event can't go on and it's part of a
1735 * group, then the whole group has to come off.
1736 */
1737 if (leader != event)
1738 group_sched_out(leader, cpuctx, ctx);
1739 if (leader->attr.pinned) {
1740 update_group_times(leader);
1741 leader->state = PERF_EVENT_STATE_ERROR;
1742 }
1743 }
1744
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001745unlock:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001746 raw_spin_unlock(&ctx->lock);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001747
1748 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001749}
1750
1751/*
1752 * Enable a event.
1753 *
1754 * If event->ctx is a cloned context, callers must make sure that
1755 * every task struct that event->ctx->task could possibly point to
1756 * remains valid. This condition is satisfied when called through
1757 * perf_event_for_each_child or perf_event_for_each as described
1758 * for perf_event_disable.
1759 */
Frederic Weisbecker44234ad2009-12-09 09:25:48 +01001760void perf_event_enable(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001761{
1762 struct perf_event_context *ctx = event->ctx;
1763 struct task_struct *task = ctx->task;
1764
1765 if (!task) {
1766 /*
1767 * Enable the event on the cpu that it's on
1768 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001769 cpu_function_call(event->cpu, __perf_event_enable, event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001770 return;
1771 }
1772
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001773 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001774 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1775 goto out;
1776
1777 /*
1778 * If the event is in error state, clear that first.
1779 * That way, if we see the event in error state below, we
1780 * know that it has gone back into error state, as distinct
1781 * from the task having been scheduled away before the
1782 * cross-call arrived.
1783 */
1784 if (event->state == PERF_EVENT_STATE_ERROR)
1785 event->state = PERF_EVENT_STATE_OFF;
1786
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001787retry:
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001788 if (!ctx->is_active) {
Peter Zijlstra1d9b4822011-11-23 12:34:20 +01001789 __perf_event_mark_enabled(event);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001790 goto out;
1791 }
1792
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001793 raw_spin_unlock_irq(&ctx->lock);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001794
1795 if (!task_function_call(task, __perf_event_enable, event))
1796 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001797
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001798 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001799
1800 /*
1801 * If the context is active and the event is still off,
1802 * we need to retry the cross-call.
1803 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001804 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1805 /*
1806 * task could have been flipped by a concurrent
1807 * perf_event_context_sched_out()
1808 */
1809 task = ctx->task;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001810 goto retry;
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001811 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001812
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001813out:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001814 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001815}
Robert Richterdcfce4a2011-10-11 17:11:08 +02001816EXPORT_SYMBOL_GPL(perf_event_enable);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001817
Avi Kivity26ca5c12011-06-29 18:42:37 +03001818int perf_event_refresh(struct perf_event *event, int refresh)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001819{
1820 /*
1821 * not supported on inherited events
1822 */
Franck Bui-Huu2e939d12010-11-23 16:21:44 +01001823 if (event->attr.inherit || !is_sampling_event(event))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001824 return -EINVAL;
1825
1826 atomic_add(refresh, &event->event_limit);
1827 perf_event_enable(event);
1828
1829 return 0;
1830}
Avi Kivity26ca5c12011-06-29 18:42:37 +03001831EXPORT_SYMBOL_GPL(perf_event_refresh);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001832
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001833static void ctx_sched_out(struct perf_event_context *ctx,
1834 struct perf_cpu_context *cpuctx,
1835 enum event_type_t event_type)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001836{
1837 struct perf_event *event;
Peter Zijlstradb24d332011-04-09 21:17:45 +02001838 int is_active = ctx->is_active;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001839
Peter Zijlstradb24d332011-04-09 21:17:45 +02001840 ctx->is_active &= ~event_type;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001841 if (likely(!ctx->nr_events))
Peter Zijlstrafacc4302011-04-09 21:17:42 +02001842 return;
1843
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001844 update_context_time(ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02001845 update_cgrp_time_from_cpuctx(cpuctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001846 if (!ctx->nr_active)
Peter Zijlstrafacc4302011-04-09 21:17:42 +02001847 return;
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01001848
Peter Zijlstra075e0b02011-04-09 21:17:40 +02001849 perf_pmu_disable(ctx->pmu);
Peter Zijlstradb24d332011-04-09 21:17:45 +02001850 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001851 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1852 group_sched_out(event, cpuctx, ctx);
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001853 }
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001854
Peter Zijlstradb24d332011-04-09 21:17:45 +02001855 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
Frederic Weisbecker889ff012010-01-09 20:04:47 +01001856 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
Xiao Guangrong8c9ed8e2009-09-25 13:51:17 +08001857 group_sched_out(event, cpuctx, ctx);
Peter Zijlstra9ed60602010-06-11 17:36:35 +02001858 }
Peter Zijlstra1b9a6442010-09-07 18:32:22 +02001859 perf_pmu_enable(ctx->pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001860}
1861
1862/*
1863 * Test whether two contexts are equivalent, i.e. whether they
1864 * have both been cloned from the same version of the same context
1865 * and they both have the same number of enabled events.
1866 * If the number of enabled events is the same, then the set
1867 * of enabled events should be the same, because these are both
1868 * inherited contexts, therefore we can't access individual events
1869 * in them directly with an fd; we can only enable/disable all
1870 * events via prctl, or enable/disable all events in a family
1871 * via ioctl, which will have the same effect on both contexts.
1872 */
1873static int context_equiv(struct perf_event_context *ctx1,
1874 struct perf_event_context *ctx2)
1875{
1876 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1877 && ctx1->parent_gen == ctx2->parent_gen
1878 && !ctx1->pin_count && !ctx2->pin_count;
1879}
1880
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001881static void __perf_event_sync_stat(struct perf_event *event,
1882 struct perf_event *next_event)
1883{
1884 u64 value;
1885
1886 if (!event->attr.inherit_stat)
1887 return;
1888
1889 /*
1890 * Update the event value, we cannot use perf_event_read()
1891 * because we're in the middle of a context switch and have IRQs
1892 * disabled, which upsets smp_call_function_single(), however
1893 * we know the event must be on the current CPU, therefore we
1894 * don't need to use it.
1895 */
1896 switch (event->state) {
1897 case PERF_EVENT_STATE_ACTIVE:
Peter Zijlstra3dbebf12009-11-20 22:19:52 +01001898 event->pmu->read(event);
1899 /* fall-through */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001900
1901 case PERF_EVENT_STATE_INACTIVE:
1902 update_event_times(event);
1903 break;
1904
1905 default:
1906 break;
1907 }
1908
1909 /*
1910 * In order to keep per-task stats reliable we need to flip the event
1911 * values when we flip the contexts.
1912 */
Peter Zijlstrae7850592010-05-21 14:43:08 +02001913 value = local64_read(&next_event->count);
1914 value = local64_xchg(&event->count, value);
1915 local64_set(&next_event->count, value);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001916
1917 swap(event->total_time_enabled, next_event->total_time_enabled);
1918 swap(event->total_time_running, next_event->total_time_running);
1919
1920 /*
1921 * Since we swizzled the values, update the user visible data too.
1922 */
1923 perf_event_update_userpage(event);
1924 perf_event_update_userpage(next_event);
1925}
1926
1927#define list_next_entry(pos, member) \
1928 list_entry(pos->member.next, typeof(*pos), member)
1929
1930static void perf_event_sync_stat(struct perf_event_context *ctx,
1931 struct perf_event_context *next_ctx)
1932{
1933 struct perf_event *event, *next_event;
1934
1935 if (!ctx->nr_stat)
1936 return;
1937
Peter Zijlstra02ffdbc2009-11-20 22:19:50 +01001938 update_context_time(ctx);
1939
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001940 event = list_first_entry(&ctx->event_list,
1941 struct perf_event, event_entry);
1942
1943 next_event = list_first_entry(&next_ctx->event_list,
1944 struct perf_event, event_entry);
1945
1946 while (&event->event_entry != &ctx->event_list &&
1947 &next_event->event_entry != &next_ctx->event_list) {
1948
1949 __perf_event_sync_stat(event, next_event);
1950
1951 event = list_next_entry(event, event_entry);
1952 next_event = list_next_entry(next_event, event_entry);
1953 }
1954}
1955
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01001956static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1957 struct task_struct *next)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001958{
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02001959 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001960 struct perf_event_context *next_ctx;
1961 struct perf_event_context *parent;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001962 struct perf_cpu_context *cpuctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001963 int do_switch = 1;
1964
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001965 if (likely(!ctx))
1966 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001967
Peter Zijlstra108b02c2010-09-06 14:32:03 +02001968 cpuctx = __get_cpu_context(ctx);
1969 if (!cpuctx->task_ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001970 return;
1971
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001972 rcu_read_lock();
1973 parent = rcu_dereference(ctx->parent_ctx);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02001974 next_ctx = next->perf_event_ctxp[ctxn];
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001975 if (parent && next_ctx &&
1976 rcu_dereference(next_ctx->parent_ctx) == parent) {
1977 /*
1978 * Looks like the two contexts are clones, so we might be
1979 * able to optimize the context switch. We lock both
1980 * contexts and check that they are clones under the
1981 * lock (including re-checking that neither has been
1982 * uncloned in the meantime). It doesn't matter which
1983 * order we take the locks because no other cpu could
1984 * be trying to lock both of these tasks.
1985 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +01001986 raw_spin_lock(&ctx->lock);
1987 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001988 if (context_equiv(ctx, next_ctx)) {
1989 /*
1990 * XXX do we need a memory barrier of sorts
1991 * wrt to rcu_dereference() of perf_event_ctxp
1992 */
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02001993 task->perf_event_ctxp[ctxn] = next_ctx;
1994 next->perf_event_ctxp[ctxn] = ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001995 ctx->task = next;
1996 next_ctx->task = task;
1997 do_switch = 0;
1998
1999 perf_event_sync_stat(ctx, next_ctx);
2000 }
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002001 raw_spin_unlock(&next_ctx->lock);
2002 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002003 }
2004 rcu_read_unlock();
2005
2006 if (do_switch) {
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002007 raw_spin_lock(&ctx->lock);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002008 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002009 cpuctx->task_ctx = NULL;
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002010 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002011 }
2012}
2013
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002014#define for_each_task_context_nr(ctxn) \
2015 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2016
2017/*
2018 * Called from scheduler to remove the events of the current task,
2019 * with interrupts disabled.
2020 *
2021 * We stop each event and update the event value in event->count.
2022 *
2023 * This does not protect us against NMI, but disable()
2024 * sets the disabled bit in the control field of event _before_
2025 * accessing the event control register. If a NMI hits, then it will
2026 * not restart the event.
2027 */
Peter Zijlstra82cd6de2010-10-14 17:57:23 +02002028void __perf_event_task_sched_out(struct task_struct *task,
2029 struct task_struct *next)
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002030{
2031 int ctxn;
2032
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002033 for_each_task_context_nr(ctxn)
2034 perf_event_context_sched_out(task, ctxn, next);
Stephane Eraniane5d13672011-02-14 11:20:01 +02002035
2036 /*
2037 * if cgroup events exist on this CPU, then we need
2038 * to check if we have to switch out PMU state.
2039 * cgroup event are system-wide mode only
2040 */
2041 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
Stephane Eraniana8d757e2011-08-25 15:58:03 +02002042 perf_cgroup_sched_out(task, next);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002043}
2044
Peter Zijlstra04dc2db2011-04-09 21:17:43 +02002045static void task_ctx_sched_out(struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002046{
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002047 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002048
2049 if (!cpuctx->task_ctx)
2050 return;
2051
2052 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2053 return;
2054
Peter Zijlstra04dc2db2011-04-09 21:17:43 +02002055 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002056 cpuctx->task_ctx = NULL;
2057}
2058
2059/*
2060 * Called with IRQs disabled
2061 */
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002062static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2063 enum event_type_t event_type)
2064{
2065 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002066}
2067
2068static void
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002069ctx_pinned_sched_in(struct perf_event_context *ctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01002070 struct perf_cpu_context *cpuctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002071{
2072 struct perf_event *event;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002073
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002074 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2075 if (event->state <= PERF_EVENT_STATE_OFF)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002076 continue;
Stephane Eranian5632ab12011-01-03 18:20:01 +02002077 if (!event_filter_match(event))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002078 continue;
2079
Stephane Eraniane5d13672011-02-14 11:20:01 +02002080 /* may need to reset tstamp_enabled */
2081 if (is_cgroup_event(event))
2082 perf_cgroup_mark_enabled(event, ctx);
2083
Xiao Guangrong8c9ed8e2009-09-25 13:51:17 +08002084 if (group_can_go_on(event, cpuctx, 1))
Peter Zijlstra6e377382010-02-11 13:21:58 +01002085 group_sched_in(event, cpuctx, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002086
2087 /*
2088 * If this pinned group hasn't been scheduled,
2089 * put it in error state.
2090 */
2091 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2092 update_group_times(event);
2093 event->state = PERF_EVENT_STATE_ERROR;
2094 }
2095 }
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002096}
2097
2098static void
2099ctx_flexible_sched_in(struct perf_event_context *ctx,
Peter Zijlstra6e377382010-02-11 13:21:58 +01002100 struct perf_cpu_context *cpuctx)
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002101{
2102 struct perf_event *event;
2103 int can_add_hw = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002104
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002105 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2106 /* Ignore events in OFF or ERROR state */
2107 if (event->state <= PERF_EVENT_STATE_OFF)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002108 continue;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002109 /*
2110 * Listen to the 'cpu' scheduling filter constraint
2111 * of events:
2112 */
Stephane Eranian5632ab12011-01-03 18:20:01 +02002113 if (!event_filter_match(event))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002114 continue;
2115
Stephane Eraniane5d13672011-02-14 11:20:01 +02002116 /* may need to reset tstamp_enabled */
2117 if (is_cgroup_event(event))
2118 perf_cgroup_mark_enabled(event, ctx);
2119
Peter Zijlstra9ed60602010-06-11 17:36:35 +02002120 if (group_can_go_on(event, cpuctx, can_add_hw)) {
Peter Zijlstra6e377382010-02-11 13:21:58 +01002121 if (group_sched_in(event, cpuctx, ctx))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002122 can_add_hw = 0;
Peter Zijlstra9ed60602010-06-11 17:36:35 +02002123 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002124 }
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002125}
2126
2127static void
2128ctx_sched_in(struct perf_event_context *ctx,
2129 struct perf_cpu_context *cpuctx,
Stephane Eraniane5d13672011-02-14 11:20:01 +02002130 enum event_type_t event_type,
2131 struct task_struct *task)
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002132{
Stephane Eraniane5d13672011-02-14 11:20:01 +02002133 u64 now;
Peter Zijlstradb24d332011-04-09 21:17:45 +02002134 int is_active = ctx->is_active;
Stephane Eraniane5d13672011-02-14 11:20:01 +02002135
Peter Zijlstradb24d332011-04-09 21:17:45 +02002136 ctx->is_active |= event_type;
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002137 if (likely(!ctx->nr_events))
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002138 return;
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002139
Stephane Eraniane5d13672011-02-14 11:20:01 +02002140 now = perf_clock();
2141 ctx->timestamp = now;
Stephane Eranian3f7cce32011-02-18 14:40:01 +02002142 perf_cgroup_set_timestamp(task, ctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002143 /*
2144 * First go through the list and put on any pinned groups
2145 * in order to give them the best chance of going on.
2146 */
Peter Zijlstradb24d332011-04-09 21:17:45 +02002147 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
Peter Zijlstra6e377382010-02-11 13:21:58 +01002148 ctx_pinned_sched_in(ctx, cpuctx);
Frederic Weisbecker5b0311e2010-01-17 11:59:13 +01002149
2150 /* Then walk through the lower prio flexible groups */
Peter Zijlstradb24d332011-04-09 21:17:45 +02002151 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
Peter Zijlstra6e377382010-02-11 13:21:58 +01002152 ctx_flexible_sched_in(ctx, cpuctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002153}
2154
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01002155static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
Stephane Eraniane5d13672011-02-14 11:20:01 +02002156 enum event_type_t event_type,
2157 struct task_struct *task)
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01002158{
2159 struct perf_event_context *ctx = &cpuctx->ctx;
2160
Stephane Eraniane5d13672011-02-14 11:20:01 +02002161 ctx_sched_in(ctx, cpuctx, event_type, task);
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01002162}
2163
Stephane Eraniane5d13672011-02-14 11:20:01 +02002164static void perf_event_context_sched_in(struct perf_event_context *ctx,
2165 struct task_struct *task)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002166{
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002167 struct perf_cpu_context *cpuctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002168
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002169 cpuctx = __get_cpu_context(ctx);
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01002170 if (cpuctx->task_ctx == ctx)
2171 return;
2172
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002173 perf_ctx_lock(cpuctx, ctx);
Peter Zijlstra1b9a6442010-09-07 18:32:22 +02002174 perf_pmu_disable(ctx->pmu);
Frederic Weisbecker329c0e02010-01-17 12:56:05 +01002175 /*
2176 * We want to keep the following priority order:
2177 * cpu pinned (that don't need to move), task pinned,
2178 * cpu flexible, task flexible.
2179 */
2180 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2181
Gleb Natapov1d5f0032011-10-23 19:10:33 +02002182 if (ctx->nr_events)
2183 cpuctx->task_ctx = ctx;
eranian@google.com9b33fa62010-03-10 22:26:05 -08002184
Gleb Natapov86b47c22011-11-22 16:08:21 +02002185 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2186
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002187 perf_pmu_enable(ctx->pmu);
2188 perf_ctx_unlock(cpuctx, ctx);
2189
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002190 /*
2191 * Since these rotations are per-cpu, we need to ensure the
2192 * cpu-context we got scheduled on is actually rotating.
2193 */
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002194 perf_pmu_rotate_start(ctx->pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002195}
2196
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002197/*
2198 * Called from scheduler to add the events of the current task
2199 * with interrupts disabled.
2200 *
2201 * We restore the event value and then enable it.
2202 *
2203 * This does not protect us against NMI, but enable()
2204 * sets the enabled bit in the control field of event _before_
2205 * accessing the event control register. If a NMI hits, then it will
2206 * keep the event running.
2207 */
Stephane Eraniana8d757e2011-08-25 15:58:03 +02002208void __perf_event_task_sched_in(struct task_struct *prev,
2209 struct task_struct *task)
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002210{
2211 struct perf_event_context *ctx;
2212 int ctxn;
2213
2214 for_each_task_context_nr(ctxn) {
2215 ctx = task->perf_event_ctxp[ctxn];
2216 if (likely(!ctx))
2217 continue;
2218
Stephane Eraniane5d13672011-02-14 11:20:01 +02002219 perf_event_context_sched_in(ctx, task);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002220 }
Stephane Eraniane5d13672011-02-14 11:20:01 +02002221 /*
2222 * if cgroup events exist on this CPU, then we need
2223 * to check if we have to switch in PMU state.
2224 * cgroup event are system-wide mode only
2225 */
2226 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
Stephane Eraniana8d757e2011-08-25 15:58:03 +02002227 perf_cgroup_sched_in(prev, task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002228}
2229
Peter Zijlstraabd50712010-01-26 18:50:16 +01002230static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2231{
2232 u64 frequency = event->attr.sample_freq;
2233 u64 sec = NSEC_PER_SEC;
2234 u64 divisor, dividend;
2235
2236 int count_fls, nsec_fls, frequency_fls, sec_fls;
2237
2238 count_fls = fls64(count);
2239 nsec_fls = fls64(nsec);
2240 frequency_fls = fls64(frequency);
2241 sec_fls = 30;
2242
2243 /*
2244 * We got @count in @nsec, with a target of sample_freq HZ
2245 * the target period becomes:
2246 *
2247 * @count * 10^9
2248 * period = -------------------
2249 * @nsec * sample_freq
2250 *
2251 */
2252
2253 /*
2254 * Reduce accuracy by one bit such that @a and @b converge
2255 * to a similar magnitude.
2256 */
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002257#define REDUCE_FLS(a, b) \
Peter Zijlstraabd50712010-01-26 18:50:16 +01002258do { \
2259 if (a##_fls > b##_fls) { \
2260 a >>= 1; \
2261 a##_fls--; \
2262 } else { \
2263 b >>= 1; \
2264 b##_fls--; \
2265 } \
2266} while (0)
2267
2268 /*
2269 * Reduce accuracy until either term fits in a u64, then proceed with
2270 * the other, so that finally we can do a u64/u64 division.
2271 */
2272 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2273 REDUCE_FLS(nsec, frequency);
2274 REDUCE_FLS(sec, count);
2275 }
2276
2277 if (count_fls + sec_fls > 64) {
2278 divisor = nsec * frequency;
2279
2280 while (count_fls + sec_fls > 64) {
2281 REDUCE_FLS(count, sec);
2282 divisor >>= 1;
2283 }
2284
2285 dividend = count * sec;
2286 } else {
2287 dividend = count * sec;
2288
2289 while (nsec_fls + frequency_fls > 64) {
2290 REDUCE_FLS(nsec, frequency);
2291 dividend >>= 1;
2292 }
2293
2294 divisor = nsec * frequency;
2295 }
2296
Peter Zijlstraf6ab91a2010-06-04 15:18:01 +02002297 if (!divisor)
2298 return dividend;
2299
Peter Zijlstraabd50712010-01-26 18:50:16 +01002300 return div64_u64(dividend, divisor);
2301}
2302
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002303static DEFINE_PER_CPU(int, perf_throttled_count);
2304static DEFINE_PER_CPU(u64, perf_throttled_seq);
2305
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002306static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002307{
2308 struct hw_perf_event *hwc = &event->hw;
Peter Zijlstraf6ab91a2010-06-04 15:18:01 +02002309 s64 period, sample_period;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002310 s64 delta;
2311
Peter Zijlstraabd50712010-01-26 18:50:16 +01002312 period = perf_calculate_period(event, nsec, count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002313
2314 delta = (s64)(period - hwc->sample_period);
2315 delta = (delta + 7) / 8; /* low pass filter */
2316
2317 sample_period = hwc->sample_period + delta;
2318
2319 if (!sample_period)
2320 sample_period = 1;
2321
2322 hwc->sample_period = sample_period;
Peter Zijlstraabd50712010-01-26 18:50:16 +01002323
Peter Zijlstrae7850592010-05-21 14:43:08 +02002324 if (local64_read(&hwc->period_left) > 8*sample_period) {
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002325 if (disable)
2326 event->pmu->stop(event, PERF_EF_UPDATE);
2327
Peter Zijlstrae7850592010-05-21 14:43:08 +02002328 local64_set(&hwc->period_left, 0);
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002329
2330 if (disable)
2331 event->pmu->start(event, PERF_EF_RELOAD);
Peter Zijlstraabd50712010-01-26 18:50:16 +01002332 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002333}
2334
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002335/*
2336 * combine freq adjustment with unthrottling to avoid two passes over the
2337 * events. At the same time, make sure, having freq events does not change
2338 * the rate of unthrottling as that would introduce bias.
2339 */
2340static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2341 int needs_unthr)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002342{
2343 struct perf_event *event;
2344 struct hw_perf_event *hwc;
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002345 u64 now, period = TICK_NSEC;
Peter Zijlstraabd50712010-01-26 18:50:16 +01002346 s64 delta;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002347
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002348 /*
2349 * only need to iterate over all events iff:
2350 * - context have events in frequency mode (needs freq adjust)
2351 * - there are events to unthrottle on this cpu
2352 */
2353 if (!(ctx->nr_freq || needs_unthr))
Peter Zijlstra0f5a2602011-11-16 14:38:16 +01002354 return;
2355
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002356 raw_spin_lock(&ctx->lock);
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002357 perf_pmu_disable(ctx->pmu);
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002358
Paul Mackerras03541f82009-10-14 16:58:03 +11002359 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002360 if (event->state != PERF_EVENT_STATE_ACTIVE)
2361 continue;
2362
Stephane Eranian5632ab12011-01-03 18:20:01 +02002363 if (!event_filter_match(event))
Peter Zijlstra5d27c232009-12-17 13:16:32 +01002364 continue;
2365
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002366 hwc = &event->hw;
2367
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002368 if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2369 hwc->interrupts = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002370 perf_log_throttle(event, 1);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02002371 event->pmu->start(event, 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002372 }
2373
2374 if (!event->attr.freq || !event->attr.sample_freq)
2375 continue;
2376
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002377 /*
2378 * stop the event and update event->count
2379 */
2380 event->pmu->stop(event, PERF_EF_UPDATE);
2381
Peter Zijlstrae7850592010-05-21 14:43:08 +02002382 now = local64_read(&event->count);
Peter Zijlstraabd50712010-01-26 18:50:16 +01002383 delta = now - hwc->freq_count_stamp;
2384 hwc->freq_count_stamp = now;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002385
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002386 /*
2387 * restart the event
2388 * reload only if value has changed
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002389 * we have stopped the event so tell that
2390 * to perf_adjust_period() to avoid stopping it
2391 * twice.
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002392 */
Peter Zijlstraabd50712010-01-26 18:50:16 +01002393 if (delta > 0)
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002394 perf_adjust_period(event, period, delta, false);
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002395
2396 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002397 }
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002398
Stephane Eranianf39d47f2012-02-07 14:39:57 +01002399 perf_pmu_enable(ctx->pmu);
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002400 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002401}
2402
2403/*
2404 * Round-robin a context's events:
2405 */
2406static void rotate_ctx(struct perf_event_context *ctx)
2407{
Thomas Gleixnerdddd3372010-11-24 10:05:55 +01002408 /*
2409 * Rotate the first entry last of non-pinned groups. Rotation might be
2410 * disabled by the inheritance code.
2411 */
2412 if (!ctx->rotate_disable)
2413 list_rotate_left(&ctx->flexible_groups);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002414}
2415
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002416/*
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002417 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2418 * because they're strictly cpu affine and rotate_start is called with IRQs
2419 * disabled, while rotate_context is called from IRQ context.
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002420 */
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002421static void perf_rotate_context(struct perf_cpu_context *cpuctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002422{
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002423 struct perf_event_context *ctx = NULL;
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002424 int rotate = 0, remove = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002425
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002426 if (cpuctx->ctx.nr_events) {
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002427 remove = 0;
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002428 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2429 rotate = 1;
2430 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002431
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002432 ctx = cpuctx->task_ctx;
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002433 if (ctx && ctx->nr_events) {
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002434 remove = 0;
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002435 if (ctx->nr_events != ctx->nr_active)
2436 rotate = 1;
2437 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002438
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002439 if (!rotate)
Peter Zijlstra0f5a2602011-11-16 14:38:16 +01002440 goto done;
2441
Peter Zijlstrafacc4302011-04-09 21:17:42 +02002442 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
Peter Zijlstra1b9a6442010-09-07 18:32:22 +02002443 perf_pmu_disable(cpuctx->ctx.pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002444
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002445 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2446 if (ctx)
2447 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
Peter Zijlstrad4944a02010-03-08 13:51:20 +01002448
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002449 rotate_ctx(&cpuctx->ctx);
2450 if (ctx)
2451 rotate_ctx(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002452
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002453 perf_event_sched_in(cpuctx, ctx, current);
Peter Zijlstra0f5a2602011-11-16 14:38:16 +01002454
2455 perf_pmu_enable(cpuctx->ctx.pmu);
2456 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02002457done:
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002458 if (remove)
2459 list_del_init(&cpuctx->rotation_list);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002460}
2461
2462void perf_event_task_tick(void)
2463{
2464 struct list_head *head = &__get_cpu_var(rotation_list);
2465 struct perf_cpu_context *cpuctx, *tmp;
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002466 struct perf_event_context *ctx;
2467 int throttled;
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002468
2469 WARN_ON(!irqs_disabled());
2470
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002471 __this_cpu_inc(perf_throttled_seq);
2472 throttled = __this_cpu_xchg(perf_throttled_count, 0);
2473
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002474 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
Stephane Eraniane050e3f2012-01-26 17:03:19 +01002475 ctx = &cpuctx->ctx;
2476 perf_adjust_freq_unthr_context(ctx, throttled);
2477
2478 ctx = cpuctx->task_ctx;
2479 if (ctx)
2480 perf_adjust_freq_unthr_context(ctx, throttled);
2481
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02002482 if (cpuctx->jiffies_interval == 1 ||
2483 !(jiffies % cpuctx->jiffies_interval))
2484 perf_rotate_context(cpuctx);
2485 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002486}
2487
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002488static int event_enable_on_exec(struct perf_event *event,
2489 struct perf_event_context *ctx)
2490{
2491 if (!event->attr.enable_on_exec)
2492 return 0;
2493
2494 event->attr.enable_on_exec = 0;
2495 if (event->state >= PERF_EVENT_STATE_INACTIVE)
2496 return 0;
2497
Peter Zijlstra1d9b4822011-11-23 12:34:20 +01002498 __perf_event_mark_enabled(event);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002499
2500 return 1;
2501}
2502
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002503/*
2504 * Enable all of a task's events that have been marked enable-on-exec.
2505 * This expects task == current.
2506 */
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002507static void perf_event_enable_on_exec(struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002508{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002509 struct perf_event *event;
2510 unsigned long flags;
2511 int enabled = 0;
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002512 int ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002513
2514 local_irq_save(flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002515 if (!ctx || !ctx->nr_events)
2516 goto out;
2517
Stephane Eraniane566b762011-04-06 02:54:54 +02002518 /*
2519 * We must ctxsw out cgroup events to avoid conflict
2520 * when invoking perf_task_event_sched_in() later on
2521 * in this function. Otherwise we end up trying to
2522 * ctxswin cgroup events which are already scheduled
2523 * in.
2524 */
Stephane Eraniana8d757e2011-08-25 15:58:03 +02002525 perf_cgroup_sched_out(current, NULL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002526
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002527 raw_spin_lock(&ctx->lock);
Peter Zijlstra04dc2db2011-04-09 21:17:43 +02002528 task_ctx_sched_out(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002529
Peter Zijlstrab79387e2011-11-22 11:25:43 +01002530 list_for_each_entry(event, &ctx->event_list, event_entry) {
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002531 ret = event_enable_on_exec(event, ctx);
2532 if (ret)
2533 enabled = 1;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002534 }
2535
2536 /*
2537 * Unclone this context if we enabled any event.
2538 */
2539 if (enabled)
2540 unclone_ctx(ctx);
2541
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002542 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002543
Stephane Eraniane566b762011-04-06 02:54:54 +02002544 /*
2545 * Also calls ctxswin for cgroup events, if any:
2546 */
Stephane Eraniane5d13672011-02-14 11:20:01 +02002547 perf_event_context_sched_in(ctx, ctx->task);
Peter Zijlstra9ed60602010-06-11 17:36:35 +02002548out:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002549 local_irq_restore(flags);
2550}
2551
2552/*
2553 * Cross CPU call to read the hardware event
2554 */
2555static void __perf_event_read(void *info)
2556{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002557 struct perf_event *event = info;
2558 struct perf_event_context *ctx = event->ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002559 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002560
2561 /*
2562 * If this is a task context, we need to check whether it is
2563 * the current task context of this cpu. If not it has been
2564 * scheduled out before the smp call arrived. In that case
2565 * event->count would have been updated to a recent sample
2566 * when the event was scheduled out.
2567 */
2568 if (ctx->task && cpuctx->task_ctx != ctx)
2569 return;
2570
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002571 raw_spin_lock(&ctx->lock);
Stephane Eraniane5d13672011-02-14 11:20:01 +02002572 if (ctx->is_active) {
Peter Zijlstra542e72f2011-01-26 15:38:35 +01002573 update_context_time(ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02002574 update_cgrp_time_from_event(event);
2575 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002576 update_event_times(event);
Peter Zijlstra542e72f2011-01-26 15:38:35 +01002577 if (event->state == PERF_EVENT_STATE_ACTIVE)
2578 event->pmu->read(event);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002579 raw_spin_unlock(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002580}
2581
Peter Zijlstrab5e58792010-05-21 14:43:12 +02002582static inline u64 perf_event_count(struct perf_event *event)
2583{
Peter Zijlstrae7850592010-05-21 14:43:08 +02002584 return local64_read(&event->count) + atomic64_read(&event->child_count);
Peter Zijlstrab5e58792010-05-21 14:43:12 +02002585}
2586
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002587static u64 perf_event_read(struct perf_event *event)
2588{
2589 /*
2590 * If event is enabled and currently active on a CPU, update the
2591 * value in the event structure:
2592 */
2593 if (event->state == PERF_EVENT_STATE_ACTIVE) {
2594 smp_call_function_single(event->oncpu,
2595 __perf_event_read, event, 1);
2596 } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
Peter Zijlstra2b8988c2009-11-20 22:19:54 +01002597 struct perf_event_context *ctx = event->ctx;
2598 unsigned long flags;
2599
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002600 raw_spin_lock_irqsave(&ctx->lock, flags);
Stephane Eranianc530ccd2010-10-15 15:26:01 +02002601 /*
2602 * may read while context is not active
2603 * (e.g., thread is blocked), in that case
2604 * we cannot update context time
2605 */
Stephane Eraniane5d13672011-02-14 11:20:01 +02002606 if (ctx->is_active) {
Stephane Eranianc530ccd2010-10-15 15:26:01 +02002607 update_context_time(ctx);
Stephane Eraniane5d13672011-02-14 11:20:01 +02002608 update_cgrp_time_from_event(event);
2609 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002610 update_event_times(event);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002611 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002612 }
2613
Peter Zijlstrab5e58792010-05-21 14:43:12 +02002614 return perf_event_count(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002615}
2616
2617/*
2618 * Initialize the perf_event context in a task_struct:
2619 */
Peter Zijlstraeb184472010-09-07 15:55:13 +02002620static void __perf_event_init_context(struct perf_event_context *ctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002621{
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002622 raw_spin_lock_init(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002623 mutex_init(&ctx->mutex);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01002624 INIT_LIST_HEAD(&ctx->pinned_groups);
2625 INIT_LIST_HEAD(&ctx->flexible_groups);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002626 INIT_LIST_HEAD(&ctx->event_list);
2627 atomic_set(&ctx->refcount, 1);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002628}
2629
Peter Zijlstraeb184472010-09-07 15:55:13 +02002630static struct perf_event_context *
2631alloc_perf_context(struct pmu *pmu, struct task_struct *task)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002632{
2633 struct perf_event_context *ctx;
Peter Zijlstraeb184472010-09-07 15:55:13 +02002634
2635 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2636 if (!ctx)
2637 return NULL;
2638
2639 __perf_event_init_context(ctx);
2640 if (task) {
2641 ctx->task = task;
2642 get_task_struct(task);
2643 }
2644 ctx->pmu = pmu;
2645
2646 return ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002647}
2648
Matt Helsley2ebd4ff2010-09-13 13:01:19 -07002649static struct task_struct *
2650find_lively_task_by_vpid(pid_t vpid)
2651{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002652 struct task_struct *task;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002653 int err;
2654
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002655 rcu_read_lock();
Matt Helsley2ebd4ff2010-09-13 13:01:19 -07002656 if (!vpid)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002657 task = current;
2658 else
Matt Helsley2ebd4ff2010-09-13 13:01:19 -07002659 task = find_task_by_vpid(vpid);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002660 if (task)
2661 get_task_struct(task);
2662 rcu_read_unlock();
2663
2664 if (!task)
2665 return ERR_PTR(-ESRCH);
2666
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002667 /* Reuse ptrace permission checks for now. */
2668 err = -EACCES;
2669 if (!ptrace_may_access(task, PTRACE_MODE_READ))
2670 goto errout;
2671
Matt Helsley2ebd4ff2010-09-13 13:01:19 -07002672 return task;
2673errout:
2674 put_task_struct(task);
2675 return ERR_PTR(err);
2676
2677}
2678
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002679/*
2680 * Returns a matching context with refcount and pincount.
2681 */
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002682static struct perf_event_context *
Matt Helsley38a81da2010-09-13 13:01:20 -07002683find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002684{
2685 struct perf_event_context *ctx;
2686 struct perf_cpu_context *cpuctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002687 unsigned long flags;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002688 int ctxn, err;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002689
Oleg Nesterov22a4ec72011-01-18 17:10:08 +01002690 if (!task) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002691 /* Must be root to operate on a CPU event: */
2692 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2693 return ERR_PTR(-EACCES);
2694
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002695 /*
2696 * We could be clever and allow to attach a event to an
2697 * offline CPU and activate it when the CPU comes up, but
2698 * that's for later.
2699 */
2700 if (!cpu_online(cpu))
2701 return ERR_PTR(-ENODEV);
2702
Peter Zijlstra108b02c2010-09-06 14:32:03 +02002703 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002704 ctx = &cpuctx->ctx;
2705 get_ctx(ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002706 ++ctx->pin_count;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002707
2708 return ctx;
2709 }
2710
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002711 err = -EINVAL;
2712 ctxn = pmu->task_ctx_nr;
2713 if (ctxn < 0)
2714 goto errout;
2715
Peter Zijlstra9ed60602010-06-11 17:36:35 +02002716retry:
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02002717 ctx = perf_lock_task_context(task, ctxn, &flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002718 if (ctx) {
2719 unclone_ctx(ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002720 ++ctx->pin_count;
Thomas Gleixnere625cce2009-11-17 18:02:06 +01002721 raw_spin_unlock_irqrestore(&ctx->lock, flags);
Peter Zijlstra9137fb22011-04-09 21:17:41 +02002722 } else {
Peter Zijlstraeb184472010-09-07 15:55:13 +02002723 ctx = alloc_perf_context(pmu, task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002724 err = -ENOMEM;
2725 if (!ctx)
2726 goto errout;
Peter Zijlstraeb184472010-09-07 15:55:13 +02002727
Oleg Nesterovdbe08d82011-01-19 19:22:07 +01002728 err = 0;
2729 mutex_lock(&task->perf_event_mutex);
2730 /*
2731 * If it has already passed perf_event_exit_task().
2732 * we must see PF_EXITING, it takes this mutex too.
2733 */
2734 if (task->flags & PF_EXITING)
2735 err = -ESRCH;
2736 else if (task->perf_event_ctxp[ctxn])
2737 err = -EAGAIN;
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002738 else {
Peter Zijlstra9137fb22011-04-09 21:17:41 +02002739 get_ctx(ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002740 ++ctx->pin_count;
Oleg Nesterovdbe08d82011-01-19 19:22:07 +01002741 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01002742 }
Oleg Nesterovdbe08d82011-01-19 19:22:07 +01002743 mutex_unlock(&task->perf_event_mutex);
2744
2745 if (unlikely(err)) {
Peter Zijlstra9137fb22011-04-09 21:17:41 +02002746 put_ctx(ctx);
Oleg Nesterovdbe08d82011-01-19 19:22:07 +01002747
2748 if (err == -EAGAIN)
2749 goto retry;
2750 goto errout;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002751 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002752 }
2753
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002754 return ctx;
2755
Peter Zijlstra9ed60602010-06-11 17:36:35 +02002756errout:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002757 return ERR_PTR(err);
2758}
2759
Li Zefan6fb29152009-10-15 11:21:42 +08002760static void perf_event_free_filter(struct perf_event *event);
2761
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002762static void free_event_rcu(struct rcu_head *head)
2763{
2764 struct perf_event *event;
2765
2766 event = container_of(head, struct perf_event, rcu_head);
2767 if (event->ns)
2768 put_pid_ns(event->ns);
Li Zefan6fb29152009-10-15 11:21:42 +08002769 perf_event_free_filter(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002770 kfree(event);
2771}
2772
Frederic Weisbecker76369132011-05-19 19:55:04 +02002773static void ring_buffer_put(struct ring_buffer *rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002774
2775static void free_event(struct perf_event *event)
2776{
Peter Zijlstrae360adb2010-10-14 14:01:34 +08002777 irq_work_sync(&event->pending);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002778
2779 if (!event->parent) {
Peter Zijlstra82cd6de2010-10-14 17:57:23 +02002780 if (event->attach_state & PERF_ATTACH_TASK)
Gleb Natapovb2029522011-11-27 17:59:09 +02002781 jump_label_dec_deferred(&perf_sched_events);
Eric B Munson3af9e852010-05-18 15:30:49 +01002782 if (event->attr.mmap || event->attr.mmap_data)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002783 atomic_dec(&nr_mmap_events);
2784 if (event->attr.comm)
2785 atomic_dec(&nr_comm_events);
2786 if (event->attr.task)
2787 atomic_dec(&nr_task_events);
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02002788 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2789 put_callchain_buffers();
Peter Zijlstra08309372011-03-03 11:31:20 +01002790 if (is_cgroup_event(event)) {
2791 atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
Gleb Natapovb2029522011-11-27 17:59:09 +02002792 jump_label_dec_deferred(&perf_sched_events);
Peter Zijlstra08309372011-03-03 11:31:20 +01002793 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002794 }
2795
Frederic Weisbecker76369132011-05-19 19:55:04 +02002796 if (event->rb) {
2797 ring_buffer_put(event->rb);
2798 event->rb = NULL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002799 }
2800
Stephane Eraniane5d13672011-02-14 11:20:01 +02002801 if (is_cgroup_event(event))
2802 perf_detach_cgroup(event);
2803
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002804 if (event->destroy)
2805 event->destroy(event);
2806
Peter Zijlstra0c67b402010-09-13 11:15:58 +02002807 if (event->ctx)
2808 put_ctx(event->ctx);
2809
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002810 call_rcu(&event->rcu_head, free_event_rcu);
2811}
2812
Arjan van de Venfb0459d2009-09-25 12:25:56 +02002813int perf_event_release_kernel(struct perf_event *event)
2814{
2815 struct perf_event_context *ctx = event->ctx;
2816
2817 WARN_ON_ONCE(ctx->parent_ctx);
Peter Zijlstraa0507c82010-05-06 15:42:53 +02002818 /*
2819 * There are two ways this annotation is useful:
2820 *
2821 * 1) there is a lock recursion from perf_event_exit_task
2822 * see the comment there.
2823 *
2824 * 2) there is a lock-inversion with mmap_sem through
2825 * perf_event_read_group(), which takes faults while
2826 * holding ctx->mutex, however this is called after
2827 * the last filedesc died, so there is no possibility
2828 * to trigger the AB-BA case.
2829 */
2830 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
Peter Zijlstra050735b2010-05-11 11:51:53 +02002831 raw_spin_lock_irq(&ctx->lock);
Peter Zijlstra8a495422010-05-27 15:47:49 +02002832 perf_group_detach(event);
Peter Zijlstra050735b2010-05-11 11:51:53 +02002833 raw_spin_unlock_irq(&ctx->lock);
Peter Zijlstrae03a9a52011-04-09 21:17:47 +02002834 perf_remove_from_context(event);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02002835 mutex_unlock(&ctx->mutex);
2836
Arjan van de Venfb0459d2009-09-25 12:25:56 +02002837 free_event(event);
2838
2839 return 0;
2840}
2841EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2842
Peter Zijlstraa66a3052009-11-23 11:37:23 +01002843/*
2844 * Called when the last reference to the file is gone.
2845 */
2846static int perf_release(struct inode *inode, struct file *file)
2847{
2848 struct perf_event *event = file->private_data;
Peter Zijlstra88821352010-11-09 19:01:43 +01002849 struct task_struct *owner;
Peter Zijlstraa66a3052009-11-23 11:37:23 +01002850
2851 file->private_data = NULL;
2852
Peter Zijlstra88821352010-11-09 19:01:43 +01002853 rcu_read_lock();
2854 owner = ACCESS_ONCE(event->owner);
2855 /*
2856 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2857 * !owner it means the list deletion is complete and we can indeed
2858 * free this event, otherwise we need to serialize on
2859 * owner->perf_event_mutex.
2860 */
2861 smp_read_barrier_depends();
2862 if (owner) {
2863 /*
2864 * Since delayed_put_task_struct() also drops the last
2865 * task reference we can safely take a new reference
2866 * while holding the rcu_read_lock().
2867 */
2868 get_task_struct(owner);
2869 }
2870 rcu_read_unlock();
2871
2872 if (owner) {
2873 mutex_lock(&owner->perf_event_mutex);
2874 /*
2875 * We have to re-check the event->owner field, if it is cleared
2876 * we raced with perf_event_exit_task(), acquiring the mutex
2877 * ensured they're done, and we can proceed with freeing the
2878 * event.
2879 */
2880 if (event->owner)
2881 list_del_init(&event->owner_entry);
2882 mutex_unlock(&owner->perf_event_mutex);
2883 put_task_struct(owner);
2884 }
2885
Peter Zijlstraa66a3052009-11-23 11:37:23 +01002886 return perf_event_release_kernel(event);
2887}
2888
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002889u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002890{
2891 struct perf_event *child;
2892 u64 total = 0;
2893
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002894 *enabled = 0;
2895 *running = 0;
2896
Peter Zijlstra6f105812009-11-20 22:19:56 +01002897 mutex_lock(&event->child_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002898 total += perf_event_read(event);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002899 *enabled += event->total_time_enabled +
2900 atomic64_read(&event->child_total_time_enabled);
2901 *running += event->total_time_running +
2902 atomic64_read(&event->child_total_time_running);
2903
2904 list_for_each_entry(child, &event->child_list, child_list) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002905 total += perf_event_read(child);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002906 *enabled += child->total_time_enabled;
2907 *running += child->total_time_running;
2908 }
Peter Zijlstra6f105812009-11-20 22:19:56 +01002909 mutex_unlock(&event->child_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002910
2911 return total;
2912}
Arjan van de Venfb0459d2009-09-25 12:25:56 +02002913EXPORT_SYMBOL_GPL(perf_event_read_value);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002914
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002915static int perf_event_read_group(struct perf_event *event,
2916 u64 read_format, char __user *buf)
2917{
2918 struct perf_event *leader = event->group_leader, *sub;
Peter Zijlstra6f105812009-11-20 22:19:56 +01002919 int n = 0, size = 0, ret = -EFAULT;
2920 struct perf_event_context *ctx = leader->ctx;
Peter Zijlstraabf48682009-11-20 22:19:49 +01002921 u64 values[5];
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002922 u64 count, enabled, running;
Peter Zijlstraabf48682009-11-20 22:19:49 +01002923
Peter Zijlstra6f105812009-11-20 22:19:56 +01002924 mutex_lock(&ctx->mutex);
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002925 count = perf_event_read_value(leader, &enabled, &running);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002926
2927 values[n++] = 1 + leader->nr_siblings;
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002928 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2929 values[n++] = enabled;
2930 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2931 values[n++] = running;
Peter Zijlstraabf48682009-11-20 22:19:49 +01002932 values[n++] = count;
2933 if (read_format & PERF_FORMAT_ID)
2934 values[n++] = primary_event_id(leader);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002935
2936 size = n * sizeof(u64);
2937
2938 if (copy_to_user(buf, values, size))
Peter Zijlstra6f105812009-11-20 22:19:56 +01002939 goto unlock;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002940
Peter Zijlstra6f105812009-11-20 22:19:56 +01002941 ret = size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002942
2943 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
Peter Zijlstraabf48682009-11-20 22:19:49 +01002944 n = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002945
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002946 values[n++] = perf_event_read_value(sub, &enabled, &running);
Peter Zijlstraabf48682009-11-20 22:19:49 +01002947 if (read_format & PERF_FORMAT_ID)
2948 values[n++] = primary_event_id(sub);
2949
2950 size = n * sizeof(u64);
2951
Stephane Eranian184d3da2009-11-23 21:40:49 -08002952 if (copy_to_user(buf + ret, values, size)) {
Peter Zijlstra6f105812009-11-20 22:19:56 +01002953 ret = -EFAULT;
2954 goto unlock;
2955 }
Peter Zijlstraabf48682009-11-20 22:19:49 +01002956
2957 ret += size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002958 }
Peter Zijlstra6f105812009-11-20 22:19:56 +01002959unlock:
2960 mutex_unlock(&ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002961
Peter Zijlstraabf48682009-11-20 22:19:49 +01002962 return ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002963}
2964
2965static int perf_event_read_one(struct perf_event *event,
2966 u64 read_format, char __user *buf)
2967{
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002968 u64 enabled, running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002969 u64 values[4];
2970 int n = 0;
2971
Peter Zijlstra59ed4462009-11-20 22:19:55 +01002972 values[n++] = perf_event_read_value(event, &enabled, &running);
2973 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2974 values[n++] = enabled;
2975 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2976 values[n++] = running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002977 if (read_format & PERF_FORMAT_ID)
2978 values[n++] = primary_event_id(event);
2979
2980 if (copy_to_user(buf, values, n * sizeof(u64)))
2981 return -EFAULT;
2982
2983 return n * sizeof(u64);
2984}
2985
2986/*
2987 * Read the performance event - simple non blocking version for now
2988 */
2989static ssize_t
2990perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2991{
2992 u64 read_format = event->attr.read_format;
2993 int ret;
2994
2995 /*
2996 * Return end-of-file for a read on a event that is in
2997 * error state (i.e. because it was pinned but it couldn't be
2998 * scheduled on to the CPU at some point).
2999 */
3000 if (event->state == PERF_EVENT_STATE_ERROR)
3001 return 0;
3002
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02003003 if (count < event->read_size)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003004 return -ENOSPC;
3005
3006 WARN_ON_ONCE(event->ctx->parent_ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003007 if (read_format & PERF_FORMAT_GROUP)
3008 ret = perf_event_read_group(event, read_format, buf);
3009 else
3010 ret = perf_event_read_one(event, read_format, buf);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003011
3012 return ret;
3013}
3014
3015static ssize_t
3016perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3017{
3018 struct perf_event *event = file->private_data;
3019
3020 return perf_read_hw(event, buf, count);
3021}
3022
3023static unsigned int perf_poll(struct file *file, poll_table *wait)
3024{
3025 struct perf_event *event = file->private_data;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003026 struct ring_buffer *rb;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003027 unsigned int events = POLL_HUP;
3028
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003029 /*
3030 * Race between perf_event_set_output() and perf_poll(): perf_poll()
3031 * grabs the rb reference but perf_event_set_output() overrides it.
3032 * Here is the timeline for two threads T1, T2:
3033 * t0: T1, rb = rcu_dereference(event->rb)
3034 * t1: T2, old_rb = event->rb
3035 * t2: T2, event->rb = new rb
3036 * t3: T2, ring_buffer_detach(old_rb)
3037 * t4: T1, ring_buffer_attach(rb1)
3038 * t5: T1, poll_wait(event->waitq)
3039 *
3040 * To avoid this problem, we grab mmap_mutex in perf_poll()
3041 * thereby ensuring that the assignment of the new ring buffer
3042 * and the detachment of the old buffer appear atomic to perf_poll()
3043 */
3044 mutex_lock(&event->mmap_mutex);
3045
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003046 rcu_read_lock();
Frederic Weisbecker76369132011-05-19 19:55:04 +02003047 rb = rcu_dereference(event->rb);
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003048 if (rb) {
3049 ring_buffer_attach(event, rb);
Frederic Weisbecker76369132011-05-19 19:55:04 +02003050 events = atomic_xchg(&rb->poll, 0);
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003051 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003052 rcu_read_unlock();
3053
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003054 mutex_unlock(&event->mmap_mutex);
3055
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003056 poll_wait(file, &event->waitq, wait);
3057
3058 return events;
3059}
3060
3061static void perf_event_reset(struct perf_event *event)
3062{
3063 (void)perf_event_read(event);
Peter Zijlstrae7850592010-05-21 14:43:08 +02003064 local64_set(&event->count, 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003065 perf_event_update_userpage(event);
3066}
3067
3068/*
3069 * Holding the top-level event's child_mutex means that any
3070 * descendant process that has inherited this event will block
3071 * in sync_child_event if it goes to exit, thus satisfying the
3072 * task existence requirements of perf_event_enable/disable.
3073 */
3074static void perf_event_for_each_child(struct perf_event *event,
3075 void (*func)(struct perf_event *))
3076{
3077 struct perf_event *child;
3078
3079 WARN_ON_ONCE(event->ctx->parent_ctx);
3080 mutex_lock(&event->child_mutex);
3081 func(event);
3082 list_for_each_entry(child, &event->child_list, child_list)
3083 func(child);
3084 mutex_unlock(&event->child_mutex);
3085}
3086
3087static void perf_event_for_each(struct perf_event *event,
3088 void (*func)(struct perf_event *))
3089{
3090 struct perf_event_context *ctx = event->ctx;
3091 struct perf_event *sibling;
3092
3093 WARN_ON_ONCE(ctx->parent_ctx);
3094 mutex_lock(&ctx->mutex);
3095 event = event->group_leader;
3096
3097 perf_event_for_each_child(event, func);
3098 func(event);
3099 list_for_each_entry(sibling, &event->sibling_list, group_entry)
3100 perf_event_for_each_child(event, func);
3101 mutex_unlock(&ctx->mutex);
3102}
3103
3104static int perf_event_period(struct perf_event *event, u64 __user *arg)
3105{
3106 struct perf_event_context *ctx = event->ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003107 int ret = 0;
3108 u64 value;
3109
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +01003110 if (!is_sampling_event(event))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003111 return -EINVAL;
3112
John Blackwoodad0cf342010-09-28 18:03:11 -04003113 if (copy_from_user(&value, arg, sizeof(value)))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003114 return -EFAULT;
3115
3116 if (!value)
3117 return -EINVAL;
3118
Thomas Gleixnere625cce2009-11-17 18:02:06 +01003119 raw_spin_lock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003120 if (event->attr.freq) {
3121 if (value > sysctl_perf_event_sample_rate) {
3122 ret = -EINVAL;
3123 goto unlock;
3124 }
3125
3126 event->attr.sample_freq = value;
3127 } else {
3128 event->attr.sample_period = value;
3129 event->hw.sample_period = value;
3130 }
3131unlock:
Thomas Gleixnere625cce2009-11-17 18:02:06 +01003132 raw_spin_unlock_irq(&ctx->lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003133
3134 return ret;
3135}
3136
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003137static const struct file_operations perf_fops;
3138
3139static struct perf_event *perf_fget_light(int fd, int *fput_needed)
3140{
3141 struct file *file;
3142
3143 file = fget_light(fd, fput_needed);
3144 if (!file)
3145 return ERR_PTR(-EBADF);
3146
3147 if (file->f_op != &perf_fops) {
3148 fput_light(file, *fput_needed);
3149 *fput_needed = 0;
3150 return ERR_PTR(-EBADF);
3151 }
3152
3153 return file->private_data;
3154}
3155
3156static int perf_event_set_output(struct perf_event *event,
3157 struct perf_event *output_event);
Li Zefan6fb29152009-10-15 11:21:42 +08003158static int perf_event_set_filter(struct perf_event *event, void __user *arg);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003159
3160static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3161{
3162 struct perf_event *event = file->private_data;
3163 void (*func)(struct perf_event *);
3164 u32 flags = arg;
3165
3166 switch (cmd) {
3167 case PERF_EVENT_IOC_ENABLE:
3168 func = perf_event_enable;
3169 break;
3170 case PERF_EVENT_IOC_DISABLE:
3171 func = perf_event_disable;
3172 break;
3173 case PERF_EVENT_IOC_RESET:
3174 func = perf_event_reset;
3175 break;
3176
3177 case PERF_EVENT_IOC_REFRESH:
3178 return perf_event_refresh(event, arg);
3179
3180 case PERF_EVENT_IOC_PERIOD:
3181 return perf_event_period(event, (u64 __user *)arg);
3182
3183 case PERF_EVENT_IOC_SET_OUTPUT:
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003184 {
3185 struct perf_event *output_event = NULL;
3186 int fput_needed = 0;
3187 int ret;
3188
3189 if (arg != -1) {
3190 output_event = perf_fget_light(arg, &fput_needed);
3191 if (IS_ERR(output_event))
3192 return PTR_ERR(output_event);
3193 }
3194
3195 ret = perf_event_set_output(event, output_event);
3196 if (output_event)
3197 fput_light(output_event->filp, fput_needed);
3198
3199 return ret;
3200 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003201
Li Zefan6fb29152009-10-15 11:21:42 +08003202 case PERF_EVENT_IOC_SET_FILTER:
3203 return perf_event_set_filter(event, (void __user *)arg);
3204
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003205 default:
3206 return -ENOTTY;
3207 }
3208
3209 if (flags & PERF_IOC_FLAG_GROUP)
3210 perf_event_for_each(event, func);
3211 else
3212 perf_event_for_each_child(event, func);
3213
3214 return 0;
3215}
3216
3217int perf_event_task_enable(void)
3218{
3219 struct perf_event *event;
3220
3221 mutex_lock(&current->perf_event_mutex);
3222 list_for_each_entry(event, &current->perf_event_list, owner_entry)
3223 perf_event_for_each_child(event, perf_event_enable);
3224 mutex_unlock(&current->perf_event_mutex);
3225
3226 return 0;
3227}
3228
3229int perf_event_task_disable(void)
3230{
3231 struct perf_event *event;
3232
3233 mutex_lock(&current->perf_event_mutex);
3234 list_for_each_entry(event, &current->perf_event_list, owner_entry)
3235 perf_event_for_each_child(event, perf_event_disable);
3236 mutex_unlock(&current->perf_event_mutex);
3237
3238 return 0;
3239}
3240
3241#ifndef PERF_EVENT_INDEX_OFFSET
3242# define PERF_EVENT_INDEX_OFFSET 0
3243#endif
3244
3245static int perf_event_index(struct perf_event *event)
3246{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02003247 if (event->hw.state & PERF_HES_STOPPED)
3248 return 0;
3249
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003250 if (event->state != PERF_EVENT_STATE_ACTIVE)
3251 return 0;
3252
3253 return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
3254}
3255
Eric B Munsonc4794292011-06-23 16:34:38 -04003256static void calc_timer_values(struct perf_event *event,
Eric B Munson7f310a52011-06-23 16:34:38 -04003257 u64 *enabled,
3258 u64 *running)
Eric B Munsonc4794292011-06-23 16:34:38 -04003259{
3260 u64 now, ctx_time;
3261
3262 now = perf_clock();
3263 ctx_time = event->shadow_ctx_time + now;
3264 *enabled = ctx_time - event->tstamp_enabled;
3265 *running = ctx_time - event->tstamp_running;
3266}
3267
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003268/*
3269 * Callers need to ensure there can be no nesting of this function, otherwise
3270 * the seqlock logic goes bad. We can not serialize this because the arch
3271 * code calls this from NMI context.
3272 */
3273void perf_event_update_userpage(struct perf_event *event)
3274{
3275 struct perf_event_mmap_page *userpg;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003276 struct ring_buffer *rb;
Eric B Munson0d641202011-06-24 12:26:26 -04003277 u64 enabled, running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003278
3279 rcu_read_lock();
Eric B Munson0d641202011-06-24 12:26:26 -04003280 /*
3281 * compute total_time_enabled, total_time_running
3282 * based on snapshot values taken when the event
3283 * was last scheduled in.
3284 *
3285 * we cannot simply called update_context_time()
3286 * because of locking issue as we can be called in
3287 * NMI context
3288 */
3289 calc_timer_values(event, &enabled, &running);
Frederic Weisbecker76369132011-05-19 19:55:04 +02003290 rb = rcu_dereference(event->rb);
3291 if (!rb)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003292 goto unlock;
3293
Frederic Weisbecker76369132011-05-19 19:55:04 +02003294 userpg = rb->user_page;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003295
3296 /*
3297 * Disable preemption so as to not let the corresponding user-space
3298 * spin too long if we get preempted.
3299 */
3300 preempt_disable();
3301 ++userpg->lock;
3302 barrier();
3303 userpg->index = perf_event_index(event);
Peter Zijlstrab5e58792010-05-21 14:43:12 +02003304 userpg->offset = perf_event_count(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003305 if (event->state == PERF_EVENT_STATE_ACTIVE)
Peter Zijlstrae7850592010-05-21 14:43:08 +02003306 userpg->offset -= local64_read(&event->hw.prev_count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003307
Eric B Munson0d641202011-06-24 12:26:26 -04003308 userpg->time_enabled = enabled +
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003309 atomic64_read(&event->child_total_time_enabled);
3310
Eric B Munson0d641202011-06-24 12:26:26 -04003311 userpg->time_running = running +
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003312 atomic64_read(&event->child_total_time_running);
3313
3314 barrier();
3315 ++userpg->lock;
3316 preempt_enable();
3317unlock:
3318 rcu_read_unlock();
3319}
3320
Peter Zijlstra906010b2009-09-21 16:08:49 +02003321static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3322{
3323 struct perf_event *event = vma->vm_file->private_data;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003324 struct ring_buffer *rb;
Peter Zijlstra906010b2009-09-21 16:08:49 +02003325 int ret = VM_FAULT_SIGBUS;
3326
3327 if (vmf->flags & FAULT_FLAG_MKWRITE) {
3328 if (vmf->pgoff == 0)
3329 ret = 0;
3330 return ret;
3331 }
3332
3333 rcu_read_lock();
Frederic Weisbecker76369132011-05-19 19:55:04 +02003334 rb = rcu_dereference(event->rb);
3335 if (!rb)
Peter Zijlstra906010b2009-09-21 16:08:49 +02003336 goto unlock;
3337
3338 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3339 goto unlock;
3340
Frederic Weisbecker76369132011-05-19 19:55:04 +02003341 vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
Peter Zijlstra906010b2009-09-21 16:08:49 +02003342 if (!vmf->page)
3343 goto unlock;
3344
3345 get_page(vmf->page);
3346 vmf->page->mapping = vma->vm_file->f_mapping;
3347 vmf->page->index = vmf->pgoff;
3348
3349 ret = 0;
3350unlock:
3351 rcu_read_unlock();
3352
3353 return ret;
3354}
3355
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003356static void ring_buffer_attach(struct perf_event *event,
3357 struct ring_buffer *rb)
3358{
3359 unsigned long flags;
3360
3361 if (!list_empty(&event->rb_entry))
3362 return;
3363
3364 spin_lock_irqsave(&rb->event_lock, flags);
3365 if (!list_empty(&event->rb_entry))
3366 goto unlock;
3367
3368 list_add(&event->rb_entry, &rb->event_list);
3369unlock:
3370 spin_unlock_irqrestore(&rb->event_lock, flags);
3371}
3372
3373static void ring_buffer_detach(struct perf_event *event,
3374 struct ring_buffer *rb)
3375{
3376 unsigned long flags;
3377
3378 if (list_empty(&event->rb_entry))
3379 return;
3380
3381 spin_lock_irqsave(&rb->event_lock, flags);
3382 list_del_init(&event->rb_entry);
3383 wake_up_all(&event->waitq);
3384 spin_unlock_irqrestore(&rb->event_lock, flags);
3385}
3386
3387static void ring_buffer_wakeup(struct perf_event *event)
3388{
3389 struct ring_buffer *rb;
3390
3391 rcu_read_lock();
3392 rb = rcu_dereference(event->rb);
Will Deacon44b7f4b2011-12-13 20:40:45 +01003393 if (!rb)
3394 goto unlock;
3395
3396 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003397 wake_up_all(&event->waitq);
Will Deacon44b7f4b2011-12-13 20:40:45 +01003398
3399unlock:
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003400 rcu_read_unlock();
3401}
3402
Frederic Weisbecker76369132011-05-19 19:55:04 +02003403static void rb_free_rcu(struct rcu_head *rcu_head)
Peter Zijlstra906010b2009-09-21 16:08:49 +02003404{
Frederic Weisbecker76369132011-05-19 19:55:04 +02003405 struct ring_buffer *rb;
Peter Zijlstra906010b2009-09-21 16:08:49 +02003406
Frederic Weisbecker76369132011-05-19 19:55:04 +02003407 rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3408 rb_free(rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003409}
3410
Frederic Weisbecker76369132011-05-19 19:55:04 +02003411static struct ring_buffer *ring_buffer_get(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003412{
Frederic Weisbecker76369132011-05-19 19:55:04 +02003413 struct ring_buffer *rb;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003414
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003415 rcu_read_lock();
Frederic Weisbecker76369132011-05-19 19:55:04 +02003416 rb = rcu_dereference(event->rb);
3417 if (rb) {
3418 if (!atomic_inc_not_zero(&rb->refcount))
3419 rb = NULL;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003420 }
3421 rcu_read_unlock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003422
Frederic Weisbecker76369132011-05-19 19:55:04 +02003423 return rb;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003424}
3425
Frederic Weisbecker76369132011-05-19 19:55:04 +02003426static void ring_buffer_put(struct ring_buffer *rb)
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003427{
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003428 struct perf_event *event, *n;
3429 unsigned long flags;
3430
Frederic Weisbecker76369132011-05-19 19:55:04 +02003431 if (!atomic_dec_and_test(&rb->refcount))
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003432 return;
3433
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003434 spin_lock_irqsave(&rb->event_lock, flags);
3435 list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3436 list_del_init(&event->rb_entry);
3437 wake_up_all(&event->waitq);
3438 }
3439 spin_unlock_irqrestore(&rb->event_lock, flags);
3440
Frederic Weisbecker76369132011-05-19 19:55:04 +02003441 call_rcu(&rb->rcu_head, rb_free_rcu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003442}
3443
3444static void perf_mmap_open(struct vm_area_struct *vma)
3445{
3446 struct perf_event *event = vma->vm_file->private_data;
3447
3448 atomic_inc(&event->mmap_count);
3449}
3450
3451static void perf_mmap_close(struct vm_area_struct *vma)
3452{
3453 struct perf_event *event = vma->vm_file->private_data;
3454
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003455 if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
Frederic Weisbecker76369132011-05-19 19:55:04 +02003456 unsigned long size = perf_data_size(event->rb);
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003457 struct user_struct *user = event->mmap_user;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003458 struct ring_buffer *rb = event->rb;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003459
Peter Zijlstra906010b2009-09-21 16:08:49 +02003460 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
Christoph Lameterbc3e53f2011-10-31 17:07:30 -07003461 vma->vm_mm->pinned_vm -= event->mmap_locked;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003462 rcu_assign_pointer(event->rb, NULL);
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003463 ring_buffer_detach(event, rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003464 mutex_unlock(&event->mmap_mutex);
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003465
Frederic Weisbecker76369132011-05-19 19:55:04 +02003466 ring_buffer_put(rb);
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003467 free_uid(user);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003468 }
3469}
3470
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04003471static const struct vm_operations_struct perf_mmap_vmops = {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003472 .open = perf_mmap_open,
3473 .close = perf_mmap_close,
3474 .fault = perf_mmap_fault,
3475 .page_mkwrite = perf_mmap_fault,
3476};
3477
3478static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3479{
3480 struct perf_event *event = file->private_data;
3481 unsigned long user_locked, user_lock_limit;
3482 struct user_struct *user = current_user();
3483 unsigned long locked, lock_limit;
Frederic Weisbecker76369132011-05-19 19:55:04 +02003484 struct ring_buffer *rb;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003485 unsigned long vma_size;
3486 unsigned long nr_pages;
3487 long user_extra, extra;
Peter Zijlstrad57e34f2010-05-28 19:41:35 +02003488 int ret = 0, flags = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003489
Peter Zijlstrac7920612010-05-18 10:33:24 +02003490 /*
3491 * Don't allow mmap() of inherited per-task counters. This would
3492 * create a performance issue due to all children writing to the
Frederic Weisbecker76369132011-05-19 19:55:04 +02003493 * same rb.
Peter Zijlstrac7920612010-05-18 10:33:24 +02003494 */
3495 if (event->cpu == -1 && event->attr.inherit)
3496 return -EINVAL;
3497
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003498 if (!(vma->vm_flags & VM_SHARED))
3499 return -EINVAL;
3500
3501 vma_size = vma->vm_end - vma->vm_start;
3502 nr_pages = (vma_size / PAGE_SIZE) - 1;
3503
3504 /*
Frederic Weisbecker76369132011-05-19 19:55:04 +02003505 * If we have rb pages ensure they're a power-of-two number, so we
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003506 * can do bitmasks instead of modulo.
3507 */
3508 if (nr_pages != 0 && !is_power_of_2(nr_pages))
3509 return -EINVAL;
3510
3511 if (vma_size != PAGE_SIZE * (1 + nr_pages))
3512 return -EINVAL;
3513
3514 if (vma->vm_pgoff != 0)
3515 return -EINVAL;
3516
3517 WARN_ON_ONCE(event->ctx->parent_ctx);
3518 mutex_lock(&event->mmap_mutex);
Frederic Weisbecker76369132011-05-19 19:55:04 +02003519 if (event->rb) {
3520 if (event->rb->nr_pages == nr_pages)
3521 atomic_inc(&event->rb->refcount);
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003522 else
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003523 ret = -EINVAL;
3524 goto unlock;
3525 }
3526
3527 user_extra = nr_pages + 1;
3528 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3529
3530 /*
3531 * Increase the limit linearly with more CPUs:
3532 */
3533 user_lock_limit *= num_online_cpus();
3534
3535 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3536
3537 extra = 0;
3538 if (user_locked > user_lock_limit)
3539 extra = user_locked - user_lock_limit;
3540
Jiri Slaby78d7d402010-03-05 13:42:54 -08003541 lock_limit = rlimit(RLIMIT_MEMLOCK);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003542 lock_limit >>= PAGE_SHIFT;
Christoph Lameterbc3e53f2011-10-31 17:07:30 -07003543 locked = vma->vm_mm->pinned_vm + extra;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003544
3545 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3546 !capable(CAP_IPC_LOCK)) {
3547 ret = -EPERM;
3548 goto unlock;
3549 }
3550
Frederic Weisbecker76369132011-05-19 19:55:04 +02003551 WARN_ON(event->rb);
Peter Zijlstra906010b2009-09-21 16:08:49 +02003552
Peter Zijlstrad57e34f2010-05-28 19:41:35 +02003553 if (vma->vm_flags & VM_WRITE)
Frederic Weisbecker76369132011-05-19 19:55:04 +02003554 flags |= RING_BUFFER_WRITABLE;
Peter Zijlstrad57e34f2010-05-28 19:41:35 +02003555
Vince Weaver4ec83632011-06-01 15:15:36 -04003556 rb = rb_alloc(nr_pages,
3557 event->attr.watermark ? event->attr.wakeup_watermark : 0,
3558 event->cpu, flags);
3559
Frederic Weisbecker76369132011-05-19 19:55:04 +02003560 if (!rb) {
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003561 ret = -ENOMEM;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003562 goto unlock;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003563 }
Frederic Weisbecker76369132011-05-19 19:55:04 +02003564 rcu_assign_pointer(event->rb, rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003565
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003566 atomic_long_add(user_extra, &user->locked_vm);
3567 event->mmap_locked = extra;
3568 event->mmap_user = get_current_user();
Christoph Lameterbc3e53f2011-10-31 17:07:30 -07003569 vma->vm_mm->pinned_vm += event->mmap_locked;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003570
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003571unlock:
Peter Zijlstraac9721f2010-05-27 12:54:41 +02003572 if (!ret)
3573 atomic_inc(&event->mmap_count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003574 mutex_unlock(&event->mmap_mutex);
3575
3576 vma->vm_flags |= VM_RESERVED;
3577 vma->vm_ops = &perf_mmap_vmops;
3578
3579 return ret;
3580}
3581
3582static int perf_fasync(int fd, struct file *filp, int on)
3583{
3584 struct inode *inode = filp->f_path.dentry->d_inode;
3585 struct perf_event *event = filp->private_data;
3586 int retval;
3587
3588 mutex_lock(&inode->i_mutex);
3589 retval = fasync_helper(fd, filp, on, &event->fasync);
3590 mutex_unlock(&inode->i_mutex);
3591
3592 if (retval < 0)
3593 return retval;
3594
3595 return 0;
3596}
3597
3598static const struct file_operations perf_fops = {
Arnd Bergmann3326c1c2010-03-23 19:09:33 +01003599 .llseek = no_llseek,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003600 .release = perf_release,
3601 .read = perf_read,
3602 .poll = perf_poll,
3603 .unlocked_ioctl = perf_ioctl,
3604 .compat_ioctl = perf_ioctl,
3605 .mmap = perf_mmap,
3606 .fasync = perf_fasync,
3607};
3608
3609/*
3610 * Perf event wakeup
3611 *
3612 * If there's data, ensure we set the poll() state and publish everything
3613 * to user-space before waking everybody up.
3614 */
3615
3616void perf_event_wakeup(struct perf_event *event)
3617{
Peter Zijlstra10c6db12011-11-26 02:47:31 +01003618 ring_buffer_wakeup(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003619
3620 if (event->pending_kill) {
3621 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3622 event->pending_kill = 0;
3623 }
3624}
3625
Peter Zijlstrae360adb2010-10-14 14:01:34 +08003626static void perf_pending_event(struct irq_work *entry)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003627{
3628 struct perf_event *event = container_of(entry,
3629 struct perf_event, pending);
3630
3631 if (event->pending_disable) {
3632 event->pending_disable = 0;
3633 __perf_event_disable(event);
3634 }
3635
3636 if (event->pending_wakeup) {
3637 event->pending_wakeup = 0;
3638 perf_event_wakeup(event);
3639 }
3640}
3641
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003642/*
Zhang, Yanmin39447b32010-04-19 13:32:41 +08003643 * We assume there is only KVM supporting the callbacks.
3644 * Later on, we might change it to a list if there is
3645 * another virtualization implementation supporting the callbacks.
3646 */
3647struct perf_guest_info_callbacks *perf_guest_cbs;
3648
3649int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3650{
3651 perf_guest_cbs = cbs;
3652 return 0;
3653}
3654EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3655
3656int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3657{
3658 perf_guest_cbs = NULL;
3659 return 0;
3660}
3661EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3662
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02003663static void __perf_event_header__init_id(struct perf_event_header *header,
3664 struct perf_sample_data *data,
3665 struct perf_event *event)
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -02003666{
3667 u64 sample_type = event->attr.sample_type;
3668
3669 data->type = sample_type;
3670 header->size += event->id_header_size;
3671
3672 if (sample_type & PERF_SAMPLE_TID) {
3673 /* namespace issues */
3674 data->tid_entry.pid = perf_event_pid(event, current);
3675 data->tid_entry.tid = perf_event_tid(event, current);
3676 }
3677
3678 if (sample_type & PERF_SAMPLE_TIME)
3679 data->time = perf_clock();
3680
3681 if (sample_type & PERF_SAMPLE_ID)
3682 data->id = primary_event_id(event);
3683
3684 if (sample_type & PERF_SAMPLE_STREAM_ID)
3685 data->stream_id = event->id;
3686
3687 if (sample_type & PERF_SAMPLE_CPU) {
3688 data->cpu_entry.cpu = raw_smp_processor_id();
3689 data->cpu_entry.reserved = 0;
3690 }
3691}
3692
Frederic Weisbecker76369132011-05-19 19:55:04 +02003693void perf_event_header__init_id(struct perf_event_header *header,
3694 struct perf_sample_data *data,
3695 struct perf_event *event)
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02003696{
3697 if (event->attr.sample_id_all)
3698 __perf_event_header__init_id(header, data, event);
3699}
3700
3701static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3702 struct perf_sample_data *data)
3703{
3704 u64 sample_type = data->type;
3705
3706 if (sample_type & PERF_SAMPLE_TID)
3707 perf_output_put(handle, data->tid_entry);
3708
3709 if (sample_type & PERF_SAMPLE_TIME)
3710 perf_output_put(handle, data->time);
3711
3712 if (sample_type & PERF_SAMPLE_ID)
3713 perf_output_put(handle, data->id);
3714
3715 if (sample_type & PERF_SAMPLE_STREAM_ID)
3716 perf_output_put(handle, data->stream_id);
3717
3718 if (sample_type & PERF_SAMPLE_CPU)
3719 perf_output_put(handle, data->cpu_entry);
3720}
3721
Frederic Weisbecker76369132011-05-19 19:55:04 +02003722void perf_event__output_id_sample(struct perf_event *event,
3723 struct perf_output_handle *handle,
3724 struct perf_sample_data *sample)
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02003725{
3726 if (event->attr.sample_id_all)
3727 __perf_event__output_id_sample(handle, sample);
3728}
3729
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003730static void perf_output_read_one(struct perf_output_handle *handle,
Stephane Eranianeed01522010-10-26 16:08:01 +02003731 struct perf_event *event,
3732 u64 enabled, u64 running)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003733{
3734 u64 read_format = event->attr.read_format;
3735 u64 values[4];
3736 int n = 0;
3737
Peter Zijlstrab5e58792010-05-21 14:43:12 +02003738 values[n++] = perf_event_count(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003739 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
Stephane Eranianeed01522010-10-26 16:08:01 +02003740 values[n++] = enabled +
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003741 atomic64_read(&event->child_total_time_enabled);
3742 }
3743 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
Stephane Eranianeed01522010-10-26 16:08:01 +02003744 values[n++] = running +
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003745 atomic64_read(&event->child_total_time_running);
3746 }
3747 if (read_format & PERF_FORMAT_ID)
3748 values[n++] = primary_event_id(event);
3749
Frederic Weisbecker76369132011-05-19 19:55:04 +02003750 __output_copy(handle, values, n * sizeof(u64));
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003751}
3752
3753/*
3754 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3755 */
3756static void perf_output_read_group(struct perf_output_handle *handle,
Stephane Eranianeed01522010-10-26 16:08:01 +02003757 struct perf_event *event,
3758 u64 enabled, u64 running)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003759{
3760 struct perf_event *leader = event->group_leader, *sub;
3761 u64 read_format = event->attr.read_format;
3762 u64 values[5];
3763 int n = 0;
3764
3765 values[n++] = 1 + leader->nr_siblings;
3766
3767 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
Stephane Eranianeed01522010-10-26 16:08:01 +02003768 values[n++] = enabled;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003769
3770 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
Stephane Eranianeed01522010-10-26 16:08:01 +02003771 values[n++] = running;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003772
3773 if (leader != event)
3774 leader->pmu->read(leader);
3775
Peter Zijlstrab5e58792010-05-21 14:43:12 +02003776 values[n++] = perf_event_count(leader);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003777 if (read_format & PERF_FORMAT_ID)
3778 values[n++] = primary_event_id(leader);
3779
Frederic Weisbecker76369132011-05-19 19:55:04 +02003780 __output_copy(handle, values, n * sizeof(u64));
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003781
3782 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3783 n = 0;
3784
3785 if (sub != event)
3786 sub->pmu->read(sub);
3787
Peter Zijlstrab5e58792010-05-21 14:43:12 +02003788 values[n++] = perf_event_count(sub);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003789 if (read_format & PERF_FORMAT_ID)
3790 values[n++] = primary_event_id(sub);
3791
Frederic Weisbecker76369132011-05-19 19:55:04 +02003792 __output_copy(handle, values, n * sizeof(u64));
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003793 }
3794}
3795
Stephane Eranianeed01522010-10-26 16:08:01 +02003796#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3797 PERF_FORMAT_TOTAL_TIME_RUNNING)
3798
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003799static void perf_output_read(struct perf_output_handle *handle,
3800 struct perf_event *event)
3801{
Eric B Munsonc4794292011-06-23 16:34:38 -04003802 u64 enabled = 0, running = 0;
Stephane Eranianeed01522010-10-26 16:08:01 +02003803 u64 read_format = event->attr.read_format;
3804
3805 /*
3806 * compute total_time_enabled, total_time_running
3807 * based on snapshot values taken when the event
3808 * was last scheduled in.
3809 *
3810 * we cannot simply called update_context_time()
3811 * because of locking issue as we are called in
3812 * NMI context
3813 */
Eric B Munsonc4794292011-06-23 16:34:38 -04003814 if (read_format & PERF_FORMAT_TOTAL_TIMES)
3815 calc_timer_values(event, &enabled, &running);
Stephane Eranianeed01522010-10-26 16:08:01 +02003816
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003817 if (event->attr.read_format & PERF_FORMAT_GROUP)
Stephane Eranianeed01522010-10-26 16:08:01 +02003818 perf_output_read_group(handle, event, enabled, running);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003819 else
Stephane Eranianeed01522010-10-26 16:08:01 +02003820 perf_output_read_one(handle, event, enabled, running);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003821}
3822
3823void perf_output_sample(struct perf_output_handle *handle,
3824 struct perf_event_header *header,
3825 struct perf_sample_data *data,
3826 struct perf_event *event)
3827{
3828 u64 sample_type = data->type;
3829
3830 perf_output_put(handle, *header);
3831
3832 if (sample_type & PERF_SAMPLE_IP)
3833 perf_output_put(handle, data->ip);
3834
3835 if (sample_type & PERF_SAMPLE_TID)
3836 perf_output_put(handle, data->tid_entry);
3837
3838 if (sample_type & PERF_SAMPLE_TIME)
3839 perf_output_put(handle, data->time);
3840
3841 if (sample_type & PERF_SAMPLE_ADDR)
3842 perf_output_put(handle, data->addr);
3843
3844 if (sample_type & PERF_SAMPLE_ID)
3845 perf_output_put(handle, data->id);
3846
3847 if (sample_type & PERF_SAMPLE_STREAM_ID)
3848 perf_output_put(handle, data->stream_id);
3849
3850 if (sample_type & PERF_SAMPLE_CPU)
3851 perf_output_put(handle, data->cpu_entry);
3852
3853 if (sample_type & PERF_SAMPLE_PERIOD)
3854 perf_output_put(handle, data->period);
3855
3856 if (sample_type & PERF_SAMPLE_READ)
3857 perf_output_read(handle, event);
3858
3859 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3860 if (data->callchain) {
3861 int size = 1;
3862
3863 if (data->callchain)
3864 size += data->callchain->nr;
3865
3866 size *= sizeof(u64);
3867
Frederic Weisbecker76369132011-05-19 19:55:04 +02003868 __output_copy(handle, data->callchain, size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003869 } else {
3870 u64 nr = 0;
3871 perf_output_put(handle, nr);
3872 }
3873 }
3874
3875 if (sample_type & PERF_SAMPLE_RAW) {
3876 if (data->raw) {
3877 perf_output_put(handle, data->raw->size);
Frederic Weisbecker76369132011-05-19 19:55:04 +02003878 __output_copy(handle, data->raw->data,
3879 data->raw->size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003880 } else {
3881 struct {
3882 u32 size;
3883 u32 data;
3884 } raw = {
3885 .size = sizeof(u32),
3886 .data = 0,
3887 };
3888 perf_output_put(handle, raw);
3889 }
3890 }
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02003891
3892 if (!event->attr.watermark) {
3893 int wakeup_events = event->attr.wakeup_events;
3894
3895 if (wakeup_events) {
3896 struct ring_buffer *rb = handle->rb;
3897 int events = local_inc_return(&rb->events);
3898
3899 if (events >= wakeup_events) {
3900 local_sub(wakeup_events, &rb->events);
3901 local_inc(&rb->wakeup);
3902 }
3903 }
3904 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003905}
3906
3907void perf_prepare_sample(struct perf_event_header *header,
3908 struct perf_sample_data *data,
3909 struct perf_event *event,
3910 struct pt_regs *regs)
3911{
3912 u64 sample_type = event->attr.sample_type;
3913
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003914 header->type = PERF_RECORD_SAMPLE;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02003915 header->size = sizeof(*header) + event->header_size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003916
3917 header->misc = 0;
3918 header->misc |= perf_misc_flags(regs);
3919
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02003920 __perf_event_header__init_id(header, data, event);
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -02003921
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02003922 if (sample_type & PERF_SAMPLE_IP)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003923 data->ip = perf_instruction_pointer(regs);
3924
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003925 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3926 int size = 1;
3927
3928 data->callchain = perf_callchain(regs);
3929
3930 if (data->callchain)
3931 size += data->callchain->nr;
3932
3933 header->size += size * sizeof(u64);
3934 }
3935
3936 if (sample_type & PERF_SAMPLE_RAW) {
3937 int size = sizeof(u32);
3938
3939 if (data->raw)
3940 size += data->raw->size;
3941 else
3942 size += sizeof(u32);
3943
3944 WARN_ON_ONCE(size & (sizeof(u64)-1));
3945 header->size += size;
3946 }
3947}
3948
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02003949static void perf_event_output(struct perf_event *event,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003950 struct perf_sample_data *data,
3951 struct pt_regs *regs)
3952{
3953 struct perf_output_handle handle;
3954 struct perf_event_header header;
3955
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02003956 /* protect the callchain buffers */
3957 rcu_read_lock();
3958
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003959 perf_prepare_sample(&header, data, event, regs);
3960
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02003961 if (perf_output_begin(&handle, event, header.size))
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02003962 goto exit;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003963
3964 perf_output_sample(&handle, &header, data, event);
3965
3966 perf_output_end(&handle);
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02003967
3968exit:
3969 rcu_read_unlock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003970}
3971
3972/*
3973 * read event_id
3974 */
3975
3976struct perf_read_event {
3977 struct perf_event_header header;
3978
3979 u32 pid;
3980 u32 tid;
3981};
3982
3983static void
3984perf_event_read_event(struct perf_event *event,
3985 struct task_struct *task)
3986{
3987 struct perf_output_handle handle;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02003988 struct perf_sample_data sample;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003989 struct perf_read_event read_event = {
3990 .header = {
3991 .type = PERF_RECORD_READ,
3992 .misc = 0,
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02003993 .size = sizeof(read_event) + event->read_size,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02003994 },
3995 .pid = perf_event_pid(event, task),
3996 .tid = perf_event_tid(event, task),
3997 };
3998 int ret;
3999
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004000 perf_event_header__init_id(&read_event.header, &sample, event);
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02004001 ret = perf_output_begin(&handle, event, read_event.header.size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004002 if (ret)
4003 return;
4004
4005 perf_output_put(&handle, read_event);
4006 perf_output_read(&handle, event);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004007 perf_event__output_id_sample(event, &handle, &sample);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004008
4009 perf_output_end(&handle);
4010}
4011
4012/*
4013 * task tracking -- fork/exit
4014 *
Eric B Munson3af9e852010-05-18 15:30:49 +01004015 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004016 */
4017
4018struct perf_task_event {
4019 struct task_struct *task;
4020 struct perf_event_context *task_ctx;
4021
4022 struct {
4023 struct perf_event_header header;
4024
4025 u32 pid;
4026 u32 ppid;
4027 u32 tid;
4028 u32 ptid;
4029 u64 time;
4030 } event_id;
4031};
4032
4033static void perf_event_task_output(struct perf_event *event,
4034 struct perf_task_event *task_event)
4035{
4036 struct perf_output_handle handle;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004037 struct perf_sample_data sample;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004038 struct task_struct *task = task_event->task;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004039 int ret, size = task_event->event_id.header.size;
Mike Galbraith8bb39f92010-03-26 11:11:33 +01004040
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004041 perf_event_header__init_id(&task_event->event_id.header, &sample, event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004042
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004043 ret = perf_output_begin(&handle, event,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02004044 task_event->event_id.header.size);
Peter Zijlstraef607772010-05-18 10:50:41 +02004045 if (ret)
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004046 goto out;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004047
4048 task_event->event_id.pid = perf_event_pid(event, task);
4049 task_event->event_id.ppid = perf_event_pid(event, current);
4050
4051 task_event->event_id.tid = perf_event_tid(event, task);
4052 task_event->event_id.ptid = perf_event_tid(event, current);
4053
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004054 perf_output_put(&handle, task_event->event_id);
4055
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004056 perf_event__output_id_sample(event, &handle, &sample);
4057
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004058 perf_output_end(&handle);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004059out:
4060 task_event->event_id.header.size = size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004061}
4062
4063static int perf_event_task_match(struct perf_event *event)
4064{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01004065 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01004066 return 0;
4067
Stephane Eranian5632ab12011-01-03 18:20:01 +02004068 if (!event_filter_match(event))
Peter Zijlstra5d27c232009-12-17 13:16:32 +01004069 return 0;
4070
Eric B Munson3af9e852010-05-18 15:30:49 +01004071 if (event->attr.comm || event->attr.mmap ||
4072 event->attr.mmap_data || event->attr.task)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004073 return 1;
4074
4075 return 0;
4076}
4077
4078static void perf_event_task_ctx(struct perf_event_context *ctx,
4079 struct perf_task_event *task_event)
4080{
4081 struct perf_event *event;
4082
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004083 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4084 if (perf_event_task_match(event))
4085 perf_event_task_output(event, task_event);
4086 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004087}
4088
4089static void perf_event_task_event(struct perf_task_event *task_event)
4090{
4091 struct perf_cpu_context *cpuctx;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004092 struct perf_event_context *ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004093 struct pmu *pmu;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004094 int ctxn;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004095
Peter Zijlstrad6ff86c2009-11-20 22:19:46 +01004096 rcu_read_lock();
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004097 list_for_each_entry_rcu(pmu, &pmus, entry) {
Peter Zijlstra41945f62010-09-16 19:17:24 +02004098 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra51676952010-12-07 14:18:20 +01004099 if (cpuctx->active_pmu != pmu)
4100 goto next;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004101 perf_event_task_ctx(&cpuctx->ctx, task_event);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004102
4103 ctx = task_event->task_ctx;
4104 if (!ctx) {
4105 ctxn = pmu->task_ctx_nr;
4106 if (ctxn < 0)
Peter Zijlstra41945f62010-09-16 19:17:24 +02004107 goto next;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004108 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4109 }
4110 if (ctx)
4111 perf_event_task_ctx(ctx, task_event);
Peter Zijlstra41945f62010-09-16 19:17:24 +02004112next:
4113 put_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004114 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004115 rcu_read_unlock();
4116}
4117
4118static void perf_event_task(struct task_struct *task,
4119 struct perf_event_context *task_ctx,
4120 int new)
4121{
4122 struct perf_task_event task_event;
4123
4124 if (!atomic_read(&nr_comm_events) &&
4125 !atomic_read(&nr_mmap_events) &&
4126 !atomic_read(&nr_task_events))
4127 return;
4128
4129 task_event = (struct perf_task_event){
4130 .task = task,
4131 .task_ctx = task_ctx,
4132 .event_id = {
4133 .header = {
4134 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4135 .misc = 0,
4136 .size = sizeof(task_event.event_id),
4137 },
4138 /* .pid */
4139 /* .ppid */
4140 /* .tid */
4141 /* .ptid */
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01004142 .time = perf_clock(),
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004143 },
4144 };
4145
4146 perf_event_task_event(&task_event);
4147}
4148
4149void perf_event_fork(struct task_struct *task)
4150{
4151 perf_event_task(task, NULL, 1);
4152}
4153
4154/*
4155 * comm tracking
4156 */
4157
4158struct perf_comm_event {
4159 struct task_struct *task;
4160 char *comm;
4161 int comm_size;
4162
4163 struct {
4164 struct perf_event_header header;
4165
4166 u32 pid;
4167 u32 tid;
4168 } event_id;
4169};
4170
4171static void perf_event_comm_output(struct perf_event *event,
4172 struct perf_comm_event *comm_event)
4173{
4174 struct perf_output_handle handle;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004175 struct perf_sample_data sample;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004176 int size = comm_event->event_id.header.size;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004177 int ret;
4178
4179 perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4180 ret = perf_output_begin(&handle, event,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02004181 comm_event->event_id.header.size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004182
4183 if (ret)
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004184 goto out;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004185
4186 comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4187 comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4188
4189 perf_output_put(&handle, comm_event->event_id);
Frederic Weisbecker76369132011-05-19 19:55:04 +02004190 __output_copy(&handle, comm_event->comm,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004191 comm_event->comm_size);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004192
4193 perf_event__output_id_sample(event, &handle, &sample);
4194
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004195 perf_output_end(&handle);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004196out:
4197 comm_event->event_id.header.size = size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004198}
4199
4200static int perf_event_comm_match(struct perf_event *event)
4201{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01004202 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01004203 return 0;
4204
Stephane Eranian5632ab12011-01-03 18:20:01 +02004205 if (!event_filter_match(event))
Peter Zijlstra5d27c232009-12-17 13:16:32 +01004206 return 0;
4207
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004208 if (event->attr.comm)
4209 return 1;
4210
4211 return 0;
4212}
4213
4214static void perf_event_comm_ctx(struct perf_event_context *ctx,
4215 struct perf_comm_event *comm_event)
4216{
4217 struct perf_event *event;
4218
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004219 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4220 if (perf_event_comm_match(event))
4221 perf_event_comm_output(event, comm_event);
4222 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004223}
4224
4225static void perf_event_comm_event(struct perf_comm_event *comm_event)
4226{
4227 struct perf_cpu_context *cpuctx;
4228 struct perf_event_context *ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004229 char comm[TASK_COMM_LEN];
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004230 unsigned int size;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004231 struct pmu *pmu;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004232 int ctxn;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004233
4234 memset(comm, 0, sizeof(comm));
Márton Németh96b02d72009-11-21 23:10:15 +01004235 strlcpy(comm, comm_event->task->comm, sizeof(comm));
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004236 size = ALIGN(strlen(comm)+1, sizeof(u64));
4237
4238 comm_event->comm = comm;
4239 comm_event->comm_size = size;
4240
4241 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
Peter Zijlstraf6595f32009-11-20 22:19:47 +01004242 rcu_read_lock();
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004243 list_for_each_entry_rcu(pmu, &pmus, entry) {
Peter Zijlstra41945f62010-09-16 19:17:24 +02004244 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra51676952010-12-07 14:18:20 +01004245 if (cpuctx->active_pmu != pmu)
4246 goto next;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004247 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004248
4249 ctxn = pmu->task_ctx_nr;
4250 if (ctxn < 0)
Peter Zijlstra41945f62010-09-16 19:17:24 +02004251 goto next;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004252
4253 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4254 if (ctx)
4255 perf_event_comm_ctx(ctx, comm_event);
Peter Zijlstra41945f62010-09-16 19:17:24 +02004256next:
4257 put_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004258 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004259 rcu_read_unlock();
4260}
4261
4262void perf_event_comm(struct task_struct *task)
4263{
4264 struct perf_comm_event comm_event;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004265 struct perf_event_context *ctx;
4266 int ctxn;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004267
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004268 for_each_task_context_nr(ctxn) {
4269 ctx = task->perf_event_ctxp[ctxn];
4270 if (!ctx)
4271 continue;
4272
4273 perf_event_enable_on_exec(ctx);
4274 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004275
4276 if (!atomic_read(&nr_comm_events))
4277 return;
4278
4279 comm_event = (struct perf_comm_event){
4280 .task = task,
4281 /* .comm */
4282 /* .comm_size */
4283 .event_id = {
4284 .header = {
4285 .type = PERF_RECORD_COMM,
4286 .misc = 0,
4287 /* .size */
4288 },
4289 /* .pid */
4290 /* .tid */
4291 },
4292 };
4293
4294 perf_event_comm_event(&comm_event);
4295}
4296
4297/*
4298 * mmap tracking
4299 */
4300
4301struct perf_mmap_event {
4302 struct vm_area_struct *vma;
4303
4304 const char *file_name;
4305 int file_size;
4306
4307 struct {
4308 struct perf_event_header header;
4309
4310 u32 pid;
4311 u32 tid;
4312 u64 start;
4313 u64 len;
4314 u64 pgoff;
4315 } event_id;
4316};
4317
4318static void perf_event_mmap_output(struct perf_event *event,
4319 struct perf_mmap_event *mmap_event)
4320{
4321 struct perf_output_handle handle;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004322 struct perf_sample_data sample;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004323 int size = mmap_event->event_id.header.size;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004324 int ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004325
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004326 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4327 ret = perf_output_begin(&handle, event,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02004328 mmap_event->event_id.header.size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004329 if (ret)
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004330 goto out;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004331
4332 mmap_event->event_id.pid = perf_event_pid(event, current);
4333 mmap_event->event_id.tid = perf_event_tid(event, current);
4334
4335 perf_output_put(&handle, mmap_event->event_id);
Frederic Weisbecker76369132011-05-19 19:55:04 +02004336 __output_copy(&handle, mmap_event->file_name,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004337 mmap_event->file_size);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004338
4339 perf_event__output_id_sample(event, &handle, &sample);
4340
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004341 perf_output_end(&handle);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004342out:
4343 mmap_event->event_id.header.size = size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004344}
4345
4346static int perf_event_mmap_match(struct perf_event *event,
Eric B Munson3af9e852010-05-18 15:30:49 +01004347 struct perf_mmap_event *mmap_event,
4348 int executable)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004349{
Peter Zijlstra6f93d0a2010-02-14 11:12:04 +01004350 if (event->state < PERF_EVENT_STATE_INACTIVE)
Peter Zijlstra22e19082010-01-18 09:12:32 +01004351 return 0;
4352
Stephane Eranian5632ab12011-01-03 18:20:01 +02004353 if (!event_filter_match(event))
Peter Zijlstra5d27c232009-12-17 13:16:32 +01004354 return 0;
4355
Eric B Munson3af9e852010-05-18 15:30:49 +01004356 if ((!executable && event->attr.mmap_data) ||
4357 (executable && event->attr.mmap))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004358 return 1;
4359
4360 return 0;
4361}
4362
4363static void perf_event_mmap_ctx(struct perf_event_context *ctx,
Eric B Munson3af9e852010-05-18 15:30:49 +01004364 struct perf_mmap_event *mmap_event,
4365 int executable)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004366{
4367 struct perf_event *event;
4368
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004369 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
Eric B Munson3af9e852010-05-18 15:30:49 +01004370 if (perf_event_mmap_match(event, mmap_event, executable))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004371 perf_event_mmap_output(event, mmap_event);
4372 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004373}
4374
4375static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4376{
4377 struct perf_cpu_context *cpuctx;
4378 struct perf_event_context *ctx;
4379 struct vm_area_struct *vma = mmap_event->vma;
4380 struct file *file = vma->vm_file;
4381 unsigned int size;
4382 char tmp[16];
4383 char *buf = NULL;
4384 const char *name;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004385 struct pmu *pmu;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004386 int ctxn;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004387
4388 memset(tmp, 0, sizeof(tmp));
4389
4390 if (file) {
4391 /*
Frederic Weisbecker76369132011-05-19 19:55:04 +02004392 * d_path works from the end of the rb backwards, so we
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004393 * need to add enough zero bytes after the string to handle
4394 * the 64bit alignment we do later.
4395 */
4396 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4397 if (!buf) {
4398 name = strncpy(tmp, "//enomem", sizeof(tmp));
4399 goto got_name;
4400 }
4401 name = d_path(&file->f_path, buf, PATH_MAX);
4402 if (IS_ERR(name)) {
4403 name = strncpy(tmp, "//toolong", sizeof(tmp));
4404 goto got_name;
4405 }
4406 } else {
4407 if (arch_vma_name(mmap_event->vma)) {
4408 name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4409 sizeof(tmp));
4410 goto got_name;
4411 }
4412
4413 if (!vma->vm_mm) {
4414 name = strncpy(tmp, "[vdso]", sizeof(tmp));
4415 goto got_name;
Eric B Munson3af9e852010-05-18 15:30:49 +01004416 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4417 vma->vm_end >= vma->vm_mm->brk) {
4418 name = strncpy(tmp, "[heap]", sizeof(tmp));
4419 goto got_name;
4420 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4421 vma->vm_end >= vma->vm_mm->start_stack) {
4422 name = strncpy(tmp, "[stack]", sizeof(tmp));
4423 goto got_name;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004424 }
4425
4426 name = strncpy(tmp, "//anon", sizeof(tmp));
4427 goto got_name;
4428 }
4429
4430got_name:
4431 size = ALIGN(strlen(name)+1, sizeof(u64));
4432
4433 mmap_event->file_name = name;
4434 mmap_event->file_size = size;
4435
4436 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4437
Peter Zijlstraf6d9dd22009-11-20 22:19:48 +01004438 rcu_read_lock();
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004439 list_for_each_entry_rcu(pmu, &pmus, entry) {
Peter Zijlstra41945f62010-09-16 19:17:24 +02004440 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra51676952010-12-07 14:18:20 +01004441 if (cpuctx->active_pmu != pmu)
4442 goto next;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004443 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4444 vma->vm_flags & VM_EXEC);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004445
4446 ctxn = pmu->task_ctx_nr;
4447 if (ctxn < 0)
Peter Zijlstra41945f62010-09-16 19:17:24 +02004448 goto next;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02004449
4450 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4451 if (ctx) {
4452 perf_event_mmap_ctx(ctx, mmap_event,
4453 vma->vm_flags & VM_EXEC);
4454 }
Peter Zijlstra41945f62010-09-16 19:17:24 +02004455next:
4456 put_cpu_ptr(pmu->pmu_cpu_context);
Peter Zijlstra108b02c2010-09-06 14:32:03 +02004457 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004458 rcu_read_unlock();
4459
4460 kfree(buf);
4461}
4462
Eric B Munson3af9e852010-05-18 15:30:49 +01004463void perf_event_mmap(struct vm_area_struct *vma)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004464{
4465 struct perf_mmap_event mmap_event;
4466
4467 if (!atomic_read(&nr_mmap_events))
4468 return;
4469
4470 mmap_event = (struct perf_mmap_event){
4471 .vma = vma,
4472 /* .file_name */
4473 /* .file_size */
4474 .event_id = {
4475 .header = {
4476 .type = PERF_RECORD_MMAP,
Zhang, Yanmin39447b32010-04-19 13:32:41 +08004477 .misc = PERF_RECORD_MISC_USER,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004478 /* .size */
4479 },
4480 /* .pid */
4481 /* .tid */
4482 .start = vma->vm_start,
4483 .len = vma->vm_end - vma->vm_start,
Peter Zijlstra3a0304e2010-02-26 10:33:41 +01004484 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004485 },
4486 };
4487
4488 perf_event_mmap_event(&mmap_event);
4489}
4490
4491/*
4492 * IRQ throttle logging
4493 */
4494
4495static void perf_log_throttle(struct perf_event *event, int enable)
4496{
4497 struct perf_output_handle handle;
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004498 struct perf_sample_data sample;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004499 int ret;
4500
4501 struct {
4502 struct perf_event_header header;
4503 u64 time;
4504 u64 id;
4505 u64 stream_id;
4506 } throttle_event = {
4507 .header = {
4508 .type = PERF_RECORD_THROTTLE,
4509 .misc = 0,
4510 .size = sizeof(throttle_event),
4511 },
4512 .time = perf_clock(),
4513 .id = primary_event_id(event),
4514 .stream_id = event->id,
4515 };
4516
4517 if (enable)
4518 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4519
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004520 perf_event_header__init_id(&throttle_event.header, &sample, event);
4521
4522 ret = perf_output_begin(&handle, event,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +02004523 throttle_event.header.size);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004524 if (ret)
4525 return;
4526
4527 perf_output_put(&handle, throttle_event);
Arnaldo Carvalho de Meloc980d102010-12-04 23:02:20 -02004528 perf_event__output_id_sample(event, &handle, &sample);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004529 perf_output_end(&handle);
4530}
4531
4532/*
4533 * Generic event overflow handling, sampling.
4534 */
4535
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004536static int __perf_event_overflow(struct perf_event *event,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004537 int throttle, struct perf_sample_data *data,
4538 struct pt_regs *regs)
4539{
4540 int events = atomic_read(&event->event_limit);
4541 struct hw_perf_event *hwc = &event->hw;
Stephane Eraniane050e3f2012-01-26 17:03:19 +01004542 u64 seq;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004543 int ret = 0;
4544
Peter Zijlstra96398822010-11-24 18:55:29 +01004545 /*
4546 * Non-sampling counters might still use the PMI to fold short
4547 * hardware counters, ignore those.
4548 */
4549 if (unlikely(!is_sampling_event(event)))
4550 return 0;
4551
Stephane Eraniane050e3f2012-01-26 17:03:19 +01004552 seq = __this_cpu_read(perf_throttled_seq);
4553 if (seq != hwc->interrupts_seq) {
4554 hwc->interrupts_seq = seq;
4555 hwc->interrupts = 1;
4556 } else {
4557 hwc->interrupts++;
4558 if (unlikely(throttle
4559 && hwc->interrupts >= max_samples_per_tick)) {
4560 __this_cpu_inc(perf_throttled_count);
Peter Zijlstra163ec432011-02-16 11:22:34 +01004561 hwc->interrupts = MAX_INTERRUPTS;
4562 perf_log_throttle(event, 0);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004563 ret = 1;
4564 }
Stephane Eraniane050e3f2012-01-26 17:03:19 +01004565 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004566
4567 if (event->attr.freq) {
4568 u64 now = perf_clock();
Peter Zijlstraabd50712010-01-26 18:50:16 +01004569 s64 delta = now - hwc->freq_time_stamp;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004570
Peter Zijlstraabd50712010-01-26 18:50:16 +01004571 hwc->freq_time_stamp = now;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004572
Peter Zijlstraabd50712010-01-26 18:50:16 +01004573 if (delta > 0 && delta < 2*TICK_NSEC)
Stephane Eranianf39d47f2012-02-07 14:39:57 +01004574 perf_adjust_period(event, delta, hwc->last_period, true);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004575 }
4576
4577 /*
4578 * XXX event_limit might not quite work as expected on inherited
4579 * events
4580 */
4581
4582 event->pending_kill = POLL_IN;
4583 if (events && atomic_dec_and_test(&event->event_limit)) {
4584 ret = 1;
4585 event->pending_kill = POLL_HUP;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004586 event->pending_disable = 1;
4587 irq_work_queue(&event->pending);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004588 }
4589
Peter Zijlstra453f19e2009-11-20 22:19:43 +01004590 if (event->overflow_handler)
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004591 event->overflow_handler(event, data, regs);
Peter Zijlstra453f19e2009-11-20 22:19:43 +01004592 else
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004593 perf_event_output(event, data, regs);
Peter Zijlstra453f19e2009-11-20 22:19:43 +01004594
Peter Zijlstraf506b3d2011-05-26 17:02:53 +02004595 if (event->fasync && event->pending_kill) {
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004596 event->pending_wakeup = 1;
4597 irq_work_queue(&event->pending);
Peter Zijlstraf506b3d2011-05-26 17:02:53 +02004598 }
4599
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004600 return ret;
4601}
4602
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004603int perf_event_overflow(struct perf_event *event,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004604 struct perf_sample_data *data,
4605 struct pt_regs *regs)
4606{
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004607 return __perf_event_overflow(event, 1, data, regs);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004608}
4609
4610/*
4611 * Generic software event infrastructure
4612 */
4613
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004614struct swevent_htable {
4615 struct swevent_hlist *swevent_hlist;
4616 struct mutex hlist_mutex;
4617 int hlist_refcount;
4618
4619 /* Recursion avoidance in each contexts */
4620 int recursion[PERF_NR_CONTEXTS];
4621};
4622
4623static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4624
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004625/*
4626 * We directly increment event->count and keep a second value in
4627 * event->hw.period_left to count intervals. This period event
4628 * is kept in the range [-sample_period, 0] so that we can use the
4629 * sign as trigger.
4630 */
4631
4632static u64 perf_swevent_set_period(struct perf_event *event)
4633{
4634 struct hw_perf_event *hwc = &event->hw;
4635 u64 period = hwc->last_period;
4636 u64 nr, offset;
4637 s64 old, val;
4638
4639 hwc->last_period = hwc->sample_period;
4640
4641again:
Peter Zijlstrae7850592010-05-21 14:43:08 +02004642 old = val = local64_read(&hwc->period_left);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004643 if (val < 0)
4644 return 0;
4645
4646 nr = div64_u64(period + val, period);
4647 offset = nr * period;
4648 val -= offset;
Peter Zijlstrae7850592010-05-21 14:43:08 +02004649 if (local64_cmpxchg(&hwc->period_left, old, val) != old)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004650 goto again;
4651
4652 return nr;
4653}
4654
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004655static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004656 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004657 struct pt_regs *regs)
4658{
4659 struct hw_perf_event *hwc = &event->hw;
4660 int throttle = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004661
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004662 if (!overflow)
4663 overflow = perf_swevent_set_period(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004664
4665 if (hwc->interrupts == MAX_INTERRUPTS)
4666 return;
4667
4668 for (; overflow; overflow--) {
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004669 if (__perf_event_overflow(event, throttle,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004670 data, regs)) {
4671 /*
4672 * We inhibit the overflow from happening when
4673 * hwc->interrupts == MAX_INTERRUPTS.
4674 */
4675 break;
4676 }
4677 throttle = 1;
4678 }
4679}
4680
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004681static void perf_swevent_event(struct perf_event *event, u64 nr,
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004682 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004683 struct pt_regs *regs)
4684{
4685 struct hw_perf_event *hwc = &event->hw;
4686
Peter Zijlstrae7850592010-05-21 14:43:08 +02004687 local64_add(nr, &event->count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004688
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004689 if (!regs)
4690 return;
4691
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +01004692 if (!is_sampling_event(event))
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004693 return;
4694
Andrew Vagin5d81e5c2011-11-07 15:54:12 +03004695 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
4696 data->period = nr;
4697 return perf_swevent_overflow(event, 1, data, regs);
4698 } else
4699 data->period = event->hw.last_period;
4700
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004701 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004702 return perf_swevent_overflow(event, 1, data, regs);
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004703
Peter Zijlstrae7850592010-05-21 14:43:08 +02004704 if (local64_add_negative(nr, &hwc->period_left))
Peter Zijlstra0cff7842009-11-20 22:19:44 +01004705 return;
4706
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004707 perf_swevent_overflow(event, 0, data, regs);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004708}
4709
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004710static int perf_exclude_event(struct perf_event *event,
4711 struct pt_regs *regs)
4712{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004713 if (event->hw.state & PERF_HES_STOPPED)
Frederic Weisbecker91b2f482011-03-07 21:27:08 +01004714 return 1;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004715
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004716 if (regs) {
4717 if (event->attr.exclude_user && user_mode(regs))
4718 return 1;
4719
4720 if (event->attr.exclude_kernel && !user_mode(regs))
4721 return 1;
4722 }
4723
4724 return 0;
4725}
4726
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004727static int perf_swevent_match(struct perf_event *event,
4728 enum perf_type_id type,
Li Zefan6fb29152009-10-15 11:21:42 +08004729 u32 event_id,
4730 struct perf_sample_data *data,
4731 struct pt_regs *regs)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004732{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004733 if (event->attr.type != type)
4734 return 0;
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004735
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004736 if (event->attr.config != event_id)
4737 return 0;
4738
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01004739 if (perf_exclude_event(event, regs))
4740 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004741
4742 return 1;
4743}
4744
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004745static inline u64 swevent_hash(u64 type, u32 event_id)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004746{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004747 u64 val = event_id | (type << 32);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004748
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004749 return hash_64(val, SWEVENT_HLIST_BITS);
4750}
4751
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004752static inline struct hlist_head *
4753__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004754{
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004755 u64 hash = swevent_hash(type, event_id);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004756
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004757 return &hlist->heads[hash];
4758}
4759
4760/* For the read side: events when they trigger */
4761static inline struct hlist_head *
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004762find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004763{
4764 struct swevent_hlist *hlist;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004765
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004766 hlist = rcu_dereference(swhash->swevent_hlist);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004767 if (!hlist)
4768 return NULL;
4769
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004770 return __find_swevent_head(hlist, type, event_id);
4771}
4772
4773/* For the event head insertion and removal in the hlist */
4774static inline struct hlist_head *
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004775find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004776{
4777 struct swevent_hlist *hlist;
4778 u32 event_id = event->attr.config;
4779 u64 type = event->attr.type;
4780
4781 /*
4782 * Event scheduling is always serialized against hlist allocation
4783 * and release. Which makes the protected version suitable here.
4784 * The context lock guarantees that.
4785 */
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004786 hlist = rcu_dereference_protected(swhash->swevent_hlist,
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004787 lockdep_is_held(&event->ctx->lock));
4788 if (!hlist)
4789 return NULL;
4790
4791 return __find_swevent_head(hlist, type, event_id);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004792}
4793
4794static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004795 u64 nr,
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004796 struct perf_sample_data *data,
4797 struct pt_regs *regs)
4798{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004799 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004800 struct perf_event *event;
4801 struct hlist_node *node;
4802 struct hlist_head *head;
4803
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004804 rcu_read_lock();
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004805 head = find_swevent_head_rcu(swhash, type, event_id);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004806 if (!head)
4807 goto end;
4808
4809 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
Li Zefan6fb29152009-10-15 11:21:42 +08004810 if (perf_swevent_match(event, type, event_id, data, regs))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004811 perf_swevent_event(event, nr, data, regs);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004812 }
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004813end:
4814 rcu_read_unlock();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004815}
4816
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004817int perf_swevent_get_recursion_context(void)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004818{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004819 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004820
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004821 return get_recursion_context(swhash->recursion);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004822}
Ingo Molnar645e8cc2009-11-22 12:20:19 +01004823EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004824
Jesper Juhlfa9f90b2010-11-28 21:39:34 +01004825inline void perf_swevent_put_recursion_context(int rctx)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004826{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004827 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02004828
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004829 put_recursion_context(swhash->recursion, rctx);
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01004830}
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004831
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004832void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004833{
Ingo Molnara4234bf2009-11-23 10:57:59 +01004834 struct perf_sample_data data;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004835 int rctx;
4836
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02004837 preempt_disable_notrace();
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004838 rctx = perf_swevent_get_recursion_context();
4839 if (rctx < 0)
4840 return;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004841
Peter Zijlstradc1d6282010-03-03 15:55:04 +01004842 perf_sample_data_init(&data, addr);
Ingo Molnara4234bf2009-11-23 10:57:59 +01004843
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02004844 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01004845
4846 perf_swevent_put_recursion_context(rctx);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02004847 preempt_enable_notrace();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004848}
4849
4850static void perf_swevent_read(struct perf_event *event)
4851{
4852}
4853
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004854static int perf_swevent_add(struct perf_event *event, int flags)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004855{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004856 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004857 struct hw_perf_event *hwc = &event->hw;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004858 struct hlist_head *head;
4859
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +01004860 if (is_sampling_event(event)) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004861 hwc->last_period = hwc->sample_period;
4862 perf_swevent_set_period(event);
4863 }
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004864
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004865 hwc->state = !(flags & PERF_EF_START);
4866
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004867 head = find_swevent_head(swhash, event);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004868 if (WARN_ON_ONCE(!head))
4869 return -EINVAL;
4870
4871 hlist_add_head_rcu(&event->hlist_entry, head);
4872
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004873 return 0;
4874}
4875
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004876static void perf_swevent_del(struct perf_event *event, int flags)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004877{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004878 hlist_del_rcu(&event->hlist_entry);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02004879}
4880
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004881static void perf_swevent_start(struct perf_event *event, int flags)
Peter Zijlstrac6df8d52010-06-03 11:21:20 +02004882{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004883 event->hw.state = 0;
Peter Zijlstrac6df8d52010-06-03 11:21:20 +02004884}
4885
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004886static void perf_swevent_stop(struct perf_event *event, int flags)
Peter Zijlstrac6df8d52010-06-03 11:21:20 +02004887{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02004888 event->hw.state = PERF_HES_STOPPED;
Peter Zijlstrac6df8d52010-06-03 11:21:20 +02004889}
4890
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004891/* Deref the hlist from the update side */
4892static inline struct swevent_hlist *
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004893swevent_hlist_deref(struct swevent_htable *swhash)
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004894{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004895 return rcu_dereference_protected(swhash->swevent_hlist,
4896 lockdep_is_held(&swhash->hlist_mutex));
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004897}
4898
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004899static void swevent_hlist_release(struct swevent_htable *swhash)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004900{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004901 struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004902
Frederic Weisbecker49f135e2010-05-20 10:17:46 +02004903 if (!hlist)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004904 return;
4905
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004906 rcu_assign_pointer(swhash->swevent_hlist, NULL);
Lai Jiangshanfa4bbc42011-03-18 12:08:29 +08004907 kfree_rcu(hlist, rcu_head);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004908}
4909
4910static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4911{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004912 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004913
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004914 mutex_lock(&swhash->hlist_mutex);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004915
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004916 if (!--swhash->hlist_refcount)
4917 swevent_hlist_release(swhash);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004918
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004919 mutex_unlock(&swhash->hlist_mutex);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004920}
4921
4922static void swevent_hlist_put(struct perf_event *event)
4923{
4924 int cpu;
4925
4926 if (event->cpu != -1) {
4927 swevent_hlist_put_cpu(event, event->cpu);
4928 return;
4929 }
4930
4931 for_each_possible_cpu(cpu)
4932 swevent_hlist_put_cpu(event, cpu);
4933}
4934
4935static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4936{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004937 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004938 int err = 0;
4939
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004940 mutex_lock(&swhash->hlist_mutex);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004941
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004942 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004943 struct swevent_hlist *hlist;
4944
4945 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4946 if (!hlist) {
4947 err = -ENOMEM;
4948 goto exit;
4949 }
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004950 rcu_assign_pointer(swhash->swevent_hlist, hlist);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004951 }
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004952 swhash->hlist_refcount++;
Peter Zijlstra9ed60602010-06-11 17:36:35 +02004953exit:
Peter Zijlstrab28ab832010-09-06 14:48:15 +02004954 mutex_unlock(&swhash->hlist_mutex);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004955
4956 return err;
4957}
4958
4959static int swevent_hlist_get(struct perf_event *event)
4960{
4961 int err;
4962 int cpu, failed_cpu;
4963
4964 if (event->cpu != -1)
4965 return swevent_hlist_get_cpu(event, event->cpu);
4966
4967 get_online_cpus();
4968 for_each_possible_cpu(cpu) {
4969 err = swevent_hlist_get_cpu(event, cpu);
4970 if (err) {
4971 failed_cpu = cpu;
4972 goto fail;
4973 }
4974 }
4975 put_online_cpus();
4976
4977 return 0;
Peter Zijlstra9ed60602010-06-11 17:36:35 +02004978fail:
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02004979 for_each_possible_cpu(cpu) {
4980 if (cpu == failed_cpu)
4981 break;
4982 swevent_hlist_put_cpu(event, cpu);
4983 }
4984
4985 put_online_cpus();
4986 return err;
4987}
4988
Jason Barond430d3d2011-03-16 17:29:47 -04004989struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
Frederic Weisbecker95476b62010-04-14 23:42:18 +02004990
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02004991static void sw_perf_event_destroy(struct perf_event *event)
4992{
4993 u64 event_id = event->attr.config;
4994
4995 WARN_ON(event->parent);
4996
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +02004997 jump_label_dec(&perf_swevent_enabled[event_id]);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02004998 swevent_hlist_put(event);
4999}
5000
5001static int perf_swevent_init(struct perf_event *event)
5002{
5003 int event_id = event->attr.config;
5004
5005 if (event->attr.type != PERF_TYPE_SOFTWARE)
5006 return -ENOENT;
5007
5008 switch (event_id) {
5009 case PERF_COUNT_SW_CPU_CLOCK:
5010 case PERF_COUNT_SW_TASK_CLOCK:
5011 return -ENOENT;
5012
5013 default:
5014 break;
5015 }
5016
Dan Carpenterce677832010-10-24 21:50:42 +02005017 if (event_id >= PERF_COUNT_SW_MAX)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005018 return -ENOENT;
5019
5020 if (!event->parent) {
5021 int err;
5022
5023 err = swevent_hlist_get(event);
5024 if (err)
5025 return err;
5026
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +02005027 jump_label_inc(&perf_swevent_enabled[event_id]);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005028 event->destroy = sw_perf_event_destroy;
5029 }
5030
5031 return 0;
5032}
5033
5034static struct pmu perf_swevent = {
Peter Zijlstra89a1e182010-09-07 17:34:50 +02005035 .task_ctx_nr = perf_sw_context,
5036
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005037 .event_init = perf_swevent_init,
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005038 .add = perf_swevent_add,
5039 .del = perf_swevent_del,
5040 .start = perf_swevent_start,
5041 .stop = perf_swevent_stop,
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005042 .read = perf_swevent_read,
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005043};
Frederic Weisbecker95476b62010-04-14 23:42:18 +02005044
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005045#ifdef CONFIG_EVENT_TRACING
5046
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005047static int perf_tp_filter_match(struct perf_event *event,
Frederic Weisbecker95476b62010-04-14 23:42:18 +02005048 struct perf_sample_data *data)
5049{
5050 void *record = data->raw->data;
5051
5052 if (likely(!event->filter) || filter_match_preds(event->filter, record))
5053 return 1;
5054 return 0;
5055}
5056
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005057static int perf_tp_event_match(struct perf_event *event,
5058 struct perf_sample_data *data,
5059 struct pt_regs *regs)
5060{
Frederic Weisbeckera0f7d0f2011-03-07 21:27:09 +01005061 if (event->hw.state & PERF_HES_STOPPED)
5062 return 0;
Peter Zijlstra580d6072010-05-20 20:54:31 +02005063 /*
5064 * All tracepoints are from kernel-space.
5065 */
5066 if (event->attr.exclude_kernel)
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005067 return 0;
5068
5069 if (!perf_tp_filter_match(event, data))
5070 return 0;
5071
5072 return 1;
5073}
5074
5075void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
Peter Zijlstraecc55f82010-05-21 15:11:34 +02005076 struct pt_regs *regs, struct hlist_head *head, int rctx)
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005077{
5078 struct perf_sample_data data;
5079 struct perf_event *event;
5080 struct hlist_node *node;
5081
5082 struct perf_raw_record raw = {
5083 .size = entry_size,
5084 .data = record,
5085 };
5086
5087 perf_sample_data_init(&data, addr);
5088 data.raw = &raw;
5089
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005090 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5091 if (perf_tp_event_match(event, &data, regs))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02005092 perf_swevent_event(event, count, &data, regs);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005093 }
Peter Zijlstraecc55f82010-05-21 15:11:34 +02005094
5095 perf_swevent_put_recursion_context(rctx);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005096}
5097EXPORT_SYMBOL_GPL(perf_tp_event);
5098
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005099static void tp_perf_event_destroy(struct perf_event *event)
5100{
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005101 perf_trace_destroy(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005102}
5103
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005104static int perf_tp_event_init(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005105{
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005106 int err;
5107
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005108 if (event->attr.type != PERF_TYPE_TRACEPOINT)
5109 return -ENOENT;
5110
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02005111 err = perf_trace_init(event);
5112 if (err)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005113 return err;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005114
5115 event->destroy = tp_perf_event_destroy;
5116
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005117 return 0;
5118}
5119
5120static struct pmu perf_tracepoint = {
Peter Zijlstra89a1e182010-09-07 17:34:50 +02005121 .task_ctx_nr = perf_sw_context,
5122
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005123 .event_init = perf_tp_event_init,
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005124 .add = perf_trace_add,
5125 .del = perf_trace_del,
5126 .start = perf_swevent_start,
5127 .stop = perf_swevent_stop,
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005128 .read = perf_swevent_read,
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005129};
5130
5131static inline void perf_tp_register(void)
5132{
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005133 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005134}
Li Zefan6fb29152009-10-15 11:21:42 +08005135
5136static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5137{
5138 char *filter_str;
5139 int ret;
5140
5141 if (event->attr.type != PERF_TYPE_TRACEPOINT)
5142 return -EINVAL;
5143
5144 filter_str = strndup_user(arg, PAGE_SIZE);
5145 if (IS_ERR(filter_str))
5146 return PTR_ERR(filter_str);
5147
5148 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5149
5150 kfree(filter_str);
5151 return ret;
5152}
5153
5154static void perf_event_free_filter(struct perf_event *event)
5155{
5156 ftrace_profile_free_filter(event);
5157}
5158
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005159#else
Li Zefan6fb29152009-10-15 11:21:42 +08005160
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005161static inline void perf_tp_register(void)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005162{
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005163}
Li Zefan6fb29152009-10-15 11:21:42 +08005164
5165static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5166{
5167 return -ENOENT;
5168}
5169
5170static void perf_event_free_filter(struct perf_event *event)
5171{
5172}
5173
Li Zefan07b139c2009-12-21 14:27:35 +08005174#endif /* CONFIG_EVENT_TRACING */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005175
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02005176#ifdef CONFIG_HAVE_HW_BREAKPOINT
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01005177void perf_bp_event(struct perf_event *bp, void *data)
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02005178{
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01005179 struct perf_sample_data sample;
5180 struct pt_regs *regs = data;
5181
Peter Zijlstradc1d6282010-03-03 15:55:04 +01005182 perf_sample_data_init(&sample, bp->attr.bp_addr);
Frederic Weisbeckerf5ffe022009-11-23 15:42:34 +01005183
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005184 if (!bp->hw.state && !perf_exclude_event(bp, regs))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02005185 perf_swevent_event(bp, 1, &sample, regs);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +02005186}
5187#endif
5188
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005189/*
5190 * hrtimer based swevent callback
5191 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005192
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005193static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005194{
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005195 enum hrtimer_restart ret = HRTIMER_RESTART;
5196 struct perf_sample_data data;
5197 struct pt_regs *regs;
5198 struct perf_event *event;
5199 u64 period;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005200
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005201 event = container_of(hrtimer, struct perf_event, hw.hrtimer);
Peter Zijlstraba3dd362011-02-15 12:41:46 +01005202
5203 if (event->state != PERF_EVENT_STATE_ACTIVE)
5204 return HRTIMER_NORESTART;
5205
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005206 event->pmu->read(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005207
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005208 perf_sample_data_init(&data, 0);
5209 data.period = event->hw.last_period;
5210 regs = get_irq_regs();
5211
5212 if (regs && !perf_exclude_event(event, regs)) {
Paul E. McKenney77aeeeb2011-11-10 16:02:52 -08005213 if (!(event->attr.exclude_idle && is_idle_task(current)))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +02005214 if (perf_event_overflow(event, &data, regs))
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005215 ret = HRTIMER_NORESTART;
5216 }
5217
5218 period = max_t(u64, 10000, event->hw.sample_period);
5219 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5220
5221 return ret;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005222}
5223
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005224static void perf_swevent_start_hrtimer(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005225{
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005226 struct hw_perf_event *hwc = &event->hw;
Franck Bui-Huu5d508e82010-11-23 16:21:45 +01005227 s64 period;
5228
5229 if (!is_sampling_event(event))
5230 return;
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005231
Franck Bui-Huu5d508e82010-11-23 16:21:45 +01005232 period = local64_read(&hwc->period_left);
5233 if (period) {
5234 if (period < 0)
5235 period = 10000;
Peter Zijlstrafa407f32010-06-24 12:35:12 +02005236
Franck Bui-Huu5d508e82010-11-23 16:21:45 +01005237 local64_set(&hwc->period_left, 0);
5238 } else {
5239 period = max_t(u64, 10000, hwc->sample_period);
5240 }
5241 __hrtimer_start_range_ns(&hwc->hrtimer,
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005242 ns_to_ktime(period), 0,
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02005243 HRTIMER_MODE_REL_PINNED, 0);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005244}
5245
5246static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5247{
5248 struct hw_perf_event *hwc = &event->hw;
5249
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +01005250 if (is_sampling_event(event)) {
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005251 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
Peter Zijlstrafa407f32010-06-24 12:35:12 +02005252 local64_set(&hwc->period_left, ktime_to_ns(remaining));
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005253
5254 hrtimer_cancel(&hwc->hrtimer);
5255 }
5256}
5257
Peter Zijlstraba3dd362011-02-15 12:41:46 +01005258static void perf_swevent_init_hrtimer(struct perf_event *event)
5259{
5260 struct hw_perf_event *hwc = &event->hw;
5261
5262 if (!is_sampling_event(event))
5263 return;
5264
5265 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5266 hwc->hrtimer.function = perf_swevent_hrtimer;
5267
5268 /*
5269 * Since hrtimers have a fixed rate, we can do a static freq->period
5270 * mapping and avoid the whole period adjust feedback stuff.
5271 */
5272 if (event->attr.freq) {
5273 long freq = event->attr.sample_freq;
5274
5275 event->attr.sample_period = NSEC_PER_SEC / freq;
5276 hwc->sample_period = event->attr.sample_period;
5277 local64_set(&hwc->period_left, hwc->sample_period);
5278 event->attr.freq = 0;
5279 }
5280}
5281
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005282/*
5283 * Software event: cpu wall time clock
5284 */
5285
5286static void cpu_clock_event_update(struct perf_event *event)
5287{
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005288 s64 prev;
5289 u64 now;
5290
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005291 now = local_clock();
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005292 prev = local64_xchg(&event->hw.prev_count, now);
5293 local64_add(now - prev, &event->count);
5294}
5295
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005296static void cpu_clock_event_start(struct perf_event *event, int flags)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005297{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005298 local64_set(&event->hw.prev_count, local_clock());
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005299 perf_swevent_start_hrtimer(event);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005300}
5301
5302static void cpu_clock_event_stop(struct perf_event *event, int flags)
5303{
5304 perf_swevent_cancel_hrtimer(event);
5305 cpu_clock_event_update(event);
5306}
5307
5308static int cpu_clock_event_add(struct perf_event *event, int flags)
5309{
5310 if (flags & PERF_EF_START)
5311 cpu_clock_event_start(event, flags);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005312
5313 return 0;
5314}
5315
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005316static void cpu_clock_event_del(struct perf_event *event, int flags)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005317{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005318 cpu_clock_event_stop(event, flags);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005319}
5320
5321static void cpu_clock_event_read(struct perf_event *event)
5322{
5323 cpu_clock_event_update(event);
5324}
5325
5326static int cpu_clock_event_init(struct perf_event *event)
5327{
5328 if (event->attr.type != PERF_TYPE_SOFTWARE)
5329 return -ENOENT;
5330
5331 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5332 return -ENOENT;
5333
Peter Zijlstraba3dd362011-02-15 12:41:46 +01005334 perf_swevent_init_hrtimer(event);
5335
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005336 return 0;
5337}
5338
5339static struct pmu perf_cpu_clock = {
Peter Zijlstra89a1e182010-09-07 17:34:50 +02005340 .task_ctx_nr = perf_sw_context,
5341
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005342 .event_init = cpu_clock_event_init,
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005343 .add = cpu_clock_event_add,
5344 .del = cpu_clock_event_del,
5345 .start = cpu_clock_event_start,
5346 .stop = cpu_clock_event_stop,
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005347 .read = cpu_clock_event_read,
5348};
5349
5350/*
5351 * Software event: task time clock
5352 */
5353
5354static void task_clock_event_update(struct perf_event *event, u64 now)
5355{
5356 u64 prev;
5357 s64 delta;
5358
5359 prev = local64_xchg(&event->hw.prev_count, now);
5360 delta = now - prev;
5361 local64_add(delta, &event->count);
5362}
5363
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005364static void task_clock_event_start(struct perf_event *event, int flags)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005365{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005366 local64_set(&event->hw.prev_count, event->ctx->time);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005367 perf_swevent_start_hrtimer(event);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005368}
5369
5370static void task_clock_event_stop(struct perf_event *event, int flags)
5371{
5372 perf_swevent_cancel_hrtimer(event);
5373 task_clock_event_update(event, event->ctx->time);
5374}
5375
5376static int task_clock_event_add(struct perf_event *event, int flags)
5377{
5378 if (flags & PERF_EF_START)
5379 task_clock_event_start(event, flags);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005380
5381 return 0;
5382}
5383
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005384static void task_clock_event_del(struct perf_event *event, int flags)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005385{
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005386 task_clock_event_stop(event, PERF_EF_UPDATE);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005387}
5388
5389static void task_clock_event_read(struct perf_event *event)
5390{
Peter Zijlstra768a06e2011-02-22 16:52:24 +01005391 u64 now = perf_clock();
5392 u64 delta = now - event->ctx->timestamp;
5393 u64 time = event->ctx->time + delta;
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005394
5395 task_clock_event_update(event, time);
5396}
5397
5398static int task_clock_event_init(struct perf_event *event)
5399{
5400 if (event->attr.type != PERF_TYPE_SOFTWARE)
5401 return -ENOENT;
5402
5403 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5404 return -ENOENT;
5405
Peter Zijlstraba3dd362011-02-15 12:41:46 +01005406 perf_swevent_init_hrtimer(event);
5407
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005408 return 0;
5409}
5410
5411static struct pmu perf_task_clock = {
Peter Zijlstra89a1e182010-09-07 17:34:50 +02005412 .task_ctx_nr = perf_sw_context,
5413
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005414 .event_init = task_clock_event_init,
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +02005415 .add = task_clock_event_add,
5416 .del = task_clock_event_del,
5417 .start = task_clock_event_start,
5418 .stop = task_clock_event_stop,
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005419 .read = task_clock_event_read,
5420};
5421
Peter Zijlstraad5133b2010-06-15 12:22:39 +02005422static void perf_pmu_nop_void(struct pmu *pmu)
5423{
5424}
5425
5426static int perf_pmu_nop_int(struct pmu *pmu)
5427{
5428 return 0;
5429}
5430
5431static void perf_pmu_start_txn(struct pmu *pmu)
5432{
5433 perf_pmu_disable(pmu);
5434}
5435
5436static int perf_pmu_commit_txn(struct pmu *pmu)
5437{
5438 perf_pmu_enable(pmu);
5439 return 0;
5440}
5441
5442static void perf_pmu_cancel_txn(struct pmu *pmu)
5443{
5444 perf_pmu_enable(pmu);
5445}
5446
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005447/*
5448 * Ensures all contexts with the same task_ctx_nr have the same
5449 * pmu_cpu_context too.
5450 */
5451static void *find_pmu_context(int ctxn)
5452{
5453 struct pmu *pmu;
5454
5455 if (ctxn < 0)
5456 return NULL;
5457
5458 list_for_each_entry(pmu, &pmus, entry) {
5459 if (pmu->task_ctx_nr == ctxn)
5460 return pmu->pmu_cpu_context;
5461 }
5462
5463 return NULL;
5464}
5465
Peter Zijlstra51676952010-12-07 14:18:20 +01005466static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005467{
Peter Zijlstra51676952010-12-07 14:18:20 +01005468 int cpu;
5469
5470 for_each_possible_cpu(cpu) {
5471 struct perf_cpu_context *cpuctx;
5472
5473 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5474
5475 if (cpuctx->active_pmu == old_pmu)
5476 cpuctx->active_pmu = pmu;
5477 }
5478}
5479
5480static void free_pmu_context(struct pmu *pmu)
5481{
5482 struct pmu *i;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005483
5484 mutex_lock(&pmus_lock);
5485 /*
5486 * Like a real lame refcount.
5487 */
Peter Zijlstra51676952010-12-07 14:18:20 +01005488 list_for_each_entry(i, &pmus, entry) {
5489 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5490 update_pmu_context(i, pmu);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005491 goto out;
Peter Zijlstra51676952010-12-07 14:18:20 +01005492 }
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005493 }
5494
Peter Zijlstra51676952010-12-07 14:18:20 +01005495 free_percpu(pmu->pmu_cpu_context);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005496out:
5497 mutex_unlock(&pmus_lock);
5498}
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005499static struct idr pmu_idr;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005500
Peter Zijlstraabe43402010-11-17 23:17:37 +01005501static ssize_t
5502type_show(struct device *dev, struct device_attribute *attr, char *page)
5503{
5504 struct pmu *pmu = dev_get_drvdata(dev);
5505
5506 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5507}
5508
5509static struct device_attribute pmu_dev_attrs[] = {
5510 __ATTR_RO(type),
5511 __ATTR_NULL,
5512};
5513
5514static int pmu_bus_running;
5515static struct bus_type pmu_bus = {
5516 .name = "event_source",
5517 .dev_attrs = pmu_dev_attrs,
5518};
5519
5520static void pmu_dev_release(struct device *dev)
5521{
5522 kfree(dev);
5523}
5524
5525static int pmu_dev_alloc(struct pmu *pmu)
5526{
5527 int ret = -ENOMEM;
5528
5529 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5530 if (!pmu->dev)
5531 goto out;
5532
5533 device_initialize(pmu->dev);
5534 ret = dev_set_name(pmu->dev, "%s", pmu->name);
5535 if (ret)
5536 goto free_dev;
5537
5538 dev_set_drvdata(pmu->dev, pmu);
5539 pmu->dev->bus = &pmu_bus;
5540 pmu->dev->release = pmu_dev_release;
5541 ret = device_add(pmu->dev);
5542 if (ret)
5543 goto free_dev;
5544
5545out:
5546 return ret;
5547
5548free_dev:
5549 put_device(pmu->dev);
5550 goto out;
5551}
5552
Peter Zijlstra547e9fd2011-01-19 12:51:39 +01005553static struct lock_class_key cpuctx_mutex;
Peter Zijlstrafacc4302011-04-09 21:17:42 +02005554static struct lock_class_key cpuctx_lock;
Peter Zijlstra547e9fd2011-01-19 12:51:39 +01005555
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005556int perf_pmu_register(struct pmu *pmu, char *name, int type)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005557{
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005558 int cpu, ret;
Peter Zijlstra33696fc2010-06-14 08:49:00 +02005559
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005560 mutex_lock(&pmus_lock);
Peter Zijlstra33696fc2010-06-14 08:49:00 +02005561 ret = -ENOMEM;
5562 pmu->pmu_disable_count = alloc_percpu(int);
5563 if (!pmu->pmu_disable_count)
5564 goto unlock;
Peter Zijlstraad5133b2010-06-15 12:22:39 +02005565
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005566 pmu->type = -1;
5567 if (!name)
5568 goto skip_type;
5569 pmu->name = name;
5570
5571 if (type < 0) {
5572 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5573 if (!err)
5574 goto free_pdc;
5575
5576 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5577 if (err) {
5578 ret = err;
5579 goto free_pdc;
5580 }
5581 }
5582 pmu->type = type;
5583
Peter Zijlstraabe43402010-11-17 23:17:37 +01005584 if (pmu_bus_running) {
5585 ret = pmu_dev_alloc(pmu);
5586 if (ret)
5587 goto free_idr;
5588 }
5589
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005590skip_type:
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005591 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5592 if (pmu->pmu_cpu_context)
5593 goto got_cpu_context;
5594
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005595 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5596 if (!pmu->pmu_cpu_context)
Peter Zijlstraabe43402010-11-17 23:17:37 +01005597 goto free_dev;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005598
5599 for_each_possible_cpu(cpu) {
5600 struct perf_cpu_context *cpuctx;
5601
5602 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
Peter Zijlstraeb184472010-09-07 15:55:13 +02005603 __perf_event_init_context(&cpuctx->ctx);
Peter Zijlstra547e9fd2011-01-19 12:51:39 +01005604 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
Peter Zijlstrafacc4302011-04-09 21:17:42 +02005605 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
Peter Zijlstrab04243e2010-09-17 11:28:48 +02005606 cpuctx->ctx.type = cpu_context;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005607 cpuctx->ctx.pmu = pmu;
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02005608 cpuctx->jiffies_interval = 1;
5609 INIT_LIST_HEAD(&cpuctx->rotation_list);
Peter Zijlstra51676952010-12-07 14:18:20 +01005610 cpuctx->active_pmu = pmu;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005611 }
5612
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02005613got_cpu_context:
Peter Zijlstraad5133b2010-06-15 12:22:39 +02005614 if (!pmu->start_txn) {
5615 if (pmu->pmu_enable) {
5616 /*
5617 * If we have pmu_enable/pmu_disable calls, install
5618 * transaction stubs that use that to try and batch
5619 * hardware accesses.
5620 */
5621 pmu->start_txn = perf_pmu_start_txn;
5622 pmu->commit_txn = perf_pmu_commit_txn;
5623 pmu->cancel_txn = perf_pmu_cancel_txn;
5624 } else {
5625 pmu->start_txn = perf_pmu_nop_void;
5626 pmu->commit_txn = perf_pmu_nop_int;
5627 pmu->cancel_txn = perf_pmu_nop_void;
5628 }
5629 }
5630
5631 if (!pmu->pmu_enable) {
5632 pmu->pmu_enable = perf_pmu_nop_void;
5633 pmu->pmu_disable = perf_pmu_nop_void;
5634 }
5635
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005636 list_add_rcu(&pmu->entry, &pmus);
Peter Zijlstra33696fc2010-06-14 08:49:00 +02005637 ret = 0;
5638unlock:
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005639 mutex_unlock(&pmus_lock);
5640
Peter Zijlstra33696fc2010-06-14 08:49:00 +02005641 return ret;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005642
Peter Zijlstraabe43402010-11-17 23:17:37 +01005643free_dev:
5644 device_del(pmu->dev);
5645 put_device(pmu->dev);
5646
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005647free_idr:
5648 if (pmu->type >= PERF_TYPE_MAX)
5649 idr_remove(&pmu_idr, pmu->type);
5650
Peter Zijlstra108b02c2010-09-06 14:32:03 +02005651free_pdc:
5652 free_percpu(pmu->pmu_disable_count);
5653 goto unlock;
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005654}
5655
5656void perf_pmu_unregister(struct pmu *pmu)
5657{
5658 mutex_lock(&pmus_lock);
5659 list_del_rcu(&pmu->entry);
5660 mutex_unlock(&pmus_lock);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005661
5662 /*
Peter Zijlstracde8e882010-09-13 11:06:55 +02005663 * We dereference the pmu list under both SRCU and regular RCU, so
5664 * synchronize against both of those.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005665 */
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005666 synchronize_srcu(&pmus_srcu);
Peter Zijlstracde8e882010-09-13 11:06:55 +02005667 synchronize_rcu();
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005668
Peter Zijlstra33696fc2010-06-14 08:49:00 +02005669 free_percpu(pmu->pmu_disable_count);
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005670 if (pmu->type >= PERF_TYPE_MAX)
5671 idr_remove(&pmu_idr, pmu->type);
Peter Zijlstraabe43402010-11-17 23:17:37 +01005672 device_del(pmu->dev);
5673 put_device(pmu->dev);
Peter Zijlstra51676952010-12-07 14:18:20 +01005674 free_pmu_context(pmu);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005675}
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005676
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005677struct pmu *perf_init_event(struct perf_event *event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005678{
Peter Zijlstra51b0fe32010-06-11 13:35:57 +02005679 struct pmu *pmu = NULL;
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005680 int idx;
Lin Ming940c5b22011-02-27 21:13:31 +08005681 int ret;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005682
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005683 idx = srcu_read_lock(&pmus_srcu);
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005684
5685 rcu_read_lock();
5686 pmu = idr_find(&pmu_idr, event->attr.type);
5687 rcu_read_unlock();
Lin Ming940c5b22011-02-27 21:13:31 +08005688 if (pmu) {
Mark Rutland7e5b2a02011-08-11 12:31:20 +01005689 event->pmu = pmu;
Lin Ming940c5b22011-02-27 21:13:31 +08005690 ret = pmu->event_init(event);
5691 if (ret)
5692 pmu = ERR_PTR(ret);
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005693 goto unlock;
Lin Ming940c5b22011-02-27 21:13:31 +08005694 }
Peter Zijlstra2e80a822010-11-17 23:17:36 +01005695
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005696 list_for_each_entry_rcu(pmu, &pmus, entry) {
Mark Rutland7e5b2a02011-08-11 12:31:20 +01005697 event->pmu = pmu;
Lin Ming940c5b22011-02-27 21:13:31 +08005698 ret = pmu->event_init(event);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005699 if (!ret)
Peter Zijlstrae5f4d332010-09-10 17:38:06 +02005700 goto unlock;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02005701
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005702 if (ret != -ENOENT) {
5703 pmu = ERR_PTR(ret);
Peter Zijlstrae5f4d332010-09-10 17:38:06 +02005704 goto unlock;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005705 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005706 }
Peter Zijlstrae5f4d332010-09-10 17:38:06 +02005707 pmu = ERR_PTR(-ENOENT);
5708unlock:
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005709 srcu_read_unlock(&pmus_srcu, idx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005710
5711 return pmu;
5712}
5713
5714/*
5715 * Allocate and initialize a event structure
5716 */
5717static struct perf_event *
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02005718perf_event_alloc(struct perf_event_attr *attr, int cpu,
Peter Zijlstrad580ff82010-10-14 17:43:23 +02005719 struct task_struct *task,
5720 struct perf_event *group_leader,
5721 struct perf_event *parent_event,
Avi Kivity4dc0da82011-06-29 18:42:35 +03005722 perf_overflow_handler_t overflow_handler,
5723 void *context)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005724{
Peter Zijlstra51b0fe32010-06-11 13:35:57 +02005725 struct pmu *pmu;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005726 struct perf_event *event;
5727 struct hw_perf_event *hwc;
5728 long err;
5729
Oleg Nesterov66832eb2011-01-18 17:10:32 +01005730 if ((unsigned)cpu >= nr_cpu_ids) {
5731 if (!task || cpu != -1)
5732 return ERR_PTR(-EINVAL);
5733 }
5734
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02005735 event = kzalloc(sizeof(*event), GFP_KERNEL);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005736 if (!event)
5737 return ERR_PTR(-ENOMEM);
5738
5739 /*
5740 * Single events are their own group leaders, with an
5741 * empty sibling list:
5742 */
5743 if (!group_leader)
5744 group_leader = event;
5745
5746 mutex_init(&event->child_mutex);
5747 INIT_LIST_HEAD(&event->child_list);
5748
5749 INIT_LIST_HEAD(&event->group_entry);
5750 INIT_LIST_HEAD(&event->event_entry);
5751 INIT_LIST_HEAD(&event->sibling_list);
Peter Zijlstra10c6db12011-11-26 02:47:31 +01005752 INIT_LIST_HEAD(&event->rb_entry);
5753
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005754 init_waitqueue_head(&event->waitq);
Peter Zijlstrae360adb2010-10-14 14:01:34 +08005755 init_irq_work(&event->pending, perf_pending_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005756
5757 mutex_init(&event->mmap_mutex);
5758
5759 event->cpu = cpu;
5760 event->attr = *attr;
5761 event->group_leader = group_leader;
5762 event->pmu = NULL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005763 event->oncpu = -1;
5764
5765 event->parent = parent_event;
5766
5767 event->ns = get_pid_ns(current->nsproxy->pid_ns);
5768 event->id = atomic64_inc_return(&perf_event_id);
5769
5770 event->state = PERF_EVENT_STATE_INACTIVE;
5771
Peter Zijlstrad580ff82010-10-14 17:43:23 +02005772 if (task) {
5773 event->attach_state = PERF_ATTACH_TASK;
5774#ifdef CONFIG_HAVE_HW_BREAKPOINT
5775 /*
5776 * hw_breakpoint is a bit difficult here..
5777 */
5778 if (attr->type == PERF_TYPE_BREAKPOINT)
5779 event->hw.bp_target = task;
5780#endif
5781 }
5782
Avi Kivity4dc0da82011-06-29 18:42:35 +03005783 if (!overflow_handler && parent_event) {
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01005784 overflow_handler = parent_event->overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +03005785 context = parent_event->overflow_handler_context;
5786 }
Oleg Nesterov66832eb2011-01-18 17:10:32 +01005787
Frederic Weisbeckerb326e952009-12-05 09:44:31 +01005788 event->overflow_handler = overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +03005789 event->overflow_handler_context = context;
Frederic Weisbecker97eaf532009-10-18 15:33:50 +02005790
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005791 if (attr->disabled)
5792 event->state = PERF_EVENT_STATE_OFF;
5793
5794 pmu = NULL;
5795
5796 hwc = &event->hw;
5797 hwc->sample_period = attr->sample_period;
5798 if (attr->freq && attr->sample_freq)
5799 hwc->sample_period = 1;
5800 hwc->last_period = hwc->sample_period;
5801
Peter Zijlstrae7850592010-05-21 14:43:08 +02005802 local64_set(&hwc->period_left, hwc->sample_period);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005803
5804 /*
5805 * we currently do not support PERF_FORMAT_GROUP on inherited events
5806 */
5807 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5808 goto done;
5809
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02005810 pmu = perf_init_event(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005811
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005812done:
5813 err = 0;
5814 if (!pmu)
5815 err = -EINVAL;
5816 else if (IS_ERR(pmu))
5817 err = PTR_ERR(pmu);
5818
5819 if (err) {
5820 if (event->ns)
5821 put_pid_ns(event->ns);
5822 kfree(event);
5823 return ERR_PTR(err);
5824 }
5825
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005826 if (!event->parent) {
Peter Zijlstra82cd6de2010-10-14 17:57:23 +02005827 if (event->attach_state & PERF_ATTACH_TASK)
Gleb Natapovb2029522011-11-27 17:59:09 +02005828 jump_label_inc(&perf_sched_events.key);
Eric B Munson3af9e852010-05-18 15:30:49 +01005829 if (event->attr.mmap || event->attr.mmap_data)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005830 atomic_inc(&nr_mmap_events);
5831 if (event->attr.comm)
5832 atomic_inc(&nr_comm_events);
5833 if (event->attr.task)
5834 atomic_inc(&nr_task_events);
Frederic Weisbecker927c7a92010-07-01 16:20:36 +02005835 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
5836 err = get_callchain_buffers();
5837 if (err) {
5838 free_event(event);
5839 return ERR_PTR(err);
5840 }
5841 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005842 }
5843
5844 return event;
5845}
5846
5847static int perf_copy_attr(struct perf_event_attr __user *uattr,
5848 struct perf_event_attr *attr)
5849{
5850 u32 size;
5851 int ret;
5852
5853 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
5854 return -EFAULT;
5855
5856 /*
5857 * zero the full structure, so that a short copy will be nice.
5858 */
5859 memset(attr, 0, sizeof(*attr));
5860
5861 ret = get_user(size, &uattr->size);
5862 if (ret)
5863 return ret;
5864
5865 if (size > PAGE_SIZE) /* silly large */
5866 goto err_size;
5867
5868 if (!size) /* abi compat */
5869 size = PERF_ATTR_SIZE_VER0;
5870
5871 if (size < PERF_ATTR_SIZE_VER0)
5872 goto err_size;
5873
5874 /*
5875 * If we're handed a bigger struct than we know of,
5876 * ensure all the unknown bits are 0 - i.e. new
5877 * user-space does not rely on any kernel feature
5878 * extensions we dont know about yet.
5879 */
5880 if (size > sizeof(*attr)) {
5881 unsigned char __user *addr;
5882 unsigned char __user *end;
5883 unsigned char val;
5884
5885 addr = (void __user *)uattr + sizeof(*attr);
5886 end = (void __user *)uattr + size;
5887
5888 for (; addr < end; addr++) {
5889 ret = get_user(val, addr);
5890 if (ret)
5891 return ret;
5892 if (val)
5893 goto err_size;
5894 }
5895 size = sizeof(*attr);
5896 }
5897
5898 ret = copy_from_user(attr, uattr, size);
5899 if (ret)
5900 return -EFAULT;
5901
Mahesh Salgaonkarcd757642010-01-30 10:25:18 +05305902 if (attr->__reserved_1)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005903 return -EINVAL;
5904
5905 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
5906 return -EINVAL;
5907
5908 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
5909 return -EINVAL;
5910
5911out:
5912 return ret;
5913
5914err_size:
5915 put_user(sizeof(*attr), &uattr->size);
5916 ret = -E2BIG;
5917 goto out;
5918}
5919
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005920static int
5921perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005922{
Frederic Weisbecker76369132011-05-19 19:55:04 +02005923 struct ring_buffer *rb = NULL, *old_rb = NULL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005924 int ret = -EINVAL;
5925
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005926 if (!output_event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005927 goto set;
5928
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005929 /* don't allow circular references */
5930 if (event == output_event)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005931 goto out;
5932
Peter Zijlstra0f139302010-05-20 14:35:15 +02005933 /*
5934 * Don't allow cross-cpu buffers
5935 */
5936 if (output_event->cpu != event->cpu)
5937 goto out;
5938
5939 /*
Frederic Weisbecker76369132011-05-19 19:55:04 +02005940 * If its not a per-cpu rb, it must be the same task.
Peter Zijlstra0f139302010-05-20 14:35:15 +02005941 */
5942 if (output_event->cpu == -1 && output_event->ctx != event->ctx)
5943 goto out;
5944
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005945set:
5946 mutex_lock(&event->mmap_mutex);
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005947 /* Can't redirect output if we've got an active mmap() */
5948 if (atomic_read(&event->mmap_count))
5949 goto unlock;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005950
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005951 if (output_event) {
Frederic Weisbecker76369132011-05-19 19:55:04 +02005952 /* get the rb we want to redirect to */
5953 rb = ring_buffer_get(output_event);
5954 if (!rb)
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005955 goto unlock;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005956 }
5957
Frederic Weisbecker76369132011-05-19 19:55:04 +02005958 old_rb = event->rb;
5959 rcu_assign_pointer(event->rb, rb);
Peter Zijlstra10c6db12011-11-26 02:47:31 +01005960 if (old_rb)
5961 ring_buffer_detach(event, old_rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005962 ret = 0;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02005963unlock:
5964 mutex_unlock(&event->mmap_mutex);
5965
Frederic Weisbecker76369132011-05-19 19:55:04 +02005966 if (old_rb)
5967 ring_buffer_put(old_rb);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005968out:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005969 return ret;
5970}
5971
5972/**
5973 * sys_perf_event_open - open a performance event, associate it to a task/cpu
5974 *
5975 * @attr_uptr: event_id type attributes for monitoring/sampling
5976 * @pid: target pid
5977 * @cpu: target cpu
5978 * @group_fd: group leader event fd
5979 */
5980SYSCALL_DEFINE5(perf_event_open,
5981 struct perf_event_attr __user *, attr_uptr,
5982 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
5983{
Peter Zijlstrab04243e2010-09-17 11:28:48 +02005984 struct perf_event *group_leader = NULL, *output_event = NULL;
5985 struct perf_event *event, *sibling;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005986 struct perf_event_attr attr;
5987 struct perf_event_context *ctx;
5988 struct file *event_file = NULL;
5989 struct file *group_file = NULL;
Matt Helsley38a81da2010-09-13 13:01:20 -07005990 struct task_struct *task = NULL;
Peter Zijlstra89a1e182010-09-07 17:34:50 +02005991 struct pmu *pmu;
Al Viroea635c62010-05-26 17:40:29 -04005992 int event_fd;
Peter Zijlstrab04243e2010-09-17 11:28:48 +02005993 int move_group = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005994 int fput_needed = 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005995 int err;
5996
5997 /* for future expandability... */
Stephane Eraniane5d13672011-02-14 11:20:01 +02005998 if (flags & ~PERF_FLAG_ALL)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005999 return -EINVAL;
6000
6001 err = perf_copy_attr(attr_uptr, &attr);
6002 if (err)
6003 return err;
6004
6005 if (!attr.exclude_kernel) {
6006 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6007 return -EACCES;
6008 }
6009
6010 if (attr.freq) {
6011 if (attr.sample_freq > sysctl_perf_event_sample_rate)
6012 return -EINVAL;
6013 }
6014
Stephane Eraniane5d13672011-02-14 11:20:01 +02006015 /*
6016 * In cgroup mode, the pid argument is used to pass the fd
6017 * opened to the cgroup directory in cgroupfs. The cpu argument
6018 * designates the cpu on which to monitor threads from that
6019 * cgroup.
6020 */
6021 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6022 return -EINVAL;
6023
Al Viroea635c62010-05-26 17:40:29 -04006024 event_fd = get_unused_fd_flags(O_RDWR);
6025 if (event_fd < 0)
6026 return event_fd;
6027
Peter Zijlstraac9721f2010-05-27 12:54:41 +02006028 if (group_fd != -1) {
6029 group_leader = perf_fget_light(group_fd, &fput_needed);
6030 if (IS_ERR(group_leader)) {
6031 err = PTR_ERR(group_leader);
Stephane Eraniand14b12d2010-09-17 11:28:47 +02006032 goto err_fd;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02006033 }
6034 group_file = group_leader->filp;
6035 if (flags & PERF_FLAG_FD_OUTPUT)
6036 output_event = group_leader;
6037 if (flags & PERF_FLAG_FD_NO_GROUP)
6038 group_leader = NULL;
6039 }
6040
Stephane Eraniane5d13672011-02-14 11:20:01 +02006041 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
Peter Zijlstrac6be5a52010-10-14 16:59:46 +02006042 task = find_lively_task_by_vpid(pid);
6043 if (IS_ERR(task)) {
6044 err = PTR_ERR(task);
6045 goto err_group_fd;
6046 }
6047 }
6048
Avi Kivity4dc0da82011-06-29 18:42:35 +03006049 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6050 NULL, NULL);
Stephane Eraniand14b12d2010-09-17 11:28:47 +02006051 if (IS_ERR(event)) {
6052 err = PTR_ERR(event);
Peter Zijlstrac6be5a52010-10-14 16:59:46 +02006053 goto err_task;
Stephane Eraniand14b12d2010-09-17 11:28:47 +02006054 }
6055
Stephane Eraniane5d13672011-02-14 11:20:01 +02006056 if (flags & PERF_FLAG_PID_CGROUP) {
6057 err = perf_cgroup_connect(pid, event, &attr, group_leader);
6058 if (err)
6059 goto err_alloc;
Peter Zijlstra08309372011-03-03 11:31:20 +01006060 /*
6061 * one more event:
6062 * - that has cgroup constraint on event->cpu
6063 * - that may need work on context switch
6064 */
6065 atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
Gleb Natapovb2029522011-11-27 17:59:09 +02006066 jump_label_inc(&perf_sched_events.key);
Stephane Eraniane5d13672011-02-14 11:20:01 +02006067 }
6068
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006069 /*
Peter Zijlstra89a1e182010-09-07 17:34:50 +02006070 * Special case software events and allow them to be part of
6071 * any hardware group.
6072 */
6073 pmu = event->pmu;
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006074
6075 if (group_leader &&
6076 (is_software_event(event) != is_software_event(group_leader))) {
6077 if (is_software_event(event)) {
6078 /*
6079 * If event and group_leader are not both a software
6080 * event, and event is, then group leader is not.
6081 *
6082 * Allow the addition of software events to !software
6083 * groups, this is safe because software events never
6084 * fail to schedule.
6085 */
6086 pmu = group_leader->pmu;
6087 } else if (is_software_event(group_leader) &&
6088 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6089 /*
6090 * In case the group is a pure software group, and we
6091 * try to add a hardware event, move the whole group to
6092 * the hardware context.
6093 */
6094 move_group = 1;
6095 }
6096 }
Peter Zijlstra89a1e182010-09-07 17:34:50 +02006097
6098 /*
6099 * Get the target context (task or percpu):
6100 */
Matt Helsley38a81da2010-09-13 13:01:20 -07006101 ctx = find_get_context(pmu, task, cpu);
Peter Zijlstra89a1e182010-09-07 17:34:50 +02006102 if (IS_ERR(ctx)) {
6103 err = PTR_ERR(ctx);
Peter Zijlstrac6be5a52010-10-14 16:59:46 +02006104 goto err_alloc;
Peter Zijlstra89a1e182010-09-07 17:34:50 +02006105 }
6106
Peter Zijlstrafd1edb32011-03-28 13:13:56 +02006107 if (task) {
6108 put_task_struct(task);
6109 task = NULL;
6110 }
6111
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006112 /*
6113 * Look up the group leader (we will attach this event to it):
6114 */
Peter Zijlstraac9721f2010-05-27 12:54:41 +02006115 if (group_leader) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006116 err = -EINVAL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006117
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006118 /*
6119 * Do not allow a recursive hierarchy (this new sibling
6120 * becoming part of another group-sibling):
6121 */
6122 if (group_leader->group_leader != group_leader)
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006123 goto err_context;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006124 /*
6125 * Do not allow to attach to a group in a different
6126 * task or CPU context:
6127 */
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006128 if (move_group) {
6129 if (group_leader->ctx->type != ctx->type)
6130 goto err_context;
6131 } else {
6132 if (group_leader->ctx != ctx)
6133 goto err_context;
6134 }
6135
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006136 /*
6137 * Only a group leader can be exclusive or pinned
6138 */
6139 if (attr.exclusive || attr.pinned)
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006140 goto err_context;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02006141 }
6142
6143 if (output_event) {
6144 err = perf_event_set_output(event, output_event);
6145 if (err)
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006146 goto err_context;
Peter Zijlstraac9721f2010-05-27 12:54:41 +02006147 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006148
Al Viroea635c62010-05-26 17:40:29 -04006149 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6150 if (IS_ERR(event_file)) {
6151 err = PTR_ERR(event_file);
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006152 goto err_context;
Al Viroea635c62010-05-26 17:40:29 -04006153 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006154
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006155 if (move_group) {
6156 struct perf_event_context *gctx = group_leader->ctx;
6157
6158 mutex_lock(&gctx->mutex);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006159 perf_remove_from_context(group_leader);
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006160 list_for_each_entry(sibling, &group_leader->sibling_list,
6161 group_entry) {
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006162 perf_remove_from_context(sibling);
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006163 put_ctx(gctx);
6164 }
6165 mutex_unlock(&gctx->mutex);
6166 put_ctx(gctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006167 }
6168
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006169 event->filp = event_file;
6170 WARN_ON_ONCE(ctx->parent_ctx);
6171 mutex_lock(&ctx->mutex);
Peter Zijlstrab04243e2010-09-17 11:28:48 +02006172
6173 if (move_group) {
6174 perf_install_in_context(ctx, group_leader, cpu);
6175 get_ctx(ctx);
6176 list_for_each_entry(sibling, &group_leader->sibling_list,
6177 group_entry) {
6178 perf_install_in_context(ctx, sibling, cpu);
6179 get_ctx(ctx);
6180 }
6181 }
6182
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006183 perf_install_in_context(ctx, event, cpu);
6184 ++ctx->generation;
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006185 perf_unpin_context(ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006186 mutex_unlock(&ctx->mutex);
6187
6188 event->owner = current;
Peter Zijlstra88821352010-11-09 19:01:43 +01006189
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006190 mutex_lock(&current->perf_event_mutex);
6191 list_add_tail(&event->owner_entry, &current->perf_event_list);
6192 mutex_unlock(&current->perf_event_mutex);
6193
Peter Zijlstra8a495422010-05-27 15:47:49 +02006194 /*
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02006195 * Precalculate sample_data sizes
6196 */
6197 perf_event__header_size(event);
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -02006198 perf_event__id_header_size(event);
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -02006199
6200 /*
Peter Zijlstra8a495422010-05-27 15:47:49 +02006201 * Drop the reference on the group_event after placing the
6202 * new event on the sibling_list. This ensures destruction
6203 * of the group leader will find the pointer to itself in
6204 * perf_group_detach().
6205 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006206 fput_light(group_file, fput_needed);
Al Viroea635c62010-05-26 17:40:29 -04006207 fd_install(event_fd, event_file);
6208 return event_fd;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006209
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006210err_context:
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006211 perf_unpin_context(ctx);
Al Viroea635c62010-05-26 17:40:29 -04006212 put_ctx(ctx);
Peter Zijlstrac6be5a52010-10-14 16:59:46 +02006213err_alloc:
6214 free_event(event);
Peter Zijlstrae7d0bc02010-10-14 16:54:51 +02006215err_task:
6216 if (task)
6217 put_task_struct(task);
Peter Zijlstra89a1e182010-09-07 17:34:50 +02006218err_group_fd:
6219 fput_light(group_file, fput_needed);
Al Viroea635c62010-05-26 17:40:29 -04006220err_fd:
6221 put_unused_fd(event_fd);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006222 return err;
6223}
6224
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006225/**
6226 * perf_event_create_kernel_counter
6227 *
6228 * @attr: attributes of the counter to create
6229 * @cpu: cpu in which the counter is bound
Matt Helsley38a81da2010-09-13 13:01:20 -07006230 * @task: task to profile (NULL for percpu)
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006231 */
6232struct perf_event *
6233perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
Matt Helsley38a81da2010-09-13 13:01:20 -07006234 struct task_struct *task,
Avi Kivity4dc0da82011-06-29 18:42:35 +03006235 perf_overflow_handler_t overflow_handler,
6236 void *context)
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006237{
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006238 struct perf_event_context *ctx;
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006239 struct perf_event *event;
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006240 int err;
6241
6242 /*
6243 * Get the target context (task or percpu):
6244 */
6245
Avi Kivity4dc0da82011-06-29 18:42:35 +03006246 event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6247 overflow_handler, context);
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01006248 if (IS_ERR(event)) {
6249 err = PTR_ERR(event);
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006250 goto err;
6251 }
6252
Matt Helsley38a81da2010-09-13 13:01:20 -07006253 ctx = find_get_context(event->pmu, task, cpu);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006254 if (IS_ERR(ctx)) {
6255 err = PTR_ERR(ctx);
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006256 goto err_free;
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01006257 }
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006258
6259 event->filp = NULL;
6260 WARN_ON_ONCE(ctx->parent_ctx);
6261 mutex_lock(&ctx->mutex);
6262 perf_install_in_context(ctx, event, cpu);
6263 ++ctx->generation;
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006264 perf_unpin_context(ctx);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006265 mutex_unlock(&ctx->mutex);
6266
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006267 return event;
6268
Peter Zijlstrac3f00c72010-08-18 14:37:15 +02006269err_free:
6270 free_event(event);
6271err:
Frederic Weisbeckerc6567f62009-11-26 05:35:41 +01006272 return ERR_PTR(err);
Arjan van de Venfb0459d2009-09-25 12:25:56 +02006273}
6274EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6275
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006276static void sync_child_event(struct perf_event *child_event,
6277 struct task_struct *child)
6278{
6279 struct perf_event *parent_event = child_event->parent;
6280 u64 child_val;
6281
6282 if (child_event->attr.inherit_stat)
6283 perf_event_read_event(child_event, child);
6284
Peter Zijlstrab5e58792010-05-21 14:43:12 +02006285 child_val = perf_event_count(child_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006286
6287 /*
6288 * Add back the child's count to the parent's count:
6289 */
Peter Zijlstraa6e6dea2010-05-21 14:27:58 +02006290 atomic64_add(child_val, &parent_event->child_count);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006291 atomic64_add(child_event->total_time_enabled,
6292 &parent_event->child_total_time_enabled);
6293 atomic64_add(child_event->total_time_running,
6294 &parent_event->child_total_time_running);
6295
6296 /*
6297 * Remove this event from the parent's list
6298 */
6299 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6300 mutex_lock(&parent_event->child_mutex);
6301 list_del_init(&child_event->child_list);
6302 mutex_unlock(&parent_event->child_mutex);
6303
6304 /*
6305 * Release the parent event, if this was the last
6306 * reference to it.
6307 */
6308 fput(parent_event->filp);
6309}
6310
6311static void
6312__perf_event_exit_task(struct perf_event *child_event,
6313 struct perf_event_context *child_ctx,
6314 struct task_struct *child)
6315{
Peter Zijlstra38b435b2011-03-15 14:37:10 +01006316 if (child_event->parent) {
6317 raw_spin_lock_irq(&child_ctx->lock);
6318 perf_group_detach(child_event);
6319 raw_spin_unlock_irq(&child_ctx->lock);
6320 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006321
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006322 perf_remove_from_context(child_event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006323
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006324 /*
Peter Zijlstra38b435b2011-03-15 14:37:10 +01006325 * It can happen that the parent exits first, and has events
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006326 * that are still around due to the child reference. These
Peter Zijlstra38b435b2011-03-15 14:37:10 +01006327 * events need to be zapped.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006328 */
Peter Zijlstra38b435b2011-03-15 14:37:10 +01006329 if (child_event->parent) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006330 sync_child_event(child_event, child);
6331 free_event(child_event);
6332 }
6333}
6334
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006335static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006336{
6337 struct perf_event *child_event, *tmp;
6338 struct perf_event_context *child_ctx;
6339 unsigned long flags;
6340
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006341 if (likely(!child->perf_event_ctxp[ctxn])) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006342 perf_event_task(child, NULL, 0);
6343 return;
6344 }
6345
6346 local_irq_save(flags);
6347 /*
6348 * We can't reschedule here because interrupts are disabled,
6349 * and either child is current or it is a task that can't be
6350 * scheduled, so we are now safe from rescheduling changing
6351 * our context.
6352 */
Oleg Nesterov806839b2011-01-21 18:45:47 +01006353 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006354
6355 /*
6356 * Take the context lock here so that if find_get_context is
6357 * reading child->perf_event_ctxp, we wait until it has
6358 * incremented the context's refcount before we do put_ctx below.
6359 */
Thomas Gleixnere625cce2009-11-17 18:02:06 +01006360 raw_spin_lock(&child_ctx->lock);
Peter Zijlstra04dc2db2011-04-09 21:17:43 +02006361 task_ctx_sched_out(child_ctx);
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006362 child->perf_event_ctxp[ctxn] = NULL;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006363 /*
6364 * If this context is a clone; unclone it so it can't get
6365 * swapped to another process while we're removing all
6366 * the events from it.
6367 */
6368 unclone_ctx(child_ctx);
Peter Zijlstra5e942bb2009-11-23 11:37:26 +01006369 update_context_time(child_ctx);
Thomas Gleixnere625cce2009-11-17 18:02:06 +01006370 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006371
6372 /*
6373 * Report the task dead after unscheduling the events so that we
6374 * won't get any samples after PERF_RECORD_EXIT. We can however still
6375 * get a few PERF_RECORD_READ events.
6376 */
6377 perf_event_task(child, child_ctx, 0);
6378
6379 /*
6380 * We can recurse on the same lock type through:
6381 *
6382 * __perf_event_exit_task()
6383 * sync_child_event()
6384 * fput(parent_event->filp)
6385 * perf_release()
6386 * mutex_lock(&ctx->mutex)
6387 *
6388 * But since its the parent context it won't be the same instance.
6389 */
Peter Zijlstraa0507c82010-05-06 15:42:53 +02006390 mutex_lock(&child_ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006391
6392again:
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006393 list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6394 group_entry)
6395 __perf_event_exit_task(child_event, child_ctx, child);
6396
6397 list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006398 group_entry)
6399 __perf_event_exit_task(child_event, child_ctx, child);
6400
6401 /*
6402 * If the last event was a group event, it will have appended all
6403 * its siblings to the list, but we obtained 'tmp' before that which
6404 * will still point to the list head terminating the iteration.
6405 */
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006406 if (!list_empty(&child_ctx->pinned_groups) ||
6407 !list_empty(&child_ctx->flexible_groups))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006408 goto again;
6409
6410 mutex_unlock(&child_ctx->mutex);
6411
6412 put_ctx(child_ctx);
6413}
6414
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006415/*
6416 * When a child task exits, feed back event values to parent events.
6417 */
6418void perf_event_exit_task(struct task_struct *child)
6419{
Peter Zijlstra88821352010-11-09 19:01:43 +01006420 struct perf_event *event, *tmp;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006421 int ctxn;
6422
Peter Zijlstra88821352010-11-09 19:01:43 +01006423 mutex_lock(&child->perf_event_mutex);
6424 list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6425 owner_entry) {
6426 list_del_init(&event->owner_entry);
6427
6428 /*
6429 * Ensure the list deletion is visible before we clear
6430 * the owner, closes a race against perf_release() where
6431 * we need to serialize on the owner->perf_event_mutex.
6432 */
6433 smp_wmb();
6434 event->owner = NULL;
6435 }
6436 mutex_unlock(&child->perf_event_mutex);
6437
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006438 for_each_task_context_nr(ctxn)
6439 perf_event_exit_task_context(child, ctxn);
6440}
6441
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006442static void perf_free_event(struct perf_event *event,
6443 struct perf_event_context *ctx)
6444{
6445 struct perf_event *parent = event->parent;
6446
6447 if (WARN_ON_ONCE(!parent))
6448 return;
6449
6450 mutex_lock(&parent->child_mutex);
6451 list_del_init(&event->child_list);
6452 mutex_unlock(&parent->child_mutex);
6453
6454 fput(parent->filp);
6455
Peter Zijlstra8a495422010-05-27 15:47:49 +02006456 perf_group_detach(event);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006457 list_del_event(event, ctx);
6458 free_event(event);
6459}
6460
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006461/*
6462 * free an unexposed, unused context as created by inheritance by
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006463 * perf_event_init_task below, used by fork() in case of fail.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006464 */
6465void perf_event_free_task(struct task_struct *task)
6466{
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006467 struct perf_event_context *ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006468 struct perf_event *event, *tmp;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006469 int ctxn;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006470
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006471 for_each_task_context_nr(ctxn) {
6472 ctx = task->perf_event_ctxp[ctxn];
6473 if (!ctx)
6474 continue;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006475
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006476 mutex_lock(&ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006477again:
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006478 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6479 group_entry)
6480 perf_free_event(event, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006481
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006482 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6483 group_entry)
6484 perf_free_event(event, ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006485
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006486 if (!list_empty(&ctx->pinned_groups) ||
6487 !list_empty(&ctx->flexible_groups))
6488 goto again;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006489
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006490 mutex_unlock(&ctx->mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006491
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006492 put_ctx(ctx);
6493 }
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006494}
6495
Peter Zijlstra4e231c72010-09-09 21:01:59 +02006496void perf_event_delayed_put(struct task_struct *task)
6497{
6498 int ctxn;
6499
6500 for_each_task_context_nr(ctxn)
6501 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6502}
6503
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006504/*
6505 * inherit a event from parent task to child task:
6506 */
6507static struct perf_event *
6508inherit_event(struct perf_event *parent_event,
6509 struct task_struct *parent,
6510 struct perf_event_context *parent_ctx,
6511 struct task_struct *child,
6512 struct perf_event *group_leader,
6513 struct perf_event_context *child_ctx)
6514{
6515 struct perf_event *child_event;
Peter Zijlstracee010e2010-09-10 12:51:54 +02006516 unsigned long flags;
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006517
6518 /*
6519 * Instead of creating recursive hierarchies of events,
6520 * we link inherited events back to the original parent,
6521 * which has a filp for sure, which we use as the reference
6522 * count:
6523 */
6524 if (parent_event->parent)
6525 parent_event = parent_event->parent;
6526
6527 child_event = perf_event_alloc(&parent_event->attr,
6528 parent_event->cpu,
Peter Zijlstrad580ff82010-10-14 17:43:23 +02006529 child,
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006530 group_leader, parent_event,
Avi Kivity4dc0da82011-06-29 18:42:35 +03006531 NULL, NULL);
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006532 if (IS_ERR(child_event))
6533 return child_event;
6534 get_ctx(child_ctx);
6535
6536 /*
6537 * Make the child state follow the state of the parent event,
6538 * not its attr.disabled bit. We hold the parent's mutex,
6539 * so we won't race with perf_event_{en, dis}able_family.
6540 */
6541 if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
6542 child_event->state = PERF_EVENT_STATE_INACTIVE;
6543 else
6544 child_event->state = PERF_EVENT_STATE_OFF;
6545
6546 if (parent_event->attr.freq) {
6547 u64 sample_period = parent_event->hw.sample_period;
6548 struct hw_perf_event *hwc = &child_event->hw;
6549
6550 hwc->sample_period = sample_period;
6551 hwc->last_period = sample_period;
6552
6553 local64_set(&hwc->period_left, sample_period);
6554 }
6555
6556 child_event->ctx = child_ctx;
6557 child_event->overflow_handler = parent_event->overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +03006558 child_event->overflow_handler_context
6559 = parent_event->overflow_handler_context;
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006560
6561 /*
Thomas Gleixner614b6782010-12-03 16:24:32 -02006562 * Precalculate sample_data sizes
6563 */
6564 perf_event__header_size(child_event);
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -02006565 perf_event__id_header_size(child_event);
Thomas Gleixner614b6782010-12-03 16:24:32 -02006566
6567 /*
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006568 * Link it up in the child's context:
6569 */
Peter Zijlstracee010e2010-09-10 12:51:54 +02006570 raw_spin_lock_irqsave(&child_ctx->lock, flags);
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006571 add_event_to_ctx(child_event, child_ctx);
Peter Zijlstracee010e2010-09-10 12:51:54 +02006572 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
Peter Zijlstra97dee4f2010-09-07 15:35:33 +02006573
6574 /*
6575 * Get a reference to the parent filp - we will fput it
6576 * when the child event exits. This is safe to do because
6577 * we are in the parent and we know that the filp still
6578 * exists and has a nonzero count:
6579 */
6580 atomic_long_inc(&parent_event->filp->f_count);
6581
6582 /*
6583 * Link this into the parent event's child list
6584 */
6585 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6586 mutex_lock(&parent_event->child_mutex);
6587 list_add_tail(&child_event->child_list, &parent_event->child_list);
6588 mutex_unlock(&parent_event->child_mutex);
6589
6590 return child_event;
6591}
6592
6593static int inherit_group(struct perf_event *parent_event,
6594 struct task_struct *parent,
6595 struct perf_event_context *parent_ctx,
6596 struct task_struct *child,
6597 struct perf_event_context *child_ctx)
6598{
6599 struct perf_event *leader;
6600 struct perf_event *sub;
6601 struct perf_event *child_ctr;
6602
6603 leader = inherit_event(parent_event, parent, parent_ctx,
6604 child, NULL, child_ctx);
6605 if (IS_ERR(leader))
6606 return PTR_ERR(leader);
6607 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
6608 child_ctr = inherit_event(sub, parent, parent_ctx,
6609 child, leader, child_ctx);
6610 if (IS_ERR(child_ctr))
6611 return PTR_ERR(child_ctr);
6612 }
6613 return 0;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006614}
6615
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006616static int
6617inherit_task_group(struct perf_event *event, struct task_struct *parent,
6618 struct perf_event_context *parent_ctx,
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006619 struct task_struct *child, int ctxn,
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006620 int *inherited_all)
6621{
6622 int ret;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006623 struct perf_event_context *child_ctx;
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006624
6625 if (!event->attr.inherit) {
6626 *inherited_all = 0;
6627 return 0;
6628 }
6629
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006630 child_ctx = child->perf_event_ctxp[ctxn];
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006631 if (!child_ctx) {
6632 /*
6633 * This is executed from the parent task context, so
6634 * inherit events that have been marked for cloning.
6635 * First allocate and initialize a context for the
6636 * child.
6637 */
6638
Peter Zijlstraeb184472010-09-07 15:55:13 +02006639 child_ctx = alloc_perf_context(event->pmu, child);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006640 if (!child_ctx)
6641 return -ENOMEM;
6642
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006643 child->perf_event_ctxp[ctxn] = child_ctx;
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006644 }
6645
6646 ret = inherit_group(event, parent, parent_ctx,
6647 child, child_ctx);
6648
6649 if (ret)
6650 *inherited_all = 0;
6651
6652 return ret;
6653}
6654
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006655/*
6656 * Initialize the perf_event context in task_struct
6657 */
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006658int perf_event_init_context(struct task_struct *child, int ctxn)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006659{
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006660 struct perf_event_context *child_ctx, *parent_ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006661 struct perf_event_context *cloned_ctx;
6662 struct perf_event *event;
6663 struct task_struct *parent = current;
6664 int inherited_all = 1;
Thomas Gleixnerdddd3372010-11-24 10:05:55 +01006665 unsigned long flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006666 int ret = 0;
6667
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006668 if (likely(!parent->perf_event_ctxp[ctxn]))
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006669 return 0;
6670
6671 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006672 * If the parent's context is a clone, pin it so it won't get
6673 * swapped under us.
6674 */
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006675 parent_ctx = perf_pin_task_context(parent, ctxn);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006676
6677 /*
6678 * No need to check if parent_ctx != NULL here; since we saw
6679 * it non-NULL earlier, the only reason for it to become NULL
6680 * is if we exit, and since we're currently in the middle of
6681 * a fork we can't be exiting at the same time.
6682 */
6683
6684 /*
6685 * Lock the parent list. No need to lock the child - not PID
6686 * hashed yet and not running, so nobody can access it.
6687 */
6688 mutex_lock(&parent_ctx->mutex);
6689
6690 /*
6691 * We dont have to disable NMIs - we are only looking at
6692 * the list, not manipulating it:
6693 */
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006694 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006695 ret = inherit_task_group(event, parent, parent_ctx,
6696 child, ctxn, &inherited_all);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006697 if (ret)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006698 break;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006699 }
6700
Thomas Gleixnerdddd3372010-11-24 10:05:55 +01006701 /*
6702 * We can't hold ctx->lock when iterating the ->flexible_group list due
6703 * to allocations, but we need to prevent rotation because
6704 * rotate_ctx() will change the list from interrupt context.
6705 */
6706 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6707 parent_ctx->rotate_disable = 1;
6708 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6709
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006710 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006711 ret = inherit_task_group(event, parent, parent_ctx,
6712 child, ctxn, &inherited_all);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006713 if (ret)
6714 break;
6715 }
6716
Thomas Gleixnerdddd3372010-11-24 10:05:55 +01006717 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6718 parent_ctx->rotate_disable = 0;
Thomas Gleixnerdddd3372010-11-24 10:05:55 +01006719
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006720 child_ctx = child->perf_event_ctxp[ctxn];
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006721
Peter Zijlstra05cbaa22009-12-30 16:00:35 +01006722 if (child_ctx && inherited_all) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006723 /*
6724 * Mark the child context as a clone of the parent
6725 * context, or of whatever the parent is a clone of.
Peter Zijlstrac5ed5142011-01-17 13:45:37 +01006726 *
6727 * Note that if the parent is a clone, the holding of
6728 * parent_ctx->lock avoids it from being uncloned.
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006729 */
Peter Zijlstrac5ed5142011-01-17 13:45:37 +01006730 cloned_ctx = parent_ctx->parent_ctx;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006731 if (cloned_ctx) {
6732 child_ctx->parent_ctx = cloned_ctx;
6733 child_ctx->parent_gen = parent_ctx->parent_gen;
6734 } else {
6735 child_ctx->parent_ctx = parent_ctx;
6736 child_ctx->parent_gen = parent_ctx->generation;
6737 }
6738 get_ctx(child_ctx->parent_ctx);
6739 }
6740
Peter Zijlstrac5ed5142011-01-17 13:45:37 +01006741 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006742 mutex_unlock(&parent_ctx->mutex);
6743
6744 perf_unpin_context(parent_ctx);
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006745 put_ctx(parent_ctx);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006746
6747 return ret;
6748}
6749
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006750/*
6751 * Initialize the perf_event context in task_struct
6752 */
6753int perf_event_init_task(struct task_struct *child)
6754{
6755 int ctxn, ret;
6756
Oleg Nesterov8550d7c2011-01-19 19:22:28 +01006757 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
6758 mutex_init(&child->perf_event_mutex);
6759 INIT_LIST_HEAD(&child->perf_event_list);
6760
Peter Zijlstra8dc85d52010-09-02 16:50:03 +02006761 for_each_task_context_nr(ctxn) {
6762 ret = perf_event_init_context(child, ctxn);
6763 if (ret)
6764 return ret;
6765 }
6766
6767 return 0;
6768}
6769
Paul Mackerras220b1402010-03-10 20:45:52 +11006770static void __init perf_event_init_all_cpus(void)
6771{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006772 struct swevent_htable *swhash;
Paul Mackerras220b1402010-03-10 20:45:52 +11006773 int cpu;
Paul Mackerras220b1402010-03-10 20:45:52 +11006774
6775 for_each_possible_cpu(cpu) {
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006776 swhash = &per_cpu(swevent_htable, cpu);
6777 mutex_init(&swhash->hlist_mutex);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02006778 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
Paul Mackerras220b1402010-03-10 20:45:52 +11006779 }
6780}
6781
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006782static void __cpuinit perf_event_init_cpu(int cpu)
6783{
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006784 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006785
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006786 mutex_lock(&swhash->hlist_mutex);
Linus Torvalds4536e4d2011-11-03 07:44:04 -07006787 if (swhash->hlist_refcount > 0) {
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02006788 struct swevent_hlist *hlist;
6789
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006790 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
6791 WARN_ON(!hlist);
6792 rcu_assign_pointer(swhash->swevent_hlist, hlist);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02006793 }
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006794 mutex_unlock(&swhash->hlist_mutex);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006795}
6796
Peter Zijlstrac2774432010-12-08 15:29:02 +01006797#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02006798static void perf_pmu_rotate_stop(struct pmu *pmu)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006799{
Peter Zijlstrae9d2b062010-09-17 11:28:50 +02006800 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
6801
6802 WARN_ON(!irqs_disabled());
6803
6804 list_del_init(&cpuctx->rotation_list);
6805}
6806
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006807static void __perf_event_exit_context(void *__info)
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006808{
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006809 struct perf_event_context *ctx = __info;
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006810 struct perf_event *event, *tmp;
6811
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006812 perf_pmu_rotate_stop(ctx->pmu);
Peter Zijlstrab5ab4cd2010-09-06 16:32:21 +02006813
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006814 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006815 __perf_remove_from_context(event);
Frederic Weisbecker889ff012010-01-09 20:04:47 +01006816 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
Peter Zijlstrafe4b04f2011-02-02 13:19:09 +01006817 __perf_remove_from_context(event);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006818}
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006819
6820static void perf_event_exit_cpu_context(int cpu)
6821{
6822 struct perf_event_context *ctx;
6823 struct pmu *pmu;
6824 int idx;
6825
6826 idx = srcu_read_lock(&pmus_srcu);
6827 list_for_each_entry_rcu(pmu, &pmus, entry) {
Peter Zijlstra917bdd12010-09-17 11:28:49 +02006828 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006829
6830 mutex_lock(&ctx->mutex);
6831 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
6832 mutex_unlock(&ctx->mutex);
6833 }
6834 srcu_read_unlock(&pmus_srcu, idx);
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006835}
6836
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006837static void perf_event_exit_cpu(int cpu)
6838{
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006839 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006840
Peter Zijlstrab28ab832010-09-06 14:48:15 +02006841 mutex_lock(&swhash->hlist_mutex);
6842 swevent_hlist_release(swhash);
6843 mutex_unlock(&swhash->hlist_mutex);
Frederic Weisbecker76e1d902010-04-05 15:35:57 +02006844
Peter Zijlstra108b02c2010-09-06 14:32:03 +02006845 perf_event_exit_cpu_context(cpu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006846}
6847#else
6848static inline void perf_event_exit_cpu(int cpu) { }
6849#endif
6850
Peter Zijlstrac2774432010-12-08 15:29:02 +01006851static int
6852perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
6853{
6854 int cpu;
6855
6856 for_each_online_cpu(cpu)
6857 perf_event_exit_cpu(cpu);
6858
6859 return NOTIFY_OK;
6860}
6861
6862/*
6863 * Run the perf reboot notifier at the very last possible moment so that
6864 * the generic watchdog code runs as long as possible.
6865 */
6866static struct notifier_block perf_reboot_notifier = {
6867 .notifier_call = perf_reboot,
6868 .priority = INT_MIN,
6869};
6870
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006871static int __cpuinit
6872perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
6873{
6874 unsigned int cpu = (long)hcpu;
6875
Linus Torvalds4536e4d2011-11-03 07:44:04 -07006876 switch (action & ~CPU_TASKS_FROZEN) {
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006877
6878 case CPU_UP_PREPARE:
Peter Zijlstra5e116372010-06-11 13:35:08 +02006879 case CPU_DOWN_FAILED:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006880 perf_event_init_cpu(cpu);
6881 break;
6882
Peter Zijlstra5e116372010-06-11 13:35:08 +02006883 case CPU_UP_CANCELED:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006884 case CPU_DOWN_PREPARE:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006885 perf_event_exit_cpu(cpu);
6886 break;
6887
6888 default:
6889 break;
6890 }
6891
6892 return NOTIFY_OK;
6893}
6894
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006895void __init perf_event_init(void)
6896{
Jason Wessel3c502e72010-11-04 17:33:01 -05006897 int ret;
6898
Peter Zijlstra2e80a822010-11-17 23:17:36 +01006899 idr_init(&pmu_idr);
6900
Paul Mackerras220b1402010-03-10 20:45:52 +11006901 perf_event_init_all_cpus();
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02006902 init_srcu_struct(&pmus_srcu);
Peter Zijlstra2e80a822010-11-17 23:17:36 +01006903 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
6904 perf_pmu_register(&perf_cpu_clock, NULL, -1);
6905 perf_pmu_register(&perf_task_clock, NULL, -1);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +02006906 perf_tp_register();
6907 perf_cpu_notifier(perf_cpu_notify);
Peter Zijlstrac2774432010-12-08 15:29:02 +01006908 register_reboot_notifier(&perf_reboot_notifier);
Jason Wessel3c502e72010-11-04 17:33:01 -05006909
6910 ret = init_hw_breakpoint();
6911 WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
Gleb Natapovb2029522011-11-27 17:59:09 +02006912
6913 /* do not patch jump label more than once per second */
6914 jump_label_rate_limit(&perf_sched_events, HZ);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02006915}
Peter Zijlstraabe43402010-11-17 23:17:37 +01006916
6917static int __init perf_event_sysfs_init(void)
6918{
6919 struct pmu *pmu;
6920 int ret;
6921
6922 mutex_lock(&pmus_lock);
6923
6924 ret = bus_register(&pmu_bus);
6925 if (ret)
6926 goto unlock;
6927
6928 list_for_each_entry(pmu, &pmus, entry) {
6929 if (!pmu->name || pmu->type < 0)
6930 continue;
6931
6932 ret = pmu_dev_alloc(pmu);
6933 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
6934 }
6935 pmu_bus_running = 1;
6936 ret = 0;
6937
6938unlock:
6939 mutex_unlock(&pmus_lock);
6940
6941 return ret;
6942}
6943device_initcall(perf_event_sysfs_init);
Stephane Eraniane5d13672011-02-14 11:20:01 +02006944
6945#ifdef CONFIG_CGROUP_PERF
6946static struct cgroup_subsys_state *perf_cgroup_create(
6947 struct cgroup_subsys *ss, struct cgroup *cont)
6948{
6949 struct perf_cgroup *jc;
Stephane Eraniane5d13672011-02-14 11:20:01 +02006950
Li Zefan1b15d052011-03-03 14:26:06 +08006951 jc = kzalloc(sizeof(*jc), GFP_KERNEL);
Stephane Eraniane5d13672011-02-14 11:20:01 +02006952 if (!jc)
6953 return ERR_PTR(-ENOMEM);
6954
Stephane Eraniane5d13672011-02-14 11:20:01 +02006955 jc->info = alloc_percpu(struct perf_cgroup_info);
6956 if (!jc->info) {
6957 kfree(jc);
6958 return ERR_PTR(-ENOMEM);
6959 }
6960
Stephane Eraniane5d13672011-02-14 11:20:01 +02006961 return &jc->css;
6962}
6963
6964static void perf_cgroup_destroy(struct cgroup_subsys *ss,
6965 struct cgroup *cont)
6966{
6967 struct perf_cgroup *jc;
6968 jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
6969 struct perf_cgroup, css);
6970 free_percpu(jc->info);
6971 kfree(jc);
6972}
6973
6974static int __perf_cgroup_move(void *info)
6975{
6976 struct task_struct *task = info;
6977 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
6978 return 0;
6979}
6980
Tejun Heobb9d97b2011-12-12 18:12:21 -08006981static void perf_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
6982 struct cgroup_taskset *tset)
Stephane Eraniane5d13672011-02-14 11:20:01 +02006983{
Tejun Heobb9d97b2011-12-12 18:12:21 -08006984 struct task_struct *task;
6985
6986 cgroup_taskset_for_each(task, cgrp, tset)
6987 task_function_call(task, __perf_cgroup_move, task);
Stephane Eraniane5d13672011-02-14 11:20:01 +02006988}
6989
Stephane Eraniane5d13672011-02-14 11:20:01 +02006990static void perf_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
6991 struct cgroup *old_cgrp, struct task_struct *task)
6992{
6993 /*
6994 * cgroup_exit() is called in the copy_process() failure path.
6995 * Ignore this case since the task hasn't ran yet, this avoids
6996 * trying to poke a half freed task state from generic code.
6997 */
6998 if (!(task->flags & PF_EXITING))
6999 return;
7000
Tejun Heobb9d97b2011-12-12 18:12:21 -08007001 task_function_call(task, __perf_cgroup_move, task);
Stephane Eraniane5d13672011-02-14 11:20:01 +02007002}
7003
7004struct cgroup_subsys perf_subsys = {
Ingo Molnare7e7ee22011-05-04 08:42:29 +02007005 .name = "perf_event",
7006 .subsys_id = perf_subsys_id,
7007 .create = perf_cgroup_create,
7008 .destroy = perf_cgroup_destroy,
7009 .exit = perf_cgroup_exit,
Tejun Heobb9d97b2011-12-12 18:12:21 -08007010 .attach = perf_cgroup_attach,
Stephane Eraniane5d13672011-02-14 11:20:01 +02007011};
7012#endif /* CONFIG_CGROUP_PERF */