blob: 2dec9bcde8b495bd43d189204e2bc9099d5ec32f [file] [log] [blame]
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001/*
Masami Hiramatsu77b44d12009-11-03 19:12:47 -05002 * Kprobes-based tracing events
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04003 *
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/module.h>
21#include <linux/uaccess.h>
22#include <linux/kprobes.h>
23#include <linux/seq_file.h>
24#include <linux/slab.h>
25#include <linux/smp.h>
26#include <linux/debugfs.h>
27#include <linux/types.h>
28#include <linux/string.h>
29#include <linux/ctype.h>
30#include <linux/ptrace.h>
Frederic Weisbeckerd7a4b412009-09-23 23:08:43 +020031#include <linux/perf_event.h>
Masami Hiramatsu93ccae72010-04-12 13:17:08 -040032#include <linux/stringify.h>
Masami Hiramatsue09c8612010-07-05 15:54:45 -030033#include <linux/limits.h>
Masami Hiramatsu93ccae72010-04-12 13:17:08 -040034#include <asm/bitsperlong.h>
Masami Hiramatsu413d37d2009-08-13 16:35:11 -040035
36#include "trace.h"
37#include "trace_output.h"
38
Masami Hiramatsua82378d2009-08-13 16:35:18 -040039#define MAX_TRACE_ARGS 128
Masami Hiramatsu413d37d2009-08-13 16:35:11 -040040#define MAX_ARGSTR_LEN 63
Masami Hiramatsu42635652009-08-13 16:35:26 -040041#define MAX_EVENT_NAME_LEN 64
Masami Hiramatsue09c8612010-07-05 15:54:45 -030042#define MAX_STRING_SIZE PATH_MAX
Masami Hiramatsuf52487e2009-09-10 19:53:53 -040043#define KPROBE_EVENT_SYSTEM "kprobes"
Masami Hiramatsu413d37d2009-08-13 16:35:11 -040044
Masami Hiramatsua703d942009-10-07 18:28:07 -040045/* Reserved field names */
Masami Hiramatsue93f4d82009-10-07 18:28:14 -040046#define FIELD_STRING_IP "__probe_ip"
Masami Hiramatsue93f4d82009-10-07 18:28:14 -040047#define FIELD_STRING_RETIP "__probe_ret_ip"
48#define FIELD_STRING_FUNC "__probe_func"
Masami Hiramatsua703d942009-10-07 18:28:07 -040049
50const char *reserved_field_names[] = {
51 "common_type",
52 "common_flags",
53 "common_preempt_count",
54 "common_pid",
55 "common_tgid",
56 "common_lock_depth",
57 FIELD_STRING_IP,
Masami Hiramatsua703d942009-10-07 18:28:07 -040058 FIELD_STRING_RETIP,
59 FIELD_STRING_FUNC,
60};
61
Masami Hiramatsu93ccae72010-04-12 13:17:08 -040062/* Printing function type */
Masami Hiramatsue09c8612010-07-05 15:54:45 -030063typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *,
64 void *);
Masami Hiramatsu93ccae72010-04-12 13:17:08 -040065#define PRINT_TYPE_FUNC_NAME(type) print_type_##type
66#define PRINT_TYPE_FMT_NAME(type) print_type_format_##type
67
68/* Printing in basic type function template */
69#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \
70static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
Masami Hiramatsue09c8612010-07-05 15:54:45 -030071 const char *name, \
72 void *data, void *ent)\
Masami Hiramatsu93ccae72010-04-12 13:17:08 -040073{ \
74 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\
75} \
76static const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
77
78DEFINE_BASIC_PRINT_TYPE_FUNC(u8, "%x", unsigned int)
79DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "%x", unsigned int)
80DEFINE_BASIC_PRINT_TYPE_FUNC(u32, "%lx", unsigned long)
81DEFINE_BASIC_PRINT_TYPE_FUNC(u64, "%llx", unsigned long long)
82DEFINE_BASIC_PRINT_TYPE_FUNC(s8, "%d", int)
83DEFINE_BASIC_PRINT_TYPE_FUNC(s16, "%d", int)
84DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%ld", long)
85DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%lld", long long)
86
Masami Hiramatsue09c8612010-07-05 15:54:45 -030087/* data_rloc: data relative location, compatible with u32 */
88#define make_data_rloc(len, roffs) \
89 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff))
90#define get_rloc_len(dl) ((u32)(dl) >> 16)
91#define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
92
93static inline void *get_rloc_data(u32 *dl)
94{
95 return (u8 *)dl + get_rloc_offs(*dl);
96}
97
98/* For data_loc conversion */
99static inline void *get_loc_data(u32 *dl, void *ent)
100{
101 return (u8 *)ent + get_rloc_offs(*dl);
102}
103
104/*
105 * Convert data_rloc to data_loc:
106 * data_rloc stores the offset from data_rloc itself, but data_loc
107 * stores the offset from event entry.
108 */
109#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
110
111/* For defining macros, define string/string_size types */
112typedef u32 string;
113typedef u32 string_size;
114
115/* Print type function for string type */
116static __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s,
117 const char *name,
118 void *data, void *ent)
119{
120 int len = *(u32 *)data >> 16;
121
122 if (!len)
123 return trace_seq_printf(s, " %s=(fault)", name);
124 else
125 return trace_seq_printf(s, " %s=\"%s\"", name,
126 (const char *)get_loc_data(data, ent));
127}
128static const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
129
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400130/* Data fetch function type */
131typedef void (*fetch_func_t)(struct pt_regs *, void *, void *);
132
133struct fetch_param {
134 fetch_func_t fn;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400135 void *data;
136};
137
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400138static __kprobes void call_fetch(struct fetch_param *fprm,
139 struct pt_regs *regs, void *dest)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400140{
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400141 return fprm->fn(regs, fprm->data, dest);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400142}
143
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300144#define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400145/*
146 * Define macro for basic types - we don't need to define s* types, because
147 * we have to care only about bitwidth at recording time.
148 */
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300149#define DEFINE_BASIC_FETCH_FUNCS(method) \
150DEFINE_FETCH_##method(u8) \
151DEFINE_FETCH_##method(u16) \
152DEFINE_FETCH_##method(u32) \
153DEFINE_FETCH_##method(u64)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400154
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300155#define CHECK_FETCH_FUNCS(method, fn) \
156 (((FETCH_FUNC_NAME(method, u8) == fn) || \
157 (FETCH_FUNC_NAME(method, u16) == fn) || \
158 (FETCH_FUNC_NAME(method, u32) == fn) || \
159 (FETCH_FUNC_NAME(method, u64) == fn) || \
160 (FETCH_FUNC_NAME(method, string) == fn) || \
161 (FETCH_FUNC_NAME(method, string_size) == fn)) \
162 && (fn != NULL))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400163
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400164/* Data fetch function templates */
165#define DEFINE_FETCH_reg(type) \
166static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300167 void *offset, void *dest) \
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400168{ \
169 *(type *)dest = (type)regs_get_register(regs, \
170 (unsigned int)((unsigned long)offset)); \
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400171}
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400172DEFINE_BASIC_FETCH_FUNCS(reg)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300173/* No string on the register */
174#define fetch_reg_string NULL
175#define fetch_reg_string_size NULL
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400176
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400177#define DEFINE_FETCH_stack(type) \
178static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
179 void *offset, void *dest) \
180{ \
181 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
182 (unsigned int)((unsigned long)offset)); \
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400183}
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400184DEFINE_BASIC_FETCH_FUNCS(stack)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300185/* No string on the stack entry */
186#define fetch_stack_string NULL
187#define fetch_stack_string_size NULL
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400188
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400189#define DEFINE_FETCH_retval(type) \
190static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\
191 void *dummy, void *dest) \
192{ \
193 *(type *)dest = (type)regs_return_value(regs); \
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400194}
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400195DEFINE_BASIC_FETCH_FUNCS(retval)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300196/* No string on the retval */
197#define fetch_retval_string NULL
198#define fetch_retval_string_size NULL
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400199
200#define DEFINE_FETCH_memory(type) \
201static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
202 void *addr, void *dest) \
203{ \
204 type retval; \
205 if (probe_kernel_address(addr, retval)) \
206 *(type *)dest = 0; \
207 else \
208 *(type *)dest = retval; \
209}
210DEFINE_BASIC_FETCH_FUNCS(memory)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300211/*
212 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
213 * length and relative data location.
214 */
215static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
216 void *addr, void *dest)
217{
218 long ret;
219 int maxlen = get_rloc_len(*(u32 *)dest);
220 u8 *dst = get_rloc_data(dest);
221 u8 *src = addr;
222 mm_segment_t old_fs = get_fs();
223 if (!maxlen)
224 return;
225 /*
226 * Try to get string again, since the string can be changed while
227 * probing.
228 */
229 set_fs(KERNEL_DS);
230 pagefault_disable();
231 do
232 ret = __copy_from_user_inatomic(dst++, src++, 1);
233 while (dst[-1] && ret == 0 && src - (u8 *)addr < maxlen);
234 dst[-1] = '\0';
235 pagefault_enable();
236 set_fs(old_fs);
237
238 if (ret < 0) { /* Failed to fetch string */
239 ((u8 *)get_rloc_data(dest))[0] = '\0';
240 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
241 } else
242 *(u32 *)dest = make_data_rloc(src - (u8 *)addr,
243 get_rloc_offs(*(u32 *)dest));
244}
245/* Return the length of string -- including null terminal byte */
246static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
247 void *addr, void *dest)
248{
249 int ret, len = 0;
250 u8 c;
251 mm_segment_t old_fs = get_fs();
252
253 set_fs(KERNEL_DS);
254 pagefault_disable();
255 do {
256 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
257 len++;
258 } while (c && ret == 0 && len < MAX_STRING_SIZE);
259 pagefault_enable();
260 set_fs(old_fs);
261
262 if (ret < 0) /* Failed to check the length */
263 *(u32 *)dest = 0;
264 else
265 *(u32 *)dest = len;
266}
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400267
268/* Memory fetching by symbol */
269struct symbol_cache {
270 char *symbol;
271 long offset;
272 unsigned long addr;
273};
274
275static unsigned long update_symbol_cache(struct symbol_cache *sc)
276{
277 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
278 if (sc->addr)
279 sc->addr += sc->offset;
280 return sc->addr;
281}
282
283static void free_symbol_cache(struct symbol_cache *sc)
284{
285 kfree(sc->symbol);
286 kfree(sc);
287}
288
289static struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
290{
291 struct symbol_cache *sc;
292
293 if (!sym || strlen(sym) == 0)
294 return NULL;
295 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
296 if (!sc)
297 return NULL;
298
299 sc->symbol = kstrdup(sym, GFP_KERNEL);
300 if (!sc->symbol) {
301 kfree(sc);
302 return NULL;
303 }
304 sc->offset = offset;
305
306 update_symbol_cache(sc);
307 return sc;
308}
309
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400310#define DEFINE_FETCH_symbol(type) \
311static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\
312 void *data, void *dest) \
313{ \
314 struct symbol_cache *sc = data; \
315 if (sc->addr) \
316 fetch_memory_##type(regs, (void *)sc->addr, dest); \
317 else \
318 *(type *)dest = 0; \
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400319}
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400320DEFINE_BASIC_FETCH_FUNCS(symbol)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300321DEFINE_FETCH_symbol(string)
322DEFINE_FETCH_symbol(string_size)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400323
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400324/* Dereference memory access function */
325struct deref_fetch_param {
326 struct fetch_param orig;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400327 long offset;
328};
329
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400330#define DEFINE_FETCH_deref(type) \
331static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\
332 void *data, void *dest) \
333{ \
334 struct deref_fetch_param *dprm = data; \
335 unsigned long addr; \
336 call_fetch(&dprm->orig, regs, &addr); \
337 if (addr) { \
338 addr += dprm->offset; \
339 fetch_memory_##type(regs, (void *)addr, dest); \
340 } else \
341 *(type *)dest = 0; \
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400342}
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400343DEFINE_BASIC_FETCH_FUNCS(deref)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300344DEFINE_FETCH_deref(string)
345DEFINE_FETCH_deref(string_size)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400346
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400347static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400348{
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300349 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400350 free_deref_fetch_param(data->orig.data);
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300351 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400352 free_symbol_cache(data->orig.data);
353 kfree(data);
354}
355
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400356/* Default (unsigned long) fetch type */
357#define __DEFAULT_FETCH_TYPE(t) u##t
358#define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
359#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
360#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
361
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300362/* Fetch types */
363enum {
364 FETCH_MTD_reg = 0,
365 FETCH_MTD_stack,
366 FETCH_MTD_retval,
367 FETCH_MTD_memory,
368 FETCH_MTD_symbol,
369 FETCH_MTD_deref,
370 FETCH_MTD_END,
371};
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400372
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300373#define ASSIGN_FETCH_FUNC(method, type) \
374 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type)
375
376#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
377 {.name = _name, \
378 .size = _size, \
379 .is_signed = sign, \
380 .print = PRINT_TYPE_FUNC_NAME(ptype), \
381 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
382 .fmttype = _fmttype, \
383 .fetch = { \
384ASSIGN_FETCH_FUNC(reg, ftype), \
385ASSIGN_FETCH_FUNC(stack, ftype), \
386ASSIGN_FETCH_FUNC(retval, ftype), \
387ASSIGN_FETCH_FUNC(memory, ftype), \
388ASSIGN_FETCH_FUNC(symbol, ftype), \
389ASSIGN_FETCH_FUNC(deref, ftype), \
390 } \
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400391 }
392
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300393#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
394 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype)
395
396#define FETCH_TYPE_STRING 0
397#define FETCH_TYPE_STRSIZE 1
398
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400399/* Fetch type information table */
400static const struct fetch_type {
401 const char *name; /* Name of type */
402 size_t size; /* Byte size of type */
403 int is_signed; /* Signed flag */
404 print_type_func_t print; /* Print functions */
405 const char *fmt; /* Fromat string */
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300406 const char *fmttype; /* Name in format file */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400407 /* Fetch functions */
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300408 fetch_func_t fetch[FETCH_MTD_END];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400409} fetch_type_table[] = {
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300410 /* Special types */
411 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
412 sizeof(u32), 1, "__data_loc char[]"),
413 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
414 string_size, sizeof(u32), 0, "u32"),
415 /* Basic types */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400416 ASSIGN_FETCH_TYPE(u8, u8, 0),
417 ASSIGN_FETCH_TYPE(u16, u16, 0),
418 ASSIGN_FETCH_TYPE(u32, u32, 0),
419 ASSIGN_FETCH_TYPE(u64, u64, 0),
420 ASSIGN_FETCH_TYPE(s8, u8, 1),
421 ASSIGN_FETCH_TYPE(s16, u16, 1),
422 ASSIGN_FETCH_TYPE(s32, u32, 1),
423 ASSIGN_FETCH_TYPE(s64, u64, 1),
424};
425
426static const struct fetch_type *find_fetch_type(const char *type)
427{
428 int i;
429
430 if (!type)
431 type = DEFAULT_FETCH_TYPE_STR;
432
433 for (i = 0; i < ARRAY_SIZE(fetch_type_table); i++)
434 if (strcmp(type, fetch_type_table[i].name) == 0)
435 return &fetch_type_table[i];
436 return NULL;
437}
438
439/* Special function : only accept unsigned long */
440static __kprobes void fetch_stack_address(struct pt_regs *regs,
441 void *dummy, void *dest)
442{
443 *(unsigned long *)dest = kernel_stack_pointer(regs);
444}
445
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300446static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
447 fetch_func_t orig_fn)
448{
449 int i;
450
451 if (type != &fetch_type_table[FETCH_TYPE_STRING])
452 return NULL; /* Only string type needs size function */
453 for (i = 0; i < FETCH_MTD_END; i++)
454 if (type->fetch[i] == orig_fn)
455 return fetch_type_table[FETCH_TYPE_STRSIZE].fetch[i];
456
457 WARN_ON(1); /* This should not happen */
458 return NULL;
459}
460
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400461/**
Masami Hiramatsu77b44d12009-11-03 19:12:47 -0500462 * Kprobe event core functions
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400463 */
464
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400465struct probe_arg {
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400466 struct fetch_param fetch;
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300467 struct fetch_param fetch_size;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400468 unsigned int offset; /* Offset from argument entry */
469 const char *name; /* Name of this argument */
470 const char *comm; /* Command of this argument */
471 const struct fetch_type *type; /* Type of this argument */
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400472};
473
Masami Hiramatsu50d78052009-09-14 16:49:20 -0400474/* Flags for trace_probe */
475#define TP_FLAG_TRACE 1
476#define TP_FLAG_PROFILE 2
477
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400478struct trace_probe {
479 struct list_head list;
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200480 struct kretprobe rp; /* Use rp.kp for kprobe use */
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -0400481 unsigned long nhit;
Masami Hiramatsu50d78052009-09-14 16:49:20 -0400482 unsigned int flags; /* For TP_FLAG_* */
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400483 const char *symbol; /* symbol name */
Steven Rostedt22392912010-04-21 12:27:06 -0400484 struct ftrace_event_class class;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400485 struct ftrace_event_call call;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400486 ssize_t size; /* trace entry size */
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400487 unsigned int nr_args;
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400488 struct probe_arg args[];
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400489};
490
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400491#define SIZEOF_TRACE_PROBE(n) \
492 (offsetof(struct trace_probe, args) + \
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400493 (sizeof(struct probe_arg) * (n)))
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400494
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400495
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400496static __kprobes int probe_is_return(struct trace_probe *tp)
497{
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200498 return tp->rp.handler != NULL;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400499}
500
501static __kprobes const char *probe_symbol(struct trace_probe *tp)
502{
503 return tp->symbol ? tp->symbol : "unknown";
504}
505
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400506static int register_probe_event(struct trace_probe *tp);
507static void unregister_probe_event(struct trace_probe *tp);
508
509static DEFINE_MUTEX(probe_lock);
510static LIST_HEAD(probe_list);
511
Masami Hiramatsu50d78052009-09-14 16:49:20 -0400512static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
513static int kretprobe_dispatcher(struct kretprobe_instance *ri,
514 struct pt_regs *regs);
515
Masami Hiramatsuda346342010-08-27 20:39:12 +0900516/* Check the name is good for event/group/fields */
517static int is_good_name(const char *name)
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500518{
519 if (!isalpha(*name) && *name != '_')
520 return 0;
521 while (*++name != '\0') {
522 if (!isalpha(*name) && !isdigit(*name) && *name != '_')
523 return 0;
524 }
525 return 1;
526}
527
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200528/*
529 * Allocate new trace_probe and initialize it (including kprobes).
530 */
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400531static struct trace_probe *alloc_trace_probe(const char *group,
532 const char *event,
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200533 void *addr,
534 const char *symbol,
535 unsigned long offs,
536 int nargs, int is_return)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400537{
538 struct trace_probe *tp;
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500539 int ret = -ENOMEM;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400540
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400541 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400542 if (!tp)
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500543 return ERR_PTR(ret);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400544
545 if (symbol) {
546 tp->symbol = kstrdup(symbol, GFP_KERNEL);
547 if (!tp->symbol)
548 goto error;
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200549 tp->rp.kp.symbol_name = tp->symbol;
550 tp->rp.kp.offset = offs;
551 } else
552 tp->rp.kp.addr = addr;
553
554 if (is_return)
Masami Hiramatsu50d78052009-09-14 16:49:20 -0400555 tp->rp.handler = kretprobe_dispatcher;
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200556 else
Masami Hiramatsu50d78052009-09-14 16:49:20 -0400557 tp->rp.kp.pre_handler = kprobe_dispatcher;
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200558
Masami Hiramatsuda346342010-08-27 20:39:12 +0900559 if (!event || !is_good_name(event)) {
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500560 ret = -EINVAL;
Masami Hiramatsu42635652009-08-13 16:35:26 -0400561 goto error;
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500562 }
563
Steven Rostedt22392912010-04-21 12:27:06 -0400564 tp->call.class = &tp->class;
Masami Hiramatsu42635652009-08-13 16:35:26 -0400565 tp->call.name = kstrdup(event, GFP_KERNEL);
566 if (!tp->call.name)
567 goto error;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400568
Masami Hiramatsuda346342010-08-27 20:39:12 +0900569 if (!group || !is_good_name(group)) {
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500570 ret = -EINVAL;
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400571 goto error;
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500572 }
573
Steven Rostedt22392912010-04-21 12:27:06 -0400574 tp->class.system = kstrdup(group, GFP_KERNEL);
575 if (!tp->class.system)
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400576 goto error;
577
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400578 INIT_LIST_HEAD(&tp->list);
579 return tp;
580error:
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400581 kfree(tp->call.name);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400582 kfree(tp->symbol);
583 kfree(tp);
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500584 return ERR_PTR(ret);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400585}
586
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400587static void free_probe_arg(struct probe_arg *arg)
588{
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300589 if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400590 free_deref_fetch_param(arg->fetch.data);
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300591 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400592 free_symbol_cache(arg->fetch.data);
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400593 kfree(arg->name);
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400594 kfree(arg->comm);
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400595}
596
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400597static void free_trace_probe(struct trace_probe *tp)
598{
599 int i;
600
601 for (i = 0; i < tp->nr_args; i++)
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400602 free_probe_arg(&tp->args[i]);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400603
Steven Rostedt8f082012010-04-20 10:47:33 -0400604 kfree(tp->call.class->system);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400605 kfree(tp->call.name);
606 kfree(tp->symbol);
607 kfree(tp);
608}
609
Masami Hiramatsudd004c42009-10-27 16:42:44 -0400610static struct trace_probe *find_probe_event(const char *event,
611 const char *group)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400612{
613 struct trace_probe *tp;
614
615 list_for_each_entry(tp, &probe_list, list)
Masami Hiramatsudd004c42009-10-27 16:42:44 -0400616 if (strcmp(tp->call.name, event) == 0 &&
Steven Rostedt8f082012010-04-20 10:47:33 -0400617 strcmp(tp->call.class->system, group) == 0)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400618 return tp;
619 return NULL;
620}
621
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400622/* Unregister a trace_probe and probe_event: call with locking probe_lock */
623static void unregister_trace_probe(struct trace_probe *tp)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400624{
625 if (probe_is_return(tp))
626 unregister_kretprobe(&tp->rp);
627 else
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200628 unregister_kprobe(&tp->rp.kp);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400629 list_del(&tp->list);
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400630 unregister_probe_event(tp);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400631}
632
633/* Register a trace_probe and probe_event */
634static int register_trace_probe(struct trace_probe *tp)
635{
636 struct trace_probe *old_tp;
637 int ret;
638
639 mutex_lock(&probe_lock);
640
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400641 /* register as an event */
Steven Rostedt8f082012010-04-20 10:47:33 -0400642 old_tp = find_probe_event(tp->call.name, tp->call.class->system);
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400643 if (old_tp) {
644 /* delete old event */
645 unregister_trace_probe(old_tp);
646 free_trace_probe(old_tp);
647 }
648 ret = register_probe_event(tp);
649 if (ret) {
Paul Bolle426d3102010-08-07 12:30:03 +0200650 pr_warning("Failed to register probe event(%d)\n", ret);
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400651 goto end;
652 }
653
Masami Hiramatsu5a0d9052009-09-14 16:49:37 -0400654 tp->rp.kp.flags |= KPROBE_FLAG_DISABLED;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400655 if (probe_is_return(tp))
656 ret = register_kretprobe(&tp->rp);
657 else
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200658 ret = register_kprobe(&tp->rp.kp);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400659
660 if (ret) {
661 pr_warning("Could not insert probe(%d)\n", ret);
662 if (ret == -EILSEQ) {
663 pr_warning("Probing address(0x%p) is not an "
664 "instruction boundary.\n",
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200665 tp->rp.kp.addr);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400666 ret = -EINVAL;
667 }
Masami Hiramatsu2d5e0672009-09-14 16:48:56 -0400668 unregister_probe_event(tp);
669 } else
670 list_add_tail(&tp->list, &probe_list);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400671end:
672 mutex_unlock(&probe_lock);
673 return ret;
674}
675
676/* Split symbol and offset. */
Masami Hiramatsu2fba0c82009-09-10 19:53:14 -0400677static int split_symbol_offset(char *symbol, unsigned long *offset)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400678{
679 char *tmp;
680 int ret;
681
682 if (!offset)
683 return -EINVAL;
684
685 tmp = strchr(symbol, '+');
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400686 if (tmp) {
687 /* skip sign because strict_strtol doesn't accept '+' */
Masami Hiramatsu2fba0c82009-09-10 19:53:14 -0400688 ret = strict_strtoul(tmp + 1, 0, offset);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400689 if (ret)
690 return ret;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400691 *tmp = '\0';
692 } else
693 *offset = 0;
694 return 0;
695}
696
697#define PARAM_MAX_ARGS 16
698#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
699
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400700static int parse_probe_vars(char *arg, const struct fetch_type *t,
701 struct fetch_param *f, int is_return)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400702{
703 int ret = 0;
704 unsigned long param;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400705
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400706 if (strcmp(arg, "retval") == 0) {
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400707 if (is_return)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300708 f->fn = t->fetch[FETCH_MTD_retval];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400709 else
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400710 ret = -EINVAL;
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400711 } else if (strncmp(arg, "stack", 5) == 0) {
712 if (arg[5] == '\0') {
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400713 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR) == 0)
714 f->fn = fetch_stack_address;
715 else
716 ret = -EINVAL;
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400717 } else if (isdigit(arg[5])) {
718 ret = strict_strtoul(arg + 5, 10, &param);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400719 if (ret || param > PARAM_MAX_STACK)
720 ret = -EINVAL;
721 else {
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300722 f->fn = t->fetch[FETCH_MTD_stack];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400723 f->data = (void *)param;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400724 }
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400725 } else
726 ret = -EINVAL;
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400727 } else
Masami Hiramatsu405b2652009-10-07 18:27:40 -0400728 ret = -EINVAL;
Masami Hiramatsu405b2652009-10-07 18:27:40 -0400729 return ret;
730}
731
Masami Hiramatsuba8665d2009-11-30 19:19:20 -0500732/* Recursive argument parser */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400733static int __parse_probe_arg(char *arg, const struct fetch_type *t,
734 struct fetch_param *f, int is_return)
Masami Hiramatsu405b2652009-10-07 18:27:40 -0400735{
736 int ret = 0;
737 unsigned long param;
738 long offset;
739 char *tmp;
740
741 switch (arg[0]) {
742 case '$':
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400743 ret = parse_probe_vars(arg + 1, t, f, is_return);
Masami Hiramatsu405b2652009-10-07 18:27:40 -0400744 break;
745 case '%': /* named register */
746 ret = regs_query_register_offset(arg + 1);
747 if (ret >= 0) {
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300748 f->fn = t->fetch[FETCH_MTD_reg];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400749 f->data = (void *)(unsigned long)ret;
Masami Hiramatsu405b2652009-10-07 18:27:40 -0400750 ret = 0;
751 }
752 break;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400753 case '@': /* memory or symbol */
754 if (isdigit(arg[1])) {
755 ret = strict_strtoul(arg + 1, 0, &param);
756 if (ret)
757 break;
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300758 f->fn = t->fetch[FETCH_MTD_memory];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400759 f->data = (void *)param;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400760 } else {
761 ret = split_symbol_offset(arg + 1, &offset);
762 if (ret)
763 break;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400764 f->data = alloc_symbol_cache(arg + 1, offset);
765 if (f->data)
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300766 f->fn = t->fetch[FETCH_MTD_symbol];
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400767 }
768 break;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400769 case '+': /* deref memory */
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400770 case '-':
771 tmp = strchr(arg, '(');
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400772 if (!tmp)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400773 break;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400774 *tmp = '\0';
775 ret = strict_strtol(arg + 1, 0, &offset);
776 if (ret)
777 break;
778 if (arg[0] == '-')
779 offset = -offset;
780 arg = tmp + 1;
781 tmp = strrchr(arg, ')');
782 if (tmp) {
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400783 struct deref_fetch_param *dprm;
784 const struct fetch_type *t2 = find_fetch_type(NULL);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400785 *tmp = '\0';
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400786 dprm = kzalloc(sizeof(struct deref_fetch_param),
787 GFP_KERNEL);
788 if (!dprm)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400789 return -ENOMEM;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400790 dprm->offset = offset;
791 ret = __parse_probe_arg(arg, t2, &dprm->orig,
792 is_return);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400793 if (ret)
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400794 kfree(dprm);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400795 else {
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300796 f->fn = t->fetch[FETCH_MTD_deref];
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400797 f->data = (void *)dprm;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400798 }
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400799 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400800 break;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400801 }
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300802 if (!ret && !f->fn) { /* Parsed, but do not find fetch method */
803 pr_info("%s type has no corresponding fetch method.\n",
804 t->name);
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400805 ret = -EINVAL;
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300806 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400807 return ret;
808}
809
Masami Hiramatsuba8665d2009-11-30 19:19:20 -0500810/* String length checking wrapper */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400811static int parse_probe_arg(char *arg, struct trace_probe *tp,
812 struct probe_arg *parg, int is_return)
Masami Hiramatsuba8665d2009-11-30 19:19:20 -0500813{
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400814 const char *t;
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300815 int ret;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400816
Masami Hiramatsuba8665d2009-11-30 19:19:20 -0500817 if (strlen(arg) > MAX_ARGSTR_LEN) {
818 pr_info("Argument is too long.: %s\n", arg);
819 return -ENOSPC;
820 }
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400821 parg->comm = kstrdup(arg, GFP_KERNEL);
822 if (!parg->comm) {
823 pr_info("Failed to allocate memory for command '%s'.\n", arg);
824 return -ENOMEM;
825 }
826 t = strchr(parg->comm, ':');
827 if (t) {
828 arg[t - parg->comm] = '\0';
829 t++;
830 }
831 parg->type = find_fetch_type(t);
832 if (!parg->type) {
833 pr_info("Unsupported type: %s\n", t);
834 return -EINVAL;
835 }
836 parg->offset = tp->size;
837 tp->size += parg->type->size;
Masami Hiramatsue09c8612010-07-05 15:54:45 -0300838 ret = __parse_probe_arg(arg, parg->type, &parg->fetch, is_return);
839 if (ret >= 0) {
840 parg->fetch_size.fn = get_fetch_size_function(parg->type,
841 parg->fetch.fn);
842 parg->fetch_size.data = parg->fetch.data;
843 }
844 return ret;
Masami Hiramatsuba8665d2009-11-30 19:19:20 -0500845}
846
Masami Hiramatsua703d942009-10-07 18:28:07 -0400847/* Return 1 if name is reserved or already used by another argument */
848static int conflict_field_name(const char *name,
849 struct probe_arg *args, int narg)
850{
851 int i;
852 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
853 if (strcmp(reserved_field_names[i], name) == 0)
854 return 1;
855 for (i = 0; i < narg; i++)
856 if (strcmp(args[i].name, name) == 0)
857 return 1;
858 return 0;
859}
860
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400861static int create_trace_probe(int argc, char **argv)
862{
863 /*
864 * Argument syntax:
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400865 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
866 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400867 * Fetch args:
Masami Hiramatsu2e06ff62009-10-07 18:27:59 -0400868 * $retval : fetch return value
869 * $stack : fetch stack address
870 * $stackN : fetch Nth of stack (N:0-)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400871 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
872 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
873 * %REG : fetch register REG
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400874 * Dereferencing memory fetch:
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400875 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400876 * Alias name of args:
877 * NAME=FETCHARG : set NAME as alias of FETCHARG.
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400878 * Type of args:
879 * FETCHARG:TYPE : use TYPE instead of unsigned long.
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400880 */
881 struct trace_probe *tp;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400882 int i, ret = 0;
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500883 int is_return = 0, is_delete = 0;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -0400884 char *symbol = NULL, *event = NULL, *group = NULL;
Masami Hiramatsuda346342010-08-27 20:39:12 +0900885 char *arg;
Masami Hiramatsu2fba0c82009-09-10 19:53:14 -0400886 unsigned long offset = 0;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400887 void *addr = NULL;
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200888 char buf[MAX_EVENT_NAME_LEN];
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400889
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500890 /* argc must be >= 1 */
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400891 if (argv[0][0] == 'p')
892 is_return = 0;
893 else if (argv[0][0] == 'r')
894 is_return = 1;
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500895 else if (argv[0][0] == '-')
896 is_delete = 1;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400897 else {
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500898 pr_info("Probe definition must be started with 'p', 'r' or"
899 " '-'.\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400900 return -EINVAL;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400901 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400902
903 if (argv[0][1] == ':') {
904 event = &argv[0][2];
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400905 if (strchr(event, '/')) {
906 group = event;
907 event = strchr(group, '/') + 1;
908 event[-1] = '\0';
909 if (strlen(group) == 0) {
Wenji Huanga5efd922010-02-24 15:40:23 +0800910 pr_info("Group name is not specified\n");
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400911 return -EINVAL;
912 }
913 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400914 if (strlen(event) == 0) {
Wenji Huanga5efd922010-02-24 15:40:23 +0800915 pr_info("Event name is not specified\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400916 return -EINVAL;
917 }
918 }
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500919 if (!group)
920 group = KPROBE_EVENT_SYSTEM;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400921
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500922 if (is_delete) {
923 if (!event) {
924 pr_info("Delete command needs an event name.\n");
925 return -EINVAL;
926 }
Srikar Dronamraju9da79ab2010-06-30 14:15:48 +0530927 mutex_lock(&probe_lock);
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500928 tp = find_probe_event(event, group);
929 if (!tp) {
Srikar Dronamraju9da79ab2010-06-30 14:15:48 +0530930 mutex_unlock(&probe_lock);
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500931 pr_info("Event %s/%s doesn't exist.\n", group, event);
932 return -ENOENT;
933 }
934 /* delete an event */
935 unregister_trace_probe(tp);
936 free_trace_probe(tp);
Srikar Dronamraju9da79ab2010-06-30 14:15:48 +0530937 mutex_unlock(&probe_lock);
Masami Hiramatsua7c312b2009-12-08 17:03:16 -0500938 return 0;
939 }
940
941 if (argc < 2) {
942 pr_info("Probe point is not specified.\n");
943 return -EINVAL;
944 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400945 if (isdigit(argv[1][0])) {
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400946 if (is_return) {
947 pr_info("Return probe point must be a symbol.\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400948 return -EINVAL;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400949 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400950 /* an address specified */
Heiko Carstensa9bb18f2010-02-10 17:23:47 +0100951 ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr);
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400952 if (ret) {
953 pr_info("Failed to parse address.\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400954 return ret;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400955 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400956 } else {
957 /* a symbol specified */
958 symbol = argv[1];
959 /* TODO: support .init module functions */
960 ret = split_symbol_offset(symbol, &offset);
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400961 if (ret) {
962 pr_info("Failed to parse symbol.\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400963 return ret;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400964 }
965 if (offset && is_return) {
966 pr_info("Return probe must be used without offset.\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400967 return -EINVAL;
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400968 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400969 }
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400970 argc -= 2; argv += 2;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400971
972 /* setup a probe */
Masami Hiramatsu42635652009-08-13 16:35:26 -0400973 if (!event) {
974 /* Make a new event name */
Masami Hiramatsu42635652009-08-13 16:35:26 -0400975 if (symbol)
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500976 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
Masami Hiramatsu42635652009-08-13 16:35:26 -0400977 is_return ? 'r' : 'p', symbol, offset);
978 else
Masami Hiramatsu6f3cf442009-12-16 17:24:08 -0500979 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
Masami Hiramatsu42635652009-08-13 16:35:26 -0400980 is_return ? 'r' : 'p', addr);
Masami Hiramatsu4a846b42009-09-11 05:31:21 +0200981 event = buf;
982 }
Masami Hiramatsuf52487e2009-09-10 19:53:53 -0400983 tp = alloc_trace_probe(group, event, addr, symbol, offset, argc,
984 is_return);
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400985 if (IS_ERR(tp)) {
986 pr_info("Failed to allocate trace_probe.(%d)\n",
987 (int)PTR_ERR(tp));
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400988 return PTR_ERR(tp);
Masami Hiramatsue63cc232009-10-16 20:07:28 -0400989 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400990
Masami Hiramatsu413d37d2009-08-13 16:35:11 -0400991 /* parse arguments */
Masami Hiramatsua82378d2009-08-13 16:35:18 -0400992 ret = 0;
993 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
Masami Hiramatsu61a52732010-08-27 20:38:46 +0900994 /* Increment count for freeing args in error case */
995 tp->nr_args++;
996
Masami Hiramatsueca0d912009-09-10 19:53:38 -0400997 /* Parse argument name */
998 arg = strchr(argv[i], '=');
Masami Hiramatsuaba91592010-08-27 20:39:06 +0900999 if (arg) {
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001000 *arg++ = '\0';
Masami Hiramatsuaba91592010-08-27 20:39:06 +09001001 tp->args[i].name = kstrdup(argv[i], GFP_KERNEL);
1002 } else {
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001003 arg = argv[i];
Masami Hiramatsuaba91592010-08-27 20:39:06 +09001004 /* If argument name is omitted, set "argN" */
1005 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
1006 tp->args[i].name = kstrdup(buf, GFP_KERNEL);
1007 }
Masami Hiramatsua703d942009-10-07 18:28:07 -04001008
Masami Hiramatsuba8665d2009-11-30 19:19:20 -05001009 if (!tp->args[i].name) {
Masami Hiramatsuaba91592010-08-27 20:39:06 +09001010 pr_info("Failed to allocate argument[%d] name.\n", i);
Masami Hiramatsuba8665d2009-11-30 19:19:20 -05001011 ret = -ENOMEM;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001012 goto error;
1013 }
Masami Hiramatsuda346342010-08-27 20:39:12 +09001014
1015 if (!is_good_name(tp->args[i].name)) {
1016 pr_info("Invalid argument[%d] name: %s\n",
1017 i, tp->args[i].name);
1018 ret = -EINVAL;
1019 goto error;
1020 }
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001021
1022 if (conflict_field_name(tp->args[i].name, tp->args, i)) {
Masami Hiramatsuaba91592010-08-27 20:39:06 +09001023 pr_info("Argument[%d] name '%s' conflicts with "
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001024 "another field.\n", i, argv[i]);
1025 ret = -EINVAL;
1026 goto error;
1027 }
Masami Hiramatsuba8665d2009-11-30 19:19:20 -05001028
1029 /* Parse fetch argument */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001030 ret = parse_probe_arg(arg, tp, &tp->args[i], is_return);
Masami Hiramatsue63cc232009-10-16 20:07:28 -04001031 if (ret) {
Masami Hiramatsuaba91592010-08-27 20:39:06 +09001032 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001033 goto error;
Masami Hiramatsue63cc232009-10-16 20:07:28 -04001034 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001035 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001036
1037 ret = register_trace_probe(tp);
1038 if (ret)
1039 goto error;
1040 return 0;
1041
1042error:
1043 free_trace_probe(tp);
1044 return ret;
1045}
1046
1047static void cleanup_all_probes(void)
1048{
1049 struct trace_probe *tp;
1050
1051 mutex_lock(&probe_lock);
1052 /* TODO: Use batch unregistration */
1053 while (!list_empty(&probe_list)) {
1054 tp = list_entry(probe_list.next, struct trace_probe, list);
1055 unregister_trace_probe(tp);
1056 free_trace_probe(tp);
1057 }
1058 mutex_unlock(&probe_lock);
1059}
1060
1061
1062/* Probes listing interfaces */
1063static void *probes_seq_start(struct seq_file *m, loff_t *pos)
1064{
1065 mutex_lock(&probe_lock);
1066 return seq_list_start(&probe_list, *pos);
1067}
1068
1069static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
1070{
1071 return seq_list_next(v, &probe_list, pos);
1072}
1073
1074static void probes_seq_stop(struct seq_file *m, void *v)
1075{
1076 mutex_unlock(&probe_lock);
1077}
1078
1079static int probes_seq_show(struct seq_file *m, void *v)
1080{
1081 struct trace_probe *tp = v;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001082 int i;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001083
1084 seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p');
Steven Rostedt8f082012010-04-20 10:47:33 -04001085 seq_printf(m, ":%s/%s", tp->call.class->system, tp->call.name);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001086
Lai Jiangshan52a11f32009-11-25 16:33:15 +08001087 if (!tp->symbol)
1088 seq_printf(m, " 0x%p", tp->rp.kp.addr);
1089 else if (tp->rp.kp.offset)
Masami Hiramatsu4a846b42009-09-11 05:31:21 +02001090 seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001091 else
Lai Jiangshan52a11f32009-11-25 16:33:15 +08001092 seq_printf(m, " %s", probe_symbol(tp));
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001093
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001094 for (i = 0; i < tp->nr_args; i++)
1095 seq_printf(m, " %s=%s", tp->args[i].name, tp->args[i].comm);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001096 seq_printf(m, "\n");
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001097
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001098 return 0;
1099}
1100
1101static const struct seq_operations probes_seq_op = {
1102 .start = probes_seq_start,
1103 .next = probes_seq_next,
1104 .stop = probes_seq_stop,
1105 .show = probes_seq_show
1106};
1107
1108static int probes_open(struct inode *inode, struct file *file)
1109{
1110 if ((file->f_mode & FMODE_WRITE) &&
1111 (file->f_flags & O_TRUNC))
1112 cleanup_all_probes();
1113
1114 return seq_open(file, &probes_seq_op);
1115}
1116
1117static int command_trace_probe(const char *buf)
1118{
1119 char **argv;
1120 int argc = 0, ret = 0;
1121
1122 argv = argv_split(GFP_KERNEL, buf, &argc);
1123 if (!argv)
1124 return -ENOMEM;
1125
1126 if (argc)
1127 ret = create_trace_probe(argc, argv);
1128
1129 argv_free(argv);
1130 return ret;
1131}
1132
1133#define WRITE_BUFSIZE 128
1134
1135static ssize_t probes_write(struct file *file, const char __user *buffer,
1136 size_t count, loff_t *ppos)
1137{
1138 char *kbuf, *tmp;
1139 int ret;
1140 size_t done;
1141 size_t size;
1142
1143 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
1144 if (!kbuf)
1145 return -ENOMEM;
1146
1147 ret = done = 0;
1148 while (done < count) {
1149 size = count - done;
1150 if (size >= WRITE_BUFSIZE)
1151 size = WRITE_BUFSIZE - 1;
1152 if (copy_from_user(kbuf, buffer + done, size)) {
1153 ret = -EFAULT;
1154 goto out;
1155 }
1156 kbuf[size] = '\0';
1157 tmp = strchr(kbuf, '\n');
1158 if (tmp) {
1159 *tmp = '\0';
1160 size = tmp - kbuf + 1;
1161 } else if (done + size < count) {
1162 pr_warning("Line length is too long: "
1163 "Should be less than %d.", WRITE_BUFSIZE);
1164 ret = -EINVAL;
1165 goto out;
1166 }
1167 done += size;
1168 /* Remove comments */
1169 tmp = strchr(kbuf, '#');
1170 if (tmp)
1171 *tmp = '\0';
1172
1173 ret = command_trace_probe(kbuf);
1174 if (ret)
1175 goto out;
1176 }
1177 ret = done;
1178out:
1179 kfree(kbuf);
1180 return ret;
1181}
1182
1183static const struct file_operations kprobe_events_ops = {
1184 .owner = THIS_MODULE,
1185 .open = probes_open,
1186 .read = seq_read,
1187 .llseek = seq_lseek,
1188 .release = seq_release,
1189 .write = probes_write,
1190};
1191
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -04001192/* Probes profiling interfaces */
1193static int probes_profile_seq_show(struct seq_file *m, void *v)
1194{
1195 struct trace_probe *tp = v;
1196
1197 seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit,
Masami Hiramatsu4a846b42009-09-11 05:31:21 +02001198 tp->rp.kp.nmissed);
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -04001199
1200 return 0;
1201}
1202
1203static const struct seq_operations profile_seq_op = {
1204 .start = probes_seq_start,
1205 .next = probes_seq_next,
1206 .stop = probes_seq_stop,
1207 .show = probes_profile_seq_show
1208};
1209
1210static int profile_open(struct inode *inode, struct file *file)
1211{
1212 return seq_open(file, &profile_seq_op);
1213}
1214
1215static const struct file_operations kprobe_profile_ops = {
1216 .owner = THIS_MODULE,
1217 .open = profile_open,
1218 .read = seq_read,
1219 .llseek = seq_lseek,
1220 .release = seq_release,
1221};
1222
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001223/* Sum up total data length for dynamic arraies (strings) */
1224static __kprobes int __get_data_size(struct trace_probe *tp,
1225 struct pt_regs *regs)
1226{
1227 int i, ret = 0;
1228 u32 len;
1229
1230 for (i = 0; i < tp->nr_args; i++)
1231 if (unlikely(tp->args[i].fetch_size.fn)) {
1232 call_fetch(&tp->args[i].fetch_size, regs, &len);
1233 ret += len;
1234 }
1235
1236 return ret;
1237}
1238
1239/* Store the value of each argument */
1240static __kprobes void store_trace_args(int ent_size, struct trace_probe *tp,
1241 struct pt_regs *regs,
1242 u8 *data, int maxlen)
1243{
1244 int i;
1245 u32 end = tp->size;
1246 u32 *dl; /* Data (relative) location */
1247
1248 for (i = 0; i < tp->nr_args; i++) {
1249 if (unlikely(tp->args[i].fetch_size.fn)) {
1250 /*
1251 * First, we set the relative location and
1252 * maximum data length to *dl
1253 */
1254 dl = (u32 *)(data + tp->args[i].offset);
1255 *dl = make_data_rloc(maxlen, end - tp->args[i].offset);
1256 /* Then try to fetch string or dynamic array data */
1257 call_fetch(&tp->args[i].fetch, regs, dl);
1258 /* Reduce maximum length */
1259 end += get_rloc_len(*dl);
1260 maxlen -= get_rloc_len(*dl);
1261 /* Trick here, convert data_rloc to data_loc */
1262 *dl = convert_rloc_to_loc(*dl,
1263 ent_size + tp->args[i].offset);
1264 } else
1265 /* Just fetching data normally */
1266 call_fetch(&tp->args[i].fetch, regs,
1267 data + tp->args[i].offset);
1268 }
1269}
1270
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001271/* Kprobe handler */
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001272static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001273{
Masami Hiramatsu4a846b42009-09-11 05:31:21 +02001274 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001275 struct kprobe_trace_entry_head *entry;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001276 struct ring_buffer_event *event;
Frederic Weisbecker8f8ffe22009-09-11 01:09:23 +02001277 struct ring_buffer *buffer;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001278 int size, dsize, pc;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001279 unsigned long irq_flags;
Masami Hiramatsu42635652009-08-13 16:35:26 -04001280 struct ftrace_event_call *call = &tp->call;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001281
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -04001282 tp->nhit++;
1283
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001284 local_save_flags(irq_flags);
1285 pc = preempt_count();
1286
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001287 dsize = __get_data_size(tp, regs);
1288 size = sizeof(*entry) + tp->size + dsize;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001289
Steven Rostedt32c0eda2010-04-23 10:38:03 -04001290 event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
1291 size, irq_flags, pc);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001292 if (!event)
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001293 return;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001294
1295 entry = ring_buffer_event_data(event);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001296 entry->ip = (unsigned long)kp->addr;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001297 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001298
Frederic Weisbecker8f8ffe22009-09-11 01:09:23 +02001299 if (!filter_current_check_discard(buffer, call, entry, event))
1300 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001301}
1302
1303/* Kretprobe handler */
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001304static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri,
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001305 struct pt_regs *regs)
1306{
1307 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001308 struct kretprobe_trace_entry_head *entry;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001309 struct ring_buffer_event *event;
Frederic Weisbecker8f8ffe22009-09-11 01:09:23 +02001310 struct ring_buffer *buffer;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001311 int size, pc, dsize;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001312 unsigned long irq_flags;
Masami Hiramatsu42635652009-08-13 16:35:26 -04001313 struct ftrace_event_call *call = &tp->call;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001314
1315 local_save_flags(irq_flags);
1316 pc = preempt_count();
1317
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001318 dsize = __get_data_size(tp, regs);
1319 size = sizeof(*entry) + tp->size + dsize;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001320
Steven Rostedt32c0eda2010-04-23 10:38:03 -04001321 event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
1322 size, irq_flags, pc);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001323 if (!event)
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001324 return;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001325
1326 entry = ring_buffer_event_data(event);
Masami Hiramatsu4a846b42009-09-11 05:31:21 +02001327 entry->func = (unsigned long)tp->rp.kp.addr;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001328 entry->ret_ip = (unsigned long)ri->ret_addr;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001329 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001330
Frederic Weisbecker8f8ffe22009-09-11 01:09:23 +02001331 if (!filter_current_check_discard(buffer, call, entry, event))
1332 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001333}
1334
1335/* Event entry printers */
1336enum print_line_t
Steven Rostedta9a57762010-04-22 18:46:14 -04001337print_kprobe_event(struct trace_iterator *iter, int flags,
1338 struct trace_event *event)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001339{
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001340 struct kprobe_trace_entry_head *field;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001341 struct trace_seq *s = &iter->seq;
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001342 struct trace_probe *tp;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001343 u8 *data;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001344 int i;
1345
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001346 field = (struct kprobe_trace_entry_head *)iter->ent;
Steven Rostedt80decc72010-04-23 10:00:22 -04001347 tp = container_of(event, struct trace_probe, call.event);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001348
Masami Hiramatsu6e9f23d2009-09-10 19:53:45 -04001349 if (!trace_seq_printf(s, "%s: (", tp->call.name))
1350 goto partial;
1351
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001352 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1353 goto partial;
1354
Masami Hiramatsu6e9f23d2009-09-10 19:53:45 -04001355 if (!trace_seq_puts(s, ")"))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001356 goto partial;
1357
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001358 data = (u8 *)&field[1];
1359 for (i = 0; i < tp->nr_args; i++)
1360 if (!tp->args[i].type->print(s, tp->args[i].name,
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001361 data + tp->args[i].offset, field))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001362 goto partial;
1363
1364 if (!trace_seq_puts(s, "\n"))
1365 goto partial;
1366
1367 return TRACE_TYPE_HANDLED;
1368partial:
1369 return TRACE_TYPE_PARTIAL_LINE;
1370}
1371
1372enum print_line_t
Steven Rostedta9a57762010-04-22 18:46:14 -04001373print_kretprobe_event(struct trace_iterator *iter, int flags,
1374 struct trace_event *event)
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001375{
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001376 struct kretprobe_trace_entry_head *field;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001377 struct trace_seq *s = &iter->seq;
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001378 struct trace_probe *tp;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001379 u8 *data;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001380 int i;
1381
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001382 field = (struct kretprobe_trace_entry_head *)iter->ent;
Steven Rostedt80decc72010-04-23 10:00:22 -04001383 tp = container_of(event, struct trace_probe, call.event);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001384
Masami Hiramatsu6e9f23d2009-09-10 19:53:45 -04001385 if (!trace_seq_printf(s, "%s: (", tp->call.name))
1386 goto partial;
1387
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001388 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1389 goto partial;
1390
1391 if (!trace_seq_puts(s, " <- "))
1392 goto partial;
1393
1394 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1395 goto partial;
1396
Masami Hiramatsu6e9f23d2009-09-10 19:53:45 -04001397 if (!trace_seq_puts(s, ")"))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001398 goto partial;
1399
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001400 data = (u8 *)&field[1];
1401 for (i = 0; i < tp->nr_args; i++)
1402 if (!tp->args[i].type->print(s, tp->args[i].name,
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001403 data + tp->args[i].offset, field))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001404 goto partial;
1405
1406 if (!trace_seq_puts(s, "\n"))
1407 goto partial;
1408
1409 return TRACE_TYPE_HANDLED;
1410partial:
1411 return TRACE_TYPE_PARTIAL_LINE;
1412}
1413
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001414static int probe_event_enable(struct ftrace_event_call *call)
1415{
1416 struct trace_probe *tp = (struct trace_probe *)call->data;
1417
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001418 tp->flags |= TP_FLAG_TRACE;
1419 if (probe_is_return(tp))
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001420 return enable_kretprobe(&tp->rp);
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001421 else
Masami Hiramatsu4a846b42009-09-11 05:31:21 +02001422 return enable_kprobe(&tp->rp.kp);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001423}
1424
1425static void probe_event_disable(struct ftrace_event_call *call)
1426{
1427 struct trace_probe *tp = (struct trace_probe *)call->data;
1428
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001429 tp->flags &= ~TP_FLAG_TRACE;
1430 if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) {
1431 if (probe_is_return(tp))
1432 disable_kretprobe(&tp->rp);
1433 else
1434 disable_kprobe(&tp->rp.kp);
1435 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001436}
1437
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001438#undef DEFINE_FIELD
1439#define DEFINE_FIELD(type, item, name, is_signed) \
1440 do { \
1441 ret = trace_define_field(event_call, #type, name, \
1442 offsetof(typeof(field), item), \
1443 sizeof(field.item), is_signed, \
1444 FILTER_OTHER); \
1445 if (ret) \
1446 return ret; \
1447 } while (0)
1448
1449static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
1450{
1451 int ret, i;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001452 struct kprobe_trace_entry_head field;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001453 struct trace_probe *tp = (struct trace_probe *)event_call->data;
1454
Masami Hiramatsua703d942009-10-07 18:28:07 -04001455 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001456 /* Set argument names as fields */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001457 for (i = 0; i < tp->nr_args; i++) {
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001458 ret = trace_define_field(event_call, tp->args[i].type->fmttype,
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001459 tp->args[i].name,
1460 sizeof(field) + tp->args[i].offset,
1461 tp->args[i].type->size,
1462 tp->args[i].type->is_signed,
1463 FILTER_OTHER);
1464 if (ret)
1465 return ret;
1466 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001467 return 0;
1468}
1469
1470static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
1471{
1472 int ret, i;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001473 struct kretprobe_trace_entry_head field;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001474 struct trace_probe *tp = (struct trace_probe *)event_call->data;
1475
Masami Hiramatsua703d942009-10-07 18:28:07 -04001476 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1477 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
Masami Hiramatsueca0d912009-09-10 19:53:38 -04001478 /* Set argument names as fields */
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001479 for (i = 0; i < tp->nr_args; i++) {
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001480 ret = trace_define_field(event_call, tp->args[i].type->fmttype,
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001481 tp->args[i].name,
1482 sizeof(field) + tp->args[i].offset,
1483 tp->args[i].type->size,
1484 tp->args[i].type->is_signed,
1485 FILTER_OTHER);
1486 if (ret)
1487 return ret;
1488 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001489 return 0;
1490}
1491
Lai Jiangshana342a0282009-12-15 15:39:49 +08001492static int __set_print_fmt(struct trace_probe *tp, char *buf, int len)
1493{
1494 int i;
1495 int pos = 0;
1496
1497 const char *fmt, *arg;
1498
1499 if (!probe_is_return(tp)) {
1500 fmt = "(%lx)";
1501 arg = "REC->" FIELD_STRING_IP;
1502 } else {
1503 fmt = "(%lx <- %lx)";
1504 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1505 }
1506
1507 /* When len=0, we just calculate the needed length */
1508#define LEN_OR_ZERO (len ? len - pos : 0)
1509
1510 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1511
1512 for (i = 0; i < tp->nr_args; i++) {
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001513 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s",
1514 tp->args[i].name, tp->args[i].type->fmt);
Lai Jiangshana342a0282009-12-15 15:39:49 +08001515 }
1516
1517 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
1518
1519 for (i = 0; i < tp->nr_args; i++) {
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001520 if (strcmp(tp->args[i].type->name, "string") == 0)
1521 pos += snprintf(buf + pos, LEN_OR_ZERO,
1522 ", __get_str(%s)",
1523 tp->args[i].name);
1524 else
1525 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
1526 tp->args[i].name);
Lai Jiangshana342a0282009-12-15 15:39:49 +08001527 }
1528
1529#undef LEN_OR_ZERO
1530
1531 /* return the length of print_fmt */
1532 return pos;
1533}
1534
1535static int set_print_fmt(struct trace_probe *tp)
1536{
1537 int len;
1538 char *print_fmt;
1539
1540 /* First: called with 0 length to calculate the needed length */
1541 len = __set_print_fmt(tp, NULL, 0);
1542 print_fmt = kmalloc(len + 1, GFP_KERNEL);
1543 if (!print_fmt)
1544 return -ENOMEM;
1545
1546 /* Second: actually write the @print_fmt */
1547 __set_print_fmt(tp, print_fmt, len + 1);
1548 tp->call.print_fmt = print_fmt;
1549
1550 return 0;
1551}
1552
Li Zefan07b139c2009-12-21 14:27:35 +08001553#ifdef CONFIG_PERF_EVENTS
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001554
1555/* Kprobe profile handler */
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001556static __kprobes void kprobe_perf_func(struct kprobe *kp,
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001557 struct pt_regs *regs)
1558{
1559 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1560 struct ftrace_event_call *call = &tp->call;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001561 struct kprobe_trace_entry_head *entry;
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02001562 struct hlist_head *head;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001563 int size, __size, dsize;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01001564 int rctx;
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001565
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001566 dsize = __get_data_size(tp, regs);
1567 __size = sizeof(*entry) + tp->size + dsize;
Masami Hiramatsu74ebb632009-09-14 16:49:28 -04001568 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1569 size -= sizeof(u32);
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001570 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
Masami Hiramatsua1a138d2009-09-25 11:20:12 -07001571 "profile buffer not large enough"))
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001572 return;
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001573
Steven Rostedtff5f1492010-05-21 11:49:57 -04001574 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
Xiao Guangrong430ad5a2010-01-28 09:32:29 +08001575 if (!entry)
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001576 return;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01001577
Masami Hiramatsua1a138d2009-09-25 11:20:12 -07001578 entry->ip = (unsigned long)kp->addr;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001579 memset(&entry[1], 0, dsize);
1580 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
Frederic Weisbecker444a2a32009-11-06 04:13:05 +01001581
Peter Zijlstra3771f072010-05-21 12:31:09 +02001582 head = this_cpu_ptr(call->perf_events);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02001583 perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001584}
1585
1586/* Kretprobe profile handler */
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001587static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001588 struct pt_regs *regs)
1589{
1590 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1591 struct ftrace_event_call *call = &tp->call;
Masami Hiramatsu93ccae72010-04-12 13:17:08 -04001592 struct kretprobe_trace_entry_head *entry;
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02001593 struct hlist_head *head;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001594 int size, __size, dsize;
Peter Zijlstra4ed7c922009-11-23 11:37:29 +01001595 int rctx;
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001596
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001597 dsize = __get_data_size(tp, regs);
1598 __size = sizeof(*entry) + tp->size + dsize;
Masami Hiramatsu74ebb632009-09-14 16:49:28 -04001599 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1600 size -= sizeof(u32);
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001601 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
Masami Hiramatsua1a138d2009-09-25 11:20:12 -07001602 "profile buffer not large enough"))
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001603 return;
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001604
Steven Rostedtff5f1492010-05-21 11:49:57 -04001605 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
Xiao Guangrong430ad5a2010-01-28 09:32:29 +08001606 if (!entry)
Xiao Guangrong1e12a4a2010-01-28 09:34:27 +08001607 return;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +01001608
Masami Hiramatsua1a138d2009-09-25 11:20:12 -07001609 entry->func = (unsigned long)tp->rp.kp.addr;
1610 entry->ret_ip = (unsigned long)ri->ret_addr;
Masami Hiramatsue09c8612010-07-05 15:54:45 -03001611 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
Frederic Weisbecker444a2a32009-11-06 04:13:05 +01001612
Peter Zijlstra3771f072010-05-21 12:31:09 +02001613 head = this_cpu_ptr(call->perf_events);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +02001614 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001615}
1616
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001617static int probe_perf_enable(struct ftrace_event_call *call)
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001618{
1619 struct trace_probe *tp = (struct trace_probe *)call->data;
1620
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001621 tp->flags |= TP_FLAG_PROFILE;
Frederic Weisbeckerd7a4b412009-09-23 23:08:43 +02001622
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001623 if (probe_is_return(tp))
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001624 return enable_kretprobe(&tp->rp);
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001625 else
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001626 return enable_kprobe(&tp->rp.kp);
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001627}
1628
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001629static void probe_perf_disable(struct ftrace_event_call *call)
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001630{
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001631 struct trace_probe *tp = (struct trace_probe *)call->data;
1632
Frederic Weisbeckerd7a4b412009-09-23 23:08:43 +02001633 tp->flags &= ~TP_FLAG_PROFILE;
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001634
Frederic Weisbeckerd7a4b412009-09-23 23:08:43 +02001635 if (!(tp->flags & TP_FLAG_TRACE)) {
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001636 if (probe_is_return(tp))
1637 disable_kretprobe(&tp->rp);
1638 else
1639 disable_kprobe(&tp->rp.kp);
1640 }
1641}
Li Zefan07b139c2009-12-21 14:27:35 +08001642#endif /* CONFIG_PERF_EVENTS */
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001643
Steven Rostedt22392912010-04-21 12:27:06 -04001644static __kprobes
1645int kprobe_register(struct ftrace_event_call *event, enum trace_reg type)
1646{
1647 switch (type) {
1648 case TRACE_REG_REGISTER:
1649 return probe_event_enable(event);
1650 case TRACE_REG_UNREGISTER:
1651 probe_event_disable(event);
1652 return 0;
1653
1654#ifdef CONFIG_PERF_EVENTS
1655 case TRACE_REG_PERF_REGISTER:
1656 return probe_perf_enable(event);
1657 case TRACE_REG_PERF_UNREGISTER:
1658 probe_perf_disable(event);
1659 return 0;
1660#endif
1661 }
1662 return 0;
1663}
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001664
1665static __kprobes
1666int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1667{
1668 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1669
1670 if (tp->flags & TP_FLAG_TRACE)
1671 kprobe_trace_func(kp, regs);
Li Zefan07b139c2009-12-21 14:27:35 +08001672#ifdef CONFIG_PERF_EVENTS
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001673 if (tp->flags & TP_FLAG_PROFILE)
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001674 kprobe_perf_func(kp, regs);
Li Zefan07b139c2009-12-21 14:27:35 +08001675#endif
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001676 return 0; /* We don't tweek kernel, so just return 0 */
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001677}
1678
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001679static __kprobes
1680int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1681{
1682 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1683
1684 if (tp->flags & TP_FLAG_TRACE)
1685 kretprobe_trace_func(ri, regs);
Li Zefan07b139c2009-12-21 14:27:35 +08001686#ifdef CONFIG_PERF_EVENTS
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001687 if (tp->flags & TP_FLAG_PROFILE)
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01001688 kretprobe_perf_func(ri, regs);
Li Zefan07b139c2009-12-21 14:27:35 +08001689#endif
Masami Hiramatsu50d78052009-09-14 16:49:20 -04001690 return 0; /* We don't tweek kernel, so just return 0 */
1691}
Masami Hiramatsue08d1c62009-09-10 19:53:30 -04001692
Steven Rostedta9a57762010-04-22 18:46:14 -04001693static struct trace_event_functions kretprobe_funcs = {
1694 .trace = print_kretprobe_event
1695};
1696
1697static struct trace_event_functions kprobe_funcs = {
1698 .trace = print_kprobe_event
1699};
1700
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001701static int register_probe_event(struct trace_probe *tp)
1702{
1703 struct ftrace_event_call *call = &tp->call;
1704 int ret;
1705
1706 /* Initialize ftrace_event_call */
Li Zefanffb9f992010-05-24 16:24:52 +08001707 INIT_LIST_HEAD(&call->class->fields);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001708 if (probe_is_return(tp)) {
Steven Rostedt80decc72010-04-23 10:00:22 -04001709 call->event.funcs = &kretprobe_funcs;
Steven Rostedt2e33af02010-04-22 10:35:55 -04001710 call->class->define_fields = kretprobe_event_define_fields;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001711 } else {
Steven Rostedt80decc72010-04-23 10:00:22 -04001712 call->event.funcs = &kprobe_funcs;
Steven Rostedt2e33af02010-04-22 10:35:55 -04001713 call->class->define_fields = kprobe_event_define_fields;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001714 }
Lai Jiangshana342a0282009-12-15 15:39:49 +08001715 if (set_print_fmt(tp) < 0)
1716 return -ENOMEM;
Steven Rostedt32c0eda2010-04-23 10:38:03 -04001717 ret = register_ftrace_event(&call->event);
1718 if (!ret) {
Lai Jiangshana342a0282009-12-15 15:39:49 +08001719 kfree(call->print_fmt);
Masami Hiramatsuff50d992009-08-13 16:35:34 -04001720 return -ENODEV;
Lai Jiangshana342a0282009-12-15 15:39:49 +08001721 }
Steven Rostedt553552c2010-04-23 11:12:36 -04001722 call->flags = 0;
Steven Rostedt22392912010-04-21 12:27:06 -04001723 call->class->reg = kprobe_register;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001724 call->data = tp;
1725 ret = trace_add_event_call(call);
Masami Hiramatsuff50d992009-08-13 16:35:34 -04001726 if (ret) {
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001727 pr_info("Failed to register kprobe event: %s\n", call->name);
Lai Jiangshana342a0282009-12-15 15:39:49 +08001728 kfree(call->print_fmt);
Steven Rostedt80decc72010-04-23 10:00:22 -04001729 unregister_ftrace_event(&call->event);
Masami Hiramatsuff50d992009-08-13 16:35:34 -04001730 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001731 return ret;
1732}
1733
1734static void unregister_probe_event(struct trace_probe *tp)
1735{
Masami Hiramatsuff50d992009-08-13 16:35:34 -04001736 /* tp->event is unregistered in trace_remove_event_call() */
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001737 trace_remove_event_call(&tp->call);
Lai Jiangshana342a0282009-12-15 15:39:49 +08001738 kfree(tp->call.print_fmt);
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001739}
1740
1741/* Make a debugfs interface for controling probe points */
1742static __init int init_kprobe_trace(void)
1743{
1744 struct dentry *d_tracer;
1745 struct dentry *entry;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001746
1747 d_tracer = tracing_init_dentry();
1748 if (!d_tracer)
1749 return 0;
1750
1751 entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1752 NULL, &kprobe_events_ops);
1753
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -04001754 /* Event list interface */
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001755 if (!entry)
1756 pr_warning("Could not create debugfs "
1757 "'kprobe_events' entry\n");
Masami Hiramatsucd7e7bd2009-08-13 16:35:42 -04001758
1759 /* Profile interface */
1760 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1761 NULL, &kprobe_profile_ops);
1762
1763 if (!entry)
1764 pr_warning("Could not create debugfs "
1765 "'kprobe_profile' entry\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001766 return 0;
1767}
1768fs_initcall(init_kprobe_trace);
1769
1770
1771#ifdef CONFIG_FTRACE_STARTUP_TEST
1772
1773static int kprobe_trace_selftest_target(int a1, int a2, int a3,
1774 int a4, int a5, int a6)
1775{
1776 return a1 + a2 + a3 + a4 + a5 + a6;
1777}
1778
1779static __init int kprobe_trace_self_tests_init(void)
1780{
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001781 int ret, warn = 0;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001782 int (*target)(int, int, int, int, int, int);
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001783 struct trace_probe *tp;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001784
1785 target = kprobe_trace_selftest_target;
1786
1787 pr_info("Testing kprobe tracing: ");
1788
1789 ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target "
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001790 "$stack $stack0 +0($stack)");
1791 if (WARN_ON_ONCE(ret)) {
1792 pr_warning("error on probing function entry.\n");
1793 warn++;
1794 } else {
1795 /* Enable trace point */
1796 tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM);
1797 if (WARN_ON_ONCE(tp == NULL)) {
1798 pr_warning("error on getting new probe.\n");
1799 warn++;
1800 } else
1801 probe_event_enable(&tp->call);
1802 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001803
1804 ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
Masami Hiramatsuf397af02009-10-16 20:07:20 -04001805 "$retval");
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001806 if (WARN_ON_ONCE(ret)) {
1807 pr_warning("error on probing function return.\n");
1808 warn++;
1809 } else {
1810 /* Enable trace point */
1811 tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM);
1812 if (WARN_ON_ONCE(tp == NULL)) {
1813 pr_warning("error on getting new probe.\n");
1814 warn++;
1815 } else
1816 probe_event_enable(&tp->call);
1817 }
1818
1819 if (warn)
1820 goto end;
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001821
1822 ret = target(1, 2, 3, 4, 5, 6);
1823
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001824 ret = command_trace_probe("-:testprobe");
1825 if (WARN_ON_ONCE(ret)) {
1826 pr_warning("error on deleting a probe.\n");
1827 warn++;
1828 }
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001829
Masami Hiramatsu231e36f2010-01-14 00:12:12 -05001830 ret = command_trace_probe("-:testprobe2");
1831 if (WARN_ON_ONCE(ret)) {
1832 pr_warning("error on deleting a probe.\n");
1833 warn++;
1834 }
1835
1836end:
1837 cleanup_all_probes();
1838 if (warn)
1839 pr_cont("NG: Some tests are failed. Please check them.\n");
1840 else
1841 pr_cont("OK\n");
Masami Hiramatsu413d37d2009-08-13 16:35:11 -04001842 return 0;
1843}
1844
1845late_initcall(kprobe_trace_self_tests_init);
1846
1847#endif