blob: e35be53f6613c4fd08c093b6865cd6f43e0b9a89 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
Hien Nguyenb94cce92005-06-23 00:09:19 -070030 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 */
34#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/hash.h>
36#include <linux/init.h>
Tim Schmielau4e57b682005-10-30 15:03:48 -080037#include <linux/slab.h>
Randy Dunlape3869792007-05-08 00:27:01 -070038#include <linux/stddef.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040039#include <linux/export.h>
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070040#include <linux/moduleloader.h>
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070041#include <linux/kallsyms.h>
Masami Hiramatsub4c6c342006-12-06 20:38:11 -080042#include <linux/freezer.h>
Srinivasa Ds346fd592007-02-20 13:57:54 -080043#include <linux/seq_file.h>
44#include <linux/debugfs.h>
Masami Hiramatsub2be84d2010-02-25 08:34:15 -050045#include <linux/sysctl.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070046#include <linux/kdebug.h>
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -050047#include <linux/memory.h>
Masami Hiramatsu4554dbc2010-02-02 16:49:18 -050048#include <linux/ftrace.h>
Masami Hiramatsuafd66252010-02-25 08:34:07 -050049#include <linux/cpu.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040050#include <linux/jump_label.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070051
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -070052#include <asm-generic/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <asm/cacheflush.h>
54#include <asm/errno.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070055#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070060
61/*
62 * Some oddball architectures like 64bit powerpc have function descriptors
63 * so this must be overridable.
64 */
65#ifndef kprobe_lookup_name
66#define kprobe_lookup_name(name, addr) \
67 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
68#endif
69
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070070static int kprobes_initialized;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
Hien Nguyenb94cce92005-06-23 00:09:19 -070072static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070074/* NOTE: change this value only with kprobe_mutex held */
Masami Hiramatsue579abe2009-04-06 19:01:01 -070075static bool kprobes_all_disarmed;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070076
Masami Hiramatsu43948f52010-10-25 22:18:01 +090077/* This protects kprobe_table and optimizing_list */
78static DEFINE_MUTEX(kprobe_mutex);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -080079static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070080static struct {
Thomas Gleixnerec4846082009-07-25 16:09:17 +020081 raw_spinlock_t lock ____cacheline_aligned_in_smp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070082} kretprobe_table_locks[KPROBE_TABLE_SIZE];
83
Thomas Gleixnerec4846082009-07-25 16:09:17 +020084static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070085{
86 return &(kretprobe_table_locks[hash].lock);
87}
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070089/*
90 * Normally, functions that we'd want to prohibit kprobes in, are marked
91 * __kprobes. But, there are cases where such functions already belong to
92 * a different section (__sched for preempt_schedule)
93 *
94 * For such cases, we now have a blacklist
95 */
Daniel Guilak544304b2008-07-10 09:38:19 -070096static struct kprobe_blackpoint kprobe_blacklist[] = {
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070097 {"preempt_schedule",},
Masami Hiramatsu65e234e2009-08-27 13:23:32 -040098 {"native_get_debugreg",},
Masami Hiramatsua00e8172009-09-08 12:47:55 -040099 {"irq_entries_start",},
100 {"common_interrupt",},
Masami Hiramatsu5ecaafd2010-02-05 01:24:34 -0500101 {"mcount",}, /* mcount can be called from everywhere */
Srinivasa Ds3d8d9962008-04-28 02:14:26 -0700102 {NULL} /* Terminator */
103};
104
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -0800105#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700106/*
107 * kprobe->ainsn.insn points to the copy of the instruction to be
108 * single-stepped. x86_64, POWER4 and above have no-exec support and
109 * stepping on the instruction on a vmalloced/kmalloced/data page
110 * is a recipe for disaster
111 */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700112struct kprobe_insn_page {
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400113 struct list_head list;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700114 kprobe_opcode_t *insns; /* Page of instruction slots */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700115 int nused;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800116 int ngarbage;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500117 char slot_used[];
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700118};
119
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500120#define KPROBE_INSN_PAGE_SIZE(slots) \
121 (offsetof(struct kprobe_insn_page, slot_used) + \
122 (sizeof(char) * (slots)))
123
124struct kprobe_insn_cache {
125 struct list_head pages; /* list of kprobe_insn_page */
126 size_t insn_size; /* size of instruction slot */
127 int nr_garbage;
128};
129
130static int slots_per_page(struct kprobe_insn_cache *c)
131{
132 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
133}
134
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800135enum kprobe_slot_state {
136 SLOT_CLEAN = 0,
137 SLOT_DIRTY = 1,
138 SLOT_USED = 2,
139};
140
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500141static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
142static struct kprobe_insn_cache kprobe_insn_slots = {
143 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
144 .insn_size = MAX_INSN_SIZE,
145 .nr_garbage = 0,
146};
147static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800148
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700149/**
Masami Hiramatsu12941562009-01-06 14:41:50 -0800150 * __get_insn_slot() - Find a slot on an executable page for an instruction.
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700151 * We allocate an executable page if there's no room on existing ones.
152 */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500153static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700154{
155 struct kprobe_insn_page *kip;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700156
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700157 retry:
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500158 list_for_each_entry(kip, &c->pages, list) {
159 if (kip->nused < slots_per_page(c)) {
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700160 int i;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500161 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800162 if (kip->slot_used[i] == SLOT_CLEAN) {
163 kip->slot_used[i] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700164 kip->nused++;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500165 return kip->insns + (i * c->insn_size);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700166 }
167 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500168 /* kip->nused is broken. Fix it. */
169 kip->nused = slots_per_page(c);
170 WARN_ON(1);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700171 }
172 }
173
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800174 /* If there are any garbage slots, collect it and try again. */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500175 if (c->nr_garbage && collect_garbage_slots(c) == 0)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800176 goto retry;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500177
178 /* All out of space. Need to allocate a new page. */
179 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700180 if (!kip)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700181 return NULL;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700182
183 /*
184 * Use module_alloc so this page is within +/- 2GB of where the
185 * kernel image and loaded module images reside. This is required
186 * so x86_64 can correctly handle the %rip-relative fixups.
187 */
188 kip->insns = module_alloc(PAGE_SIZE);
189 if (!kip->insns) {
190 kfree(kip);
191 return NULL;
192 }
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400193 INIT_LIST_HEAD(&kip->list);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500194 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800195 kip->slot_used[0] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700196 kip->nused = 1;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800197 kip->ngarbage = 0;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500198 list_add(&kip->list, &c->pages);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700199 return kip->insns;
200}
201
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500202
Masami Hiramatsu12941562009-01-06 14:41:50 -0800203kprobe_opcode_t __kprobes *get_insn_slot(void)
204{
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500205 kprobe_opcode_t *ret = NULL;
206
Masami Hiramatsu12941562009-01-06 14:41:50 -0800207 mutex_lock(&kprobe_insn_mutex);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500208 ret = __get_insn_slot(&kprobe_insn_slots);
Masami Hiramatsu12941562009-01-06 14:41:50 -0800209 mutex_unlock(&kprobe_insn_mutex);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500210
Masami Hiramatsu12941562009-01-06 14:41:50 -0800211 return ret;
212}
213
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800214/* Return 1 if all garbages are collected, otherwise 0. */
215static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
216{
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800217 kip->slot_used[idx] = SLOT_CLEAN;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800218 kip->nused--;
219 if (kip->nused == 0) {
220 /*
221 * Page is no longer in use. Free it unless
222 * it's the last one. We keep the last one
223 * so as not to have to set it up again the
224 * next time somebody inserts a probe.
225 */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500226 if (!list_is_singular(&kip->list)) {
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400227 list_del(&kip->list);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800228 module_free(NULL, kip->insns);
229 kfree(kip);
230 }
231 return 1;
232 }
233 return 0;
234}
235
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500236static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800237{
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400238 struct kprobe_insn_page *kip, *next;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800239
Masami Hiramatsu615d0eb2010-02-02 16:49:04 -0500240 /* Ensure no-one is interrupted on the garbages */
241 synchronize_sched();
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800242
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500243 list_for_each_entry_safe(kip, next, &c->pages, list) {
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800244 int i;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800245 if (kip->ngarbage == 0)
246 continue;
247 kip->ngarbage = 0; /* we will collect all garbages */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500248 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800249 if (kip->slot_used[i] == SLOT_DIRTY &&
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800250 collect_one_slot(kip, i))
251 break;
252 }
253 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500254 c->nr_garbage = 0;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800255 return 0;
256}
257
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500258static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
259 kprobe_opcode_t *slot, int dirty)
260{
261 struct kprobe_insn_page *kip;
262
263 list_for_each_entry(kip, &c->pages, list) {
Masami Hiramatsu83ff56f2010-03-09 10:22:19 -0500264 long idx = ((long)slot - (long)kip->insns) /
265 (c->insn_size * sizeof(kprobe_opcode_t));
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500266 if (idx >= 0 && idx < slots_per_page(c)) {
267 WARN_ON(kip->slot_used[idx] != SLOT_USED);
268 if (dirty) {
269 kip->slot_used[idx] = SLOT_DIRTY;
270 kip->ngarbage++;
271 if (++c->nr_garbage > slots_per_page(c))
272 collect_garbage_slots(c);
273 } else
274 collect_one_slot(kip, idx);
275 return;
276 }
277 }
278 /* Could not free this slot. */
279 WARN_ON(1);
280}
281
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800282void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700283{
Masami Hiramatsu12941562009-01-06 14:41:50 -0800284 mutex_lock(&kprobe_insn_mutex);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500285 __free_insn_slot(&kprobe_insn_slots, slot, dirty);
Masami Hiramatsu12941562009-01-06 14:41:50 -0800286 mutex_unlock(&kprobe_insn_mutex);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700287}
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500288#ifdef CONFIG_OPTPROBES
289/* For optimized_kprobe buffer */
290static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
291static struct kprobe_insn_cache kprobe_optinsn_slots = {
292 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
293 /* .insn_size is initialized later */
294 .nr_garbage = 0,
295};
296/* Get a slot for optimized_kprobe buffer */
297kprobe_opcode_t __kprobes *get_optinsn_slot(void)
298{
299 kprobe_opcode_t *ret = NULL;
300
301 mutex_lock(&kprobe_optinsn_mutex);
302 ret = __get_insn_slot(&kprobe_optinsn_slots);
303 mutex_unlock(&kprobe_optinsn_mutex);
304
305 return ret;
306}
307
308void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
309{
310 mutex_lock(&kprobe_optinsn_mutex);
311 __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
312 mutex_unlock(&kprobe_optinsn_mutex);
313}
314#endif
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -0800315#endif
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700316
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800317/* We have preemption disabled.. so it is safe to use __ versions */
318static inline void set_kprobe_instance(struct kprobe *kp)
319{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600320 __this_cpu_write(kprobe_instance, kp);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800321}
322
323static inline void reset_kprobe_instance(void)
324{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600325 __this_cpu_write(kprobe_instance, NULL);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800326}
327
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800328/*
329 * This routine is called either:
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800330 * - under the kprobe_mutex - during kprobe_[un]register()
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800331 * OR
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800332 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800333 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700334struct kprobe __kprobes *get_kprobe(void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
336 struct hlist_head *head;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800337 struct kprobe *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
339 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800340 hlist_for_each_entry_rcu(p, head, hlist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 if (p->addr == addr)
342 return p;
343 }
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 return NULL;
346}
347
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500348static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
349
350/* Return true if the kprobe is an aggregator */
351static inline int kprobe_aggrprobe(struct kprobe *p)
352{
353 return p->pre_handler == aggr_pre_handler;
354}
355
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900356/* Return true(!0) if the kprobe is unused */
357static inline int kprobe_unused(struct kprobe *p)
358{
359 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
360 list_empty(&p->list);
361}
362
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500363/*
364 * Keep all fields in the kprobe consistent
365 */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900366static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500367{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900368 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
369 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500370}
371
372#ifdef CONFIG_OPTPROBES
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500373/* NOTE: change this value only with kprobe_mutex held */
374static bool kprobes_allow_optimization;
375
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500376/*
377 * Call all pre_handler on the list, but ignores its return value.
378 * This must be called from arch-dep optimized caller.
379 */
380void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
381{
382 struct kprobe *kp;
383
384 list_for_each_entry_rcu(kp, &p->list, list) {
385 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
386 set_kprobe_instance(kp);
387 kp->pre_handler(kp, regs);
388 }
389 reset_kprobe_instance();
390 }
391}
392
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900393/* Free optimized instructions and optimized_kprobe */
394static __kprobes void free_aggr_kprobe(struct kprobe *p)
395{
396 struct optimized_kprobe *op;
397
398 op = container_of(p, struct optimized_kprobe, kp);
399 arch_remove_optimized_kprobe(op);
400 arch_remove_kprobe(p);
401 kfree(op);
402}
403
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500404/* Return true(!0) if the kprobe is ready for optimization. */
405static inline int kprobe_optready(struct kprobe *p)
406{
407 struct optimized_kprobe *op;
408
409 if (kprobe_aggrprobe(p)) {
410 op = container_of(p, struct optimized_kprobe, kp);
411 return arch_prepared_optinsn(&op->optinsn);
412 }
413
414 return 0;
415}
416
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900417/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
418static inline int kprobe_disarmed(struct kprobe *p)
419{
420 struct optimized_kprobe *op;
421
422 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
423 if (!kprobe_aggrprobe(p))
424 return kprobe_disabled(p);
425
426 op = container_of(p, struct optimized_kprobe, kp);
427
428 return kprobe_disabled(p) && list_empty(&op->list);
429}
430
431/* Return true(!0) if the probe is queued on (un)optimizing lists */
432static int __kprobes kprobe_queued(struct kprobe *p)
433{
434 struct optimized_kprobe *op;
435
436 if (kprobe_aggrprobe(p)) {
437 op = container_of(p, struct optimized_kprobe, kp);
438 if (!list_empty(&op->list))
439 return 1;
440 }
441 return 0;
442}
443
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500444/*
445 * Return an optimized kprobe whose optimizing code replaces
446 * instructions including addr (exclude breakpoint).
447 */
Namhyung Kim6376b222010-09-15 10:04:28 +0900448static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500449{
450 int i;
451 struct kprobe *p = NULL;
452 struct optimized_kprobe *op;
453
454 /* Don't check i == 0, since that is a breakpoint case. */
455 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
456 p = get_kprobe((void *)(addr - i));
457
458 if (p && kprobe_optready(p)) {
459 op = container_of(p, struct optimized_kprobe, kp);
460 if (arch_within_optimized_kprobe(op, addr))
461 return p;
462 }
463
464 return NULL;
465}
466
467/* Optimization staging list, protected by kprobe_mutex */
468static LIST_HEAD(optimizing_list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900469static LIST_HEAD(unoptimizing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500470
471static void kprobe_optimizer(struct work_struct *work);
472static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
473#define OPTIMIZE_DELAY 5
474
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900475/*
476 * Optimize (replace a breakpoint with a jump) kprobes listed on
477 * optimizing_list.
478 */
479static __kprobes void do_optimize_kprobes(void)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500480{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900481 /* Optimization never be done when disarmed */
482 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
483 list_empty(&optimizing_list))
484 return;
485
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500486 /*
487 * The optimization/unoptimization refers online_cpus via
488 * stop_machine() and cpu-hotplug modifies online_cpus.
489 * And same time, text_mutex will be held in cpu-hotplug and here.
490 * This combination can cause a deadlock (cpu-hotplug try to lock
491 * text_mutex but stop_machine can not be done because online_cpus
492 * has been changed)
493 * To avoid this deadlock, we need to call get_online_cpus()
494 * for preventing cpu-hotplug outside of text_mutex locking.
495 */
496 get_online_cpus();
497 mutex_lock(&text_mutex);
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900498 arch_optimize_kprobes(&optimizing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500499 mutex_unlock(&text_mutex);
500 put_online_cpus();
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900501}
502
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900503/*
504 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
505 * if need) kprobes listed on unoptimizing_list.
506 */
507static __kprobes void do_unoptimize_kprobes(struct list_head *free_list)
508{
509 struct optimized_kprobe *op, *tmp;
510
511 /* Unoptimization must be done anytime */
512 if (list_empty(&unoptimizing_list))
513 return;
514
515 /* Ditto to do_optimize_kprobes */
516 get_online_cpus();
517 mutex_lock(&text_mutex);
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900518 arch_unoptimize_kprobes(&unoptimizing_list, free_list);
519 /* Loop free_list for disarming */
520 list_for_each_entry_safe(op, tmp, free_list, list) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900521 /* Disarm probes if marked disabled */
522 if (kprobe_disabled(&op->kp))
523 arch_disarm_kprobe(&op->kp);
524 if (kprobe_unused(&op->kp)) {
525 /*
526 * Remove unused probes from hash list. After waiting
527 * for synchronization, these probes are reclaimed.
528 * (reclaiming is done by do_free_cleaned_kprobes.)
529 */
530 hlist_del_rcu(&op->kp.hlist);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900531 } else
532 list_del_init(&op->list);
533 }
534 mutex_unlock(&text_mutex);
535 put_online_cpus();
536}
537
538/* Reclaim all kprobes on the free_list */
539static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list)
540{
541 struct optimized_kprobe *op, *tmp;
542
543 list_for_each_entry_safe(op, tmp, free_list, list) {
544 BUG_ON(!kprobe_unused(&op->kp));
545 list_del_init(&op->list);
546 free_aggr_kprobe(&op->kp);
547 }
548}
549
550/* Start optimizer after OPTIMIZE_DELAY passed */
551static __kprobes void kick_kprobe_optimizer(void)
552{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800553 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900554}
555
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900556/* Kprobe jump optimizer */
557static __kprobes void kprobe_optimizer(struct work_struct *work)
558{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900559 LIST_HEAD(free_list);
560
Steven Rostedt72ef3792012-06-05 19:28:14 +0900561 mutex_lock(&kprobe_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900562 /* Lock modules while optimizing kprobes */
563 mutex_lock(&module_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900564
565 /*
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900566 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
567 * kprobes before waiting for quiesence period.
568 */
569 do_unoptimize_kprobes(&free_list);
570
571 /*
572 * Step 2: Wait for quiesence period to ensure all running interrupts
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900573 * are done. Because optprobe may modify multiple instructions
574 * there is a chance that Nth instruction is interrupted. In that
575 * case, running interrupt can return to 2nd-Nth byte of jump
576 * instruction. This wait is for avoiding it.
577 */
578 synchronize_sched();
579
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900580 /* Step 3: Optimize kprobes after quiesence period */
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900581 do_optimize_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900582
583 /* Step 4: Free cleaned kprobes after quiesence period */
584 do_free_cleaned_kprobes(&free_list);
585
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500586 mutex_unlock(&module_mutex);
Steven Rostedt72ef3792012-06-05 19:28:14 +0900587 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900588
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900589 /* Step 5: Kick optimizer again if needed */
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900590 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900591 kick_kprobe_optimizer();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900592}
593
594/* Wait for completing optimization and unoptimization */
595static __kprobes void wait_for_kprobe_optimizer(void)
596{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800597 mutex_lock(&kprobe_mutex);
598
599 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
600 mutex_unlock(&kprobe_mutex);
601
602 /* this will also make optimizing_work execute immmediately */
603 flush_delayed_work(&optimizing_work);
604 /* @optimizing_work might not have been queued yet, relax */
605 cpu_relax();
606
607 mutex_lock(&kprobe_mutex);
608 }
609
610 mutex_unlock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500611}
612
613/* Optimize kprobe if p is ready to be optimized */
614static __kprobes void optimize_kprobe(struct kprobe *p)
615{
616 struct optimized_kprobe *op;
617
618 /* Check if the kprobe is disabled or not ready for optimization. */
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500619 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500620 (kprobe_disabled(p) || kprobes_all_disarmed))
621 return;
622
623 /* Both of break_handler and post_handler are not supported. */
624 if (p->break_handler || p->post_handler)
625 return;
626
627 op = container_of(p, struct optimized_kprobe, kp);
628
629 /* Check there is no other kprobes at the optimized instructions */
630 if (arch_check_optimized_kprobe(op) < 0)
631 return;
632
633 /* Check if it is already optimized. */
634 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
635 return;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500636 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900637
638 if (!list_empty(&op->list))
639 /* This is under unoptimizing. Just dequeue the probe */
640 list_del_init(&op->list);
641 else {
642 list_add(&op->list, &optimizing_list);
643 kick_kprobe_optimizer();
644 }
645}
646
647/* Short cut to direct unoptimizing */
648static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)
649{
650 get_online_cpus();
651 arch_unoptimize_kprobe(op);
652 put_online_cpus();
653 if (kprobe_disabled(&op->kp))
654 arch_disarm_kprobe(&op->kp);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500655}
656
657/* Unoptimize a kprobe if p is optimized */
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900658static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500659{
660 struct optimized_kprobe *op;
661
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900662 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
663 return; /* This is not an optprobe nor optimized */
664
665 op = container_of(p, struct optimized_kprobe, kp);
666 if (!kprobe_optimized(p)) {
667 /* Unoptimized or unoptimizing case */
668 if (force && !list_empty(&op->list)) {
669 /*
670 * Only if this is unoptimizing kprobe and forced,
671 * forcibly unoptimize it. (No need to unoptimize
672 * unoptimized kprobe again :)
673 */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500674 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900675 force_unoptimize_kprobe(op);
676 }
677 return;
678 }
679
680 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
681 if (!list_empty(&op->list)) {
682 /* Dequeue from the optimization queue */
683 list_del_init(&op->list);
684 return;
685 }
686 /* Optimized kprobe case */
687 if (force)
688 /* Forcibly update the code: this is a special case */
689 force_unoptimize_kprobe(op);
690 else {
691 list_add(&op->list, &unoptimizing_list);
692 kick_kprobe_optimizer();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500693 }
694}
695
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900696/* Cancel unoptimizing for reusing */
697static void reuse_unused_kprobe(struct kprobe *ap)
698{
699 struct optimized_kprobe *op;
700
701 BUG_ON(!kprobe_unused(ap));
702 /*
703 * Unused kprobe MUST be on the way of delayed unoptimizing (means
704 * there is still a relative jump) and disabled.
705 */
706 op = container_of(ap, struct optimized_kprobe, kp);
707 if (unlikely(list_empty(&op->list)))
708 printk(KERN_WARNING "Warning: found a stray unused "
709 "aggrprobe@%p\n", ap->addr);
710 /* Enable the probe again */
711 ap->flags &= ~KPROBE_FLAG_DISABLED;
712 /* Optimize it again (remove from op->list) */
713 BUG_ON(!kprobe_optready(ap));
714 optimize_kprobe(ap);
715}
716
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500717/* Remove optimized instructions */
718static void __kprobes kill_optimized_kprobe(struct kprobe *p)
719{
720 struct optimized_kprobe *op;
721
722 op = container_of(p, struct optimized_kprobe, kp);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900723 if (!list_empty(&op->list))
724 /* Dequeue from the (un)optimization queue */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500725 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900726
727 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
728 /* Don't touch the code, because it is already freed. */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500729 arch_remove_optimized_kprobe(op);
730}
731
732/* Try to prepare optimized instructions */
733static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
734{
735 struct optimized_kprobe *op;
736
737 op = container_of(p, struct optimized_kprobe, kp);
738 arch_prepare_optimized_kprobe(op);
739}
740
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500741/* Allocate new optimized_kprobe and try to prepare optimized instructions */
742static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
743{
744 struct optimized_kprobe *op;
745
746 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
747 if (!op)
748 return NULL;
749
750 INIT_LIST_HEAD(&op->list);
751 op->kp.addr = p->addr;
752 arch_prepare_optimized_kprobe(op);
753
754 return &op->kp;
755}
756
757static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
758
759/*
760 * Prepare an optimized_kprobe and optimize it
761 * NOTE: p must be a normal registered kprobe
762 */
763static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
764{
765 struct kprobe *ap;
766 struct optimized_kprobe *op;
767
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900768 /* Impossible to optimize ftrace-based kprobe */
769 if (kprobe_ftrace(p))
770 return;
771
Masami Hiramatsu25764282012-06-05 19:28:26 +0900772 /* For preparing optimization, jump_label_text_reserved() is called */
773 jump_label_lock();
774 mutex_lock(&text_mutex);
775
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500776 ap = alloc_aggr_kprobe(p);
777 if (!ap)
Masami Hiramatsu25764282012-06-05 19:28:26 +0900778 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500779
780 op = container_of(ap, struct optimized_kprobe, kp);
781 if (!arch_prepared_optinsn(&op->optinsn)) {
782 /* If failed to setup optimizing, fallback to kprobe */
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900783 arch_remove_optimized_kprobe(op);
784 kfree(op);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900785 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500786 }
787
788 init_aggr_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900789 optimize_kprobe(ap); /* This just kicks optimizer thread */
790
791out:
792 mutex_unlock(&text_mutex);
793 jump_label_unlock();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500794}
795
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500796#ifdef CONFIG_SYSCTL
Masami Hiramatsu43948f52010-10-25 22:18:01 +0900797/* This should be called with kprobe_mutex locked */
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500798static void __kprobes optimize_all_kprobes(void)
799{
800 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500801 struct kprobe *p;
802 unsigned int i;
803
804 /* If optimization is already allowed, just return */
805 if (kprobes_allow_optimization)
806 return;
807
808 kprobes_allow_optimization = true;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500809 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
810 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800811 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500812 if (!kprobe_disabled(p))
813 optimize_kprobe(p);
814 }
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500815 printk(KERN_INFO "Kprobes globally optimized\n");
816}
817
Masami Hiramatsu43948f52010-10-25 22:18:01 +0900818/* This should be called with kprobe_mutex locked */
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500819static void __kprobes unoptimize_all_kprobes(void)
820{
821 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500822 struct kprobe *p;
823 unsigned int i;
824
825 /* If optimization is already prohibited, just return */
826 if (!kprobes_allow_optimization)
827 return;
828
829 kprobes_allow_optimization = false;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500830 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
831 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800832 hlist_for_each_entry_rcu(p, head, hlist) {
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500833 if (!kprobe_disabled(p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900834 unoptimize_kprobe(p, false);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500835 }
836 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900837 /* Wait for unoptimizing completion */
838 wait_for_kprobe_optimizer();
839 printk(KERN_INFO "Kprobes globally unoptimized\n");
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500840}
841
842int sysctl_kprobes_optimization;
843int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
844 void __user *buffer, size_t *length,
845 loff_t *ppos)
846{
847 int ret;
848
849 mutex_lock(&kprobe_mutex);
850 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
851 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
852
853 if (sysctl_kprobes_optimization)
854 optimize_all_kprobes();
855 else
856 unoptimize_all_kprobes();
857 mutex_unlock(&kprobe_mutex);
858
859 return ret;
860}
861#endif /* CONFIG_SYSCTL */
862
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900863/* Put a breakpoint for a probe. Must be called with text_mutex locked */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500864static void __kprobes __arm_kprobe(struct kprobe *p)
865{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900866 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500867
868 /* Check collision with other optimized kprobes */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900869 _p = get_optimized_kprobe((unsigned long)p->addr);
870 if (unlikely(_p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900871 /* Fallback to unoptimized kprobe */
872 unoptimize_kprobe(_p, true);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500873
874 arch_arm_kprobe(p);
875 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
876}
877
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900878/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
879static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500880{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900881 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500882
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900883 unoptimize_kprobe(p, false); /* Try to unoptimize */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500884
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900885 if (!kprobe_queued(p)) {
886 arch_disarm_kprobe(p);
887 /* If another kprobe was blocked, optimize it. */
888 _p = get_optimized_kprobe((unsigned long)p->addr);
889 if (unlikely(_p) && reopt)
890 optimize_kprobe(_p);
891 }
892 /* TODO: reoptimize others after unoptimized this probe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500893}
894
895#else /* !CONFIG_OPTPROBES */
896
897#define optimize_kprobe(p) do {} while (0)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900898#define unoptimize_kprobe(p, f) do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500899#define kill_optimized_kprobe(p) do {} while (0)
900#define prepare_optimized_kprobe(p) do {} while (0)
901#define try_to_optimize_kprobe(p) do {} while (0)
902#define __arm_kprobe(p) arch_arm_kprobe(p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900903#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
904#define kprobe_disarmed(p) kprobe_disabled(p)
905#define wait_for_kprobe_optimizer() do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500906
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900907/* There should be no unused kprobes can be reused without optimization */
908static void reuse_unused_kprobe(struct kprobe *ap)
909{
910 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
911 BUG_ON(kprobe_unused(ap));
912}
913
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500914static __kprobes void free_aggr_kprobe(struct kprobe *p)
915{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900916 arch_remove_kprobe(p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500917 kfree(p);
918}
919
920static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
921{
922 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
923}
924#endif /* CONFIG_OPTPROBES */
925
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +0900926#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900927static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
Masami Hiramatsue5253892012-06-05 19:28:38 +0900928 .func = kprobe_ftrace_handler,
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900929 .flags = FTRACE_OPS_FL_SAVE_REGS,
930};
931static int kprobe_ftrace_enabled;
932
933/* Must ensure p->addr is really on ftrace */
934static int __kprobes prepare_kprobe(struct kprobe *p)
935{
936 if (!kprobe_ftrace(p))
937 return arch_prepare_kprobe(p);
938
939 return arch_prepare_kprobe_ftrace(p);
940}
941
942/* Caller must lock kprobe_mutex */
943static void __kprobes arm_kprobe_ftrace(struct kprobe *p)
944{
945 int ret;
946
947 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
948 (unsigned long)p->addr, 0, 0);
949 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret);
950 kprobe_ftrace_enabled++;
951 if (kprobe_ftrace_enabled == 1) {
952 ret = register_ftrace_function(&kprobe_ftrace_ops);
953 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
954 }
955}
956
957/* Caller must lock kprobe_mutex */
958static void __kprobes disarm_kprobe_ftrace(struct kprobe *p)
959{
960 int ret;
961
962 kprobe_ftrace_enabled--;
963 if (kprobe_ftrace_enabled == 0) {
964 ret = unregister_ftrace_function(&kprobe_ftrace_ops);
965 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
966 }
967 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
968 (unsigned long)p->addr, 1, 0);
969 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret);
970}
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +0900971#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900972#define prepare_kprobe(p) arch_prepare_kprobe(p)
973#define arm_kprobe_ftrace(p) do {} while (0)
974#define disarm_kprobe_ftrace(p) do {} while (0)
975#endif
976
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400977/* Arm a kprobe with text_mutex */
978static void __kprobes arm_kprobe(struct kprobe *kp)
979{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900980 if (unlikely(kprobe_ftrace(kp))) {
981 arm_kprobe_ftrace(kp);
982 return;
983 }
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500984 /*
985 * Here, since __arm_kprobe() doesn't use stop_machine(),
986 * this doesn't cause deadlock on text_mutex. So, we don't
987 * need get_online_cpus().
988 */
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400989 mutex_lock(&text_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500990 __arm_kprobe(kp);
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400991 mutex_unlock(&text_mutex);
992}
993
994/* Disarm a kprobe with text_mutex */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900995static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt)
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400996{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900997 if (unlikely(kprobe_ftrace(kp))) {
998 disarm_kprobe_ftrace(kp);
999 return;
1000 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001001 /* Ditto */
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001002 mutex_lock(&text_mutex);
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001003 __disarm_kprobe(kp, reopt);
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001004 mutex_unlock(&text_mutex);
1005}
1006
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001007/*
1008 * Aggregate handlers for multiple kprobes support - these handlers
1009 * take care of invoking the individual kprobe handlers on p->list
1010 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001011static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001012{
1013 struct kprobe *kp;
1014
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001015 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001016 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001017 set_kprobe_instance(kp);
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001018 if (kp->pre_handler(kp, regs))
1019 return 1;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001020 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001021 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001022 }
1023 return 0;
1024}
1025
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001026static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1027 unsigned long flags)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001028{
1029 struct kprobe *kp;
1030
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001031 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001032 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001033 set_kprobe_instance(kp);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001034 kp->post_handler(kp, regs, flags);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001035 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001036 }
1037 }
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001038}
1039
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001040static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1041 int trapnr)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001042{
Christoph Lameterb76834b2010-12-06 11:16:25 -06001043 struct kprobe *cur = __this_cpu_read(kprobe_instance);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001044
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001045 /*
1046 * if we faulted "during" the execution of a user specified
1047 * probe handler, invoke just that probe's fault handler
1048 */
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001049 if (cur && cur->fault_handler) {
1050 if (cur->fault_handler(cur, regs, trapnr))
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001051 return 1;
1052 }
1053 return 0;
1054}
1055
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001056static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001057{
Christoph Lameterb76834b2010-12-06 11:16:25 -06001058 struct kprobe *cur = __this_cpu_read(kprobe_instance);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001059 int ret = 0;
1060
1061 if (cur && cur->break_handler) {
1062 if (cur->break_handler(cur, regs))
1063 ret = 1;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001064 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001065 reset_kprobe_instance();
1066 return ret;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001067}
1068
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001069/* Walks the list and increments nmissed count for multiprobe case */
1070void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
1071{
1072 struct kprobe *kp;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001073 if (!kprobe_aggrprobe(p)) {
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001074 p->nmissed++;
1075 } else {
1076 list_for_each_entry_rcu(kp, &p->list, list)
1077 kp->nmissed++;
1078 }
1079 return;
1080}
1081
bibo,mao99219a32006-10-02 02:17:35 -07001082void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
1083 struct hlist_head *head)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001084{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001085 struct kretprobe *rp = ri->rp;
1086
Hien Nguyenb94cce92005-06-23 00:09:19 -07001087 /* remove rp inst off the rprobe_inst_table */
1088 hlist_del(&ri->hlist);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001089 INIT_HLIST_NODE(&ri->hlist);
1090 if (likely(rp)) {
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001091 raw_spin_lock(&rp->lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001092 hlist_add_head(&ri->hlist, &rp->free_instances);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001093 raw_spin_unlock(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001094 } else
1095 /* Unregistering */
bibo,mao99219a32006-10-02 02:17:35 -07001096 hlist_add_head(&ri->hlist, head);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001097}
1098
Masami Hiramatsu017c39b2009-01-06 14:41:51 -08001099void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001100 struct hlist_head **head, unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001101__acquires(hlist_lock)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001102{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001103 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001104 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001105
1106 *head = &kretprobe_inst_table[hash];
1107 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001108 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001109}
1110
Masami Hiramatsu017c39b2009-01-06 14:41:51 -08001111static void __kprobes kretprobe_table_lock(unsigned long hash,
1112 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001113__acquires(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001114{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001115 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1116 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001117}
1118
Masami Hiramatsu017c39b2009-01-06 14:41:51 -08001119void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
1120 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001121__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001122{
1123 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001124 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001125
1126 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001127 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001128}
1129
Namhyung Kim6376b222010-09-15 10:04:28 +09001130static void __kprobes kretprobe_table_unlock(unsigned long hash,
1131 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001132__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001133{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001134 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1135 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001136}
1137
Hien Nguyenb94cce92005-06-23 00:09:19 -07001138/*
bibo maoc6fd91f2006-03-26 01:38:20 -08001139 * This function is called from finish_task_switch when task tk becomes dead,
1140 * so that we can recycle any function-return probe instances associated
1141 * with this task. These left over instances represent probed functions
1142 * that have been called but will never return.
Hien Nguyenb94cce92005-06-23 00:09:19 -07001143 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001144void __kprobes kprobe_flush_task(struct task_struct *tk)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001145{
bibo,mao62c27be2006-10-02 02:17:33 -07001146 struct kretprobe_instance *ri;
bibo,mao99219a32006-10-02 02:17:35 -07001147 struct hlist_head *head, empty_rp;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001148 struct hlist_node *tmp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001149 unsigned long hash, flags = 0;
Rusty Lynch802eae72005-06-27 15:17:08 -07001150
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001151 if (unlikely(!kprobes_initialized))
1152 /* Early boot. kretprobe_table_locks not yet initialized. */
1153 return;
1154
Ananth N Mavinakayanahallid496aab2012-01-20 14:34:04 -08001155 INIT_HLIST_HEAD(&empty_rp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001156 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1157 head = &kretprobe_inst_table[hash];
1158 kretprobe_table_lock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001159 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
bibo,mao62c27be2006-10-02 02:17:33 -07001160 if (ri->task == tk)
bibo,mao99219a32006-10-02 02:17:35 -07001161 recycle_rp_inst(ri, &empty_rp);
bibo,mao62c27be2006-10-02 02:17:33 -07001162 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001163 kretprobe_table_unlock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001164 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
bibo,mao99219a32006-10-02 02:17:35 -07001165 hlist_del(&ri->hlist);
1166 kfree(ri);
1167 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001168}
1169
Hien Nguyenb94cce92005-06-23 00:09:19 -07001170static inline void free_rp_inst(struct kretprobe *rp)
1171{
1172 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001173 struct hlist_node *next;
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001174
Sasha Levinb67bfe02013-02-27 17:06:00 -08001175 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001176 hlist_del(&ri->hlist);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001177 kfree(ri);
1178 }
1179}
1180
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001181static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
1182{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001183 unsigned long flags, hash;
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001184 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001185 struct hlist_node *next;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001186 struct hlist_head *head;
1187
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001188 /* No race here */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001189 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1190 kretprobe_table_lock(hash, &flags);
1191 head = &kretprobe_inst_table[hash];
Sasha Levinb67bfe02013-02-27 17:06:00 -08001192 hlist_for_each_entry_safe(ri, next, head, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001193 if (ri->rp == rp)
1194 ri->rp = NULL;
1195 }
1196 kretprobe_table_unlock(hash, &flags);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001197 }
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001198 free_rp_inst(rp);
1199}
1200
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001201/*
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001202* Add the new probe to ap->list. Fail if this is the
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001203* second jprobe at the address - two jprobes can't coexist
1204*/
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001205static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001206{
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001207 BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001208
1209 if (p->break_handler || p->post_handler)
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001210 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001211
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001212 if (p->break_handler) {
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001213 if (ap->break_handler)
mao, bibo36721652006-06-26 00:25:22 -07001214 return -EEXIST;
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001215 list_add_tail_rcu(&p->list, &ap->list);
1216 ap->break_handler = aggr_break_handler;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001217 } else
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001218 list_add_rcu(&p->list, &ap->list);
1219 if (p->post_handler && !ap->post_handler)
1220 ap->post_handler = aggr_post_handler;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001221
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001222 return 0;
1223}
1224
1225/*
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001226 * Fill in the required fields of the "manager kprobe". Replace the
1227 * earlier kprobe in the hlist with the manager kprobe
1228 */
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001229static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001230{
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001231 /* Copy p's insn slot to ap */
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001232 copy_kprobe(p, ap);
bibo, maoa9ad9652006-07-30 03:03:26 -07001233 flush_insn_slot(ap);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001234 ap->addr = p->addr;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001235 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001236 ap->pre_handler = aggr_pre_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001237 ap->fault_handler = aggr_fault_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001238 /* We don't care the kprobe which has gone. */
1239 if (p->post_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -07001240 ap->post_handler = aggr_post_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001241 if (p->break_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -07001242 ap->break_handler = aggr_break_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001243
1244 INIT_LIST_HEAD(&ap->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001245 INIT_HLIST_NODE(&ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001246
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001247 list_add_rcu(&p->list, &ap->list);
Keshavamurthy Anil Sadad0f32005-12-12 00:37:12 -08001248 hlist_replace_rcu(&p->hlist, &ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001249}
1250
1251/*
1252 * This is the second or subsequent kprobe at the address - handle
1253 * the intricacies
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001254 */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001255static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001256 struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001257{
1258 int ret = 0;
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001259 struct kprobe *ap = orig_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001260
Masami Hiramatsu25764282012-06-05 19:28:26 +09001261 /* For preparing optimization, jump_label_text_reserved() is called */
1262 jump_label_lock();
1263 /*
1264 * Get online CPUs to avoid text_mutex deadlock.with stop machine,
1265 * which is invoked by unoptimize_kprobe() in add_new_kprobe()
1266 */
1267 get_online_cpus();
1268 mutex_lock(&text_mutex);
1269
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001270 if (!kprobe_aggrprobe(orig_p)) {
1271 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1272 ap = alloc_aggr_kprobe(orig_p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001273 if (!ap) {
1274 ret = -ENOMEM;
1275 goto out;
1276 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001277 init_aggr_kprobe(ap, orig_p);
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001278 } else if (kprobe_unused(ap))
Masami Hiramatsu0490cd12010-12-03 18:54:16 +09001279 /* This probe is going to die. Rescue it */
1280 reuse_unused_kprobe(ap);
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001281
1282 if (kprobe_gone(ap)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001283 /*
1284 * Attempting to insert new probe at the same location that
1285 * had a probe in the module vaddr area which already
1286 * freed. So, the instruction slot has already been
1287 * released. We need a new slot for the new probe.
1288 */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001289 ret = arch_prepare_kprobe(ap);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001290 if (ret)
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001291 /*
1292 * Even if fail to allocate new slot, don't need to
1293 * free aggr_probe. It will be used next time, or
1294 * freed by unregister_kprobe.
1295 */
Masami Hiramatsu25764282012-06-05 19:28:26 +09001296 goto out;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001297
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001298 /* Prepare optimized instructions if possible. */
1299 prepare_optimized_kprobe(ap);
1300
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001301 /*
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001302 * Clear gone flag to prevent allocating new slot again, and
1303 * set disabled flag because it is not armed yet.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001304 */
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001305 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1306 | KPROBE_FLAG_DISABLED;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001307 }
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001308
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001309 /* Copy ap's insn slot to p */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001310 copy_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001311 ret = add_new_kprobe(ap, p);
1312
1313out:
1314 mutex_unlock(&text_mutex);
1315 put_online_cpus();
1316 jump_label_unlock();
1317
1318 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1319 ap->flags &= ~KPROBE_FLAG_DISABLED;
1320 if (!kprobes_all_disarmed)
1321 /* Arm the breakpoint again. */
1322 arm_kprobe(ap);
1323 }
1324 return ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001325}
1326
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001327static int __kprobes in_kprobes_functions(unsigned long addr)
1328{
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001329 struct kprobe_blackpoint *kb;
1330
Christoph Hellwig6f716ac2007-05-08 00:34:13 -07001331 if (addr >= (unsigned long)__kprobes_text_start &&
1332 addr < (unsigned long)__kprobes_text_end)
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001333 return -EINVAL;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001334 /*
1335 * If there exists a kprobe_blacklist, verify and
1336 * fail any probe registration in the prohibited area
1337 */
1338 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1339 if (kb->start_addr) {
1340 if (addr >= kb->start_addr &&
1341 addr < (kb->start_addr + kb->range))
1342 return -EINVAL;
1343 }
1344 }
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001345 return 0;
1346}
1347
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001348/*
1349 * If we have a symbol_name argument, look it up and add the offset field
1350 * to it. This way, we can specify a relative address to a symbol.
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001351 * This returns encoded errors if it fails to look up symbol or invalid
1352 * combination of parameters.
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001353 */
1354static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1355{
1356 kprobe_opcode_t *addr = p->addr;
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001357
1358 if ((p->symbol_name && p->addr) ||
1359 (!p->symbol_name && !p->addr))
1360 goto invalid;
1361
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001362 if (p->symbol_name) {
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001363 kprobe_lookup_name(p->symbol_name, addr);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001364 if (!addr)
1365 return ERR_PTR(-ENOENT);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001366 }
1367
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001368 addr = (kprobe_opcode_t *)(((char *)addr) + p->offset);
1369 if (addr)
1370 return addr;
1371
1372invalid:
1373 return ERR_PTR(-EINVAL);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001374}
1375
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301376/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1377static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1378{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001379 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301380
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001381 ap = get_kprobe(p->addr);
1382 if (unlikely(!ap))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301383 return NULL;
1384
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001385 if (p != ap) {
1386 list_for_each_entry_rcu(list_p, &ap->list, list)
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301387 if (list_p == p)
1388 /* kprobe p is a valid probe */
1389 goto valid;
1390 return NULL;
1391 }
1392valid:
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001393 return ap;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301394}
1395
1396/* Return error if the kprobe is being re-registered */
1397static inline int check_kprobe_rereg(struct kprobe *p)
1398{
1399 int ret = 0;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301400
1401 mutex_lock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001402 if (__get_valid_kprobe(p))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301403 ret = -EINVAL;
1404 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001405
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301406 return ret;
1407}
1408
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001409static __kprobes int check_kprobe_address_safe(struct kprobe *p,
1410 struct module **probed_mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411{
1412 int ret = 0;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001413 unsigned long ftrace_addr;
1414
1415 /*
1416 * If the address is located on a ftrace nop, set the
1417 * breakpoint to the following instruction.
1418 */
1419 ftrace_addr = ftrace_location((unsigned long)p->addr);
1420 if (ftrace_addr) {
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001421#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001422 /* Given address is not on the instruction boundary */
1423 if ((unsigned long)p->addr != ftrace_addr)
1424 return -EILSEQ;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001425 p->flags |= KPROBE_FLAG_FTRACE;
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001426#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001427 return -EINVAL;
1428#endif
1429 }
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001430
1431 jump_label_lock();
1432 preempt_disable();
1433
1434 /* Ensure it is not in reserved area nor out of text */
1435 if (!kernel_text_address((unsigned long) p->addr) ||
1436 in_kprobes_functions((unsigned long) p->addr) ||
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001437 jump_label_text_reserved(p->addr, p->addr)) {
1438 ret = -EINVAL;
1439 goto out;
1440 }
1441
1442 /* Check if are we probing a module */
1443 *probed_mod = __module_text_address((unsigned long) p->addr);
1444 if (*probed_mod) {
1445 /*
1446 * We must hold a refcount of the probed module while updating
1447 * its code to prohibit unexpected unloading.
1448 */
1449 if (unlikely(!try_module_get(*probed_mod))) {
1450 ret = -ENOENT;
1451 goto out;
1452 }
1453
1454 /*
1455 * If the module freed .init.text, we couldn't insert
1456 * kprobes in there.
1457 */
1458 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1459 (*probed_mod)->state != MODULE_STATE_COMING) {
1460 module_put(*probed_mod);
1461 *probed_mod = NULL;
1462 ret = -ENOENT;
1463 }
1464 }
1465out:
1466 preempt_enable();
1467 jump_label_unlock();
1468
1469 return ret;
1470}
1471
1472int __kprobes register_kprobe(struct kprobe *p)
1473{
1474 int ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001475 struct kprobe *old_p;
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001476 struct module *probed_mod;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001477 kprobe_opcode_t *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001479 /* Adjust probe address from symbol */
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001480 addr = kprobe_addr(p);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001481 if (IS_ERR(addr))
1482 return PTR_ERR(addr);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001483 p->addr = addr;
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -07001484
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301485 ret = check_kprobe_rereg(p);
1486 if (ret)
1487 return ret;
1488
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001489 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1490 p->flags &= KPROBE_FLAG_DISABLED;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001491 p->nmissed = 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001492 INIT_LIST_HEAD(&p->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001493
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001494 ret = check_kprobe_address_safe(p, &probed_mod);
1495 if (ret)
1496 return ret;
1497
1498 mutex_lock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001499
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001500 old_p = get_kprobe(p->addr);
1501 if (old_p) {
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001502 /* Since this may unoptimize old_p, locking text_mutex. */
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001503 ret = register_aggr_kprobe(old_p, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 goto out;
1505 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
Masami Hiramatsu25764282012-06-05 19:28:26 +09001507 mutex_lock(&text_mutex); /* Avoiding text modification */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001508 ret = prepare_kprobe(p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001509 mutex_unlock(&text_mutex);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -07001510 if (ret)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001511 goto out;
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001512
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001513 INIT_HLIST_NODE(&p->hlist);
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001514 hlist_add_head_rcu(&p->hlist,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1516
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001517 if (!kprobes_all_disarmed && !kprobe_disabled(p))
Masami Hiramatsu25764282012-06-05 19:28:26 +09001518 arm_kprobe(p);
Christoph Hellwig74a0b572007-10-16 01:24:07 -07001519
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001520 /* Try to optimize kprobe */
1521 try_to_optimize_kprobe(p);
1522
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523out:
Ingo Molnar7a7d1cf2006-03-23 03:00:35 -08001524 mutex_unlock(&kprobe_mutex);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001525
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001526 if (probed_mod)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001527 module_put(probed_mod);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001528
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 return ret;
1530}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001531EXPORT_SYMBOL_GPL(register_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001533/* Check if all probes on the aggrprobe are disabled */
1534static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)
1535{
1536 struct kprobe *kp;
1537
1538 list_for_each_entry_rcu(kp, &ap->list, list)
1539 if (!kprobe_disabled(kp))
1540 /*
1541 * There is an active probe on the list.
1542 * We can't disable this ap.
1543 */
1544 return 0;
1545
1546 return 1;
1547}
1548
1549/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1550static struct kprobe *__kprobes __disable_kprobe(struct kprobe *p)
1551{
1552 struct kprobe *orig_p;
1553
1554 /* Get an original kprobe for return */
1555 orig_p = __get_valid_kprobe(p);
1556 if (unlikely(orig_p == NULL))
1557 return NULL;
1558
1559 if (!kprobe_disabled(p)) {
1560 /* Disable probe if it is a child probe */
1561 if (p != orig_p)
1562 p->flags |= KPROBE_FLAG_DISABLED;
1563
1564 /* Try to disarm and disable this/parent probe */
1565 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001566 disarm_kprobe(orig_p, true);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001567 orig_p->flags |= KPROBE_FLAG_DISABLED;
1568 }
1569 }
1570
1571 return orig_p;
1572}
1573
Masami Hiramatsu98616682008-04-28 02:14:28 -07001574/*
1575 * Unregister a kprobe without a scheduler synchronization.
1576 */
1577static int __kprobes __unregister_kprobe_top(struct kprobe *p)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001578{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001579 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001580
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001581 /* Disable kprobe. This will disarm it if needed. */
1582 ap = __disable_kprobe(p);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001583 if (ap == NULL)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001584 return -EINVAL;
1585
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001586 if (ap == p)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001587 /*
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001588 * This probe is an independent(and non-optimized) kprobe
1589 * (not an aggrprobe). Remove from the hash list.
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001590 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001591 goto disarmed;
1592
1593 /* Following process expects this probe is an aggrprobe */
1594 WARN_ON(!kprobe_aggrprobe(ap));
1595
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001596 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1597 /*
1598 * !disarmed could be happen if the probe is under delayed
1599 * unoptimizing.
1600 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001601 goto disarmed;
1602 else {
1603 /* If disabling probe has special handlers, update aggrprobe */
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001604 if (p->break_handler && !kprobe_gone(p))
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001605 ap->break_handler = NULL;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001606 if (p->post_handler && !kprobe_gone(p)) {
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001607 list_for_each_entry_rcu(list_p, &ap->list, list) {
Masami Hiramatsu98616682008-04-28 02:14:28 -07001608 if ((list_p != p) && (list_p->post_handler))
1609 goto noclean;
1610 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001611 ap->post_handler = NULL;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001612 }
1613noclean:
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001614 /*
1615 * Remove from the aggrprobe: this path will do nothing in
1616 * __unregister_kprobe_bottom().
1617 */
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001618 list_del_rcu(&p->list);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001619 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1620 /*
1621 * Try to optimize this probe again, because post
1622 * handler may have been changed.
1623 */
1624 optimize_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001625 }
Masami Hiramatsu98616682008-04-28 02:14:28 -07001626 return 0;
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001627
1628disarmed:
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001629 BUG_ON(!kprobe_disarmed(ap));
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001630 hlist_del_rcu(&ap->hlist);
1631 return 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001632}
Mao, Bibob3e55c72005-12-12 00:37:00 -08001633
Masami Hiramatsu98616682008-04-28 02:14:28 -07001634static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1635{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001636 struct kprobe *ap;
Mao, Bibob3e55c72005-12-12 00:37:00 -08001637
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001638 if (list_empty(&p->list))
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001639 /* This is an independent kprobe */
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -08001640 arch_remove_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001641 else if (list_is_singular(&p->list)) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001642 /* This is the last child of an aggrprobe */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001643 ap = list_entry(p->list.next, struct kprobe, list);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001644 list_del(&p->list);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001645 free_aggr_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001646 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001647 /* Otherwise, do nothing. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648}
1649
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001650int __kprobes register_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001651{
1652 int i, ret = 0;
1653
1654 if (num <= 0)
1655 return -EINVAL;
1656 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001657 ret = register_kprobe(kps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001658 if (ret < 0) {
1659 if (i > 0)
1660 unregister_kprobes(kps, i);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001661 break;
1662 }
1663 }
1664 return ret;
1665}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001666EXPORT_SYMBOL_GPL(register_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001667
Masami Hiramatsu98616682008-04-28 02:14:28 -07001668void __kprobes unregister_kprobe(struct kprobe *p)
1669{
1670 unregister_kprobes(&p, 1);
1671}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001672EXPORT_SYMBOL_GPL(unregister_kprobe);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001673
Masami Hiramatsu98616682008-04-28 02:14:28 -07001674void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1675{
1676 int i;
1677
1678 if (num <= 0)
1679 return;
1680 mutex_lock(&kprobe_mutex);
1681 for (i = 0; i < num; i++)
1682 if (__unregister_kprobe_top(kps[i]) < 0)
1683 kps[i]->addr = NULL;
1684 mutex_unlock(&kprobe_mutex);
1685
1686 synchronize_sched();
1687 for (i = 0; i < num; i++)
1688 if (kps[i]->addr)
1689 __unregister_kprobe_bottom(kps[i]);
1690}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001691EXPORT_SYMBOL_GPL(unregister_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001692
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693static struct notifier_block kprobe_exceptions_nb = {
1694 .notifier_call = kprobe_exceptions_notify,
Anil S Keshavamurthy3d5631e2006-06-26 00:25:28 -07001695 .priority = 0x7fffffff /* we need to be notified first */
1696};
1697
Michael Ellerman3d7e3382007-07-19 01:48:11 -07001698unsigned long __weak arch_deref_entry_point(void *entry)
1699{
1700 return (unsigned long)entry;
1701}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001703int __kprobes register_jprobes(struct jprobe **jps, int num)
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001704{
1705 struct jprobe *jp;
1706 int ret = 0, i;
1707
1708 if (num <= 0)
1709 return -EINVAL;
1710 for (i = 0; i < num; i++) {
Namhyung Kim05662bd2010-09-15 10:04:27 +09001711 unsigned long addr, offset;
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001712 jp = jps[i];
1713 addr = arch_deref_entry_point(jp->entry);
1714
Namhyung Kim05662bd2010-09-15 10:04:27 +09001715 /* Verify probepoint is a function entry point */
1716 if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1717 offset == 0) {
1718 jp->kp.pre_handler = setjmp_pre_handler;
1719 jp->kp.break_handler = longjmp_break_handler;
1720 ret = register_kprobe(&jp->kp);
1721 } else
1722 ret = -EINVAL;
Namhyung Kimedbaadb2010-09-15 10:04:26 +09001723
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001724 if (ret < 0) {
1725 if (i > 0)
1726 unregister_jprobes(jps, i);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001727 break;
1728 }
1729 }
1730 return ret;
1731}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001732EXPORT_SYMBOL_GPL(register_jprobes);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001733
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001734int __kprobes register_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735{
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001736 return register_jprobes(&jp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001738EXPORT_SYMBOL_GPL(register_jprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001740void __kprobes unregister_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001741{
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001742 unregister_jprobes(&jp, 1);
1743}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001744EXPORT_SYMBOL_GPL(unregister_jprobe);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001745
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001746void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1747{
1748 int i;
1749
1750 if (num <= 0)
1751 return;
1752 mutex_lock(&kprobe_mutex);
1753 for (i = 0; i < num; i++)
1754 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1755 jps[i]->kp.addr = NULL;
1756 mutex_unlock(&kprobe_mutex);
1757
1758 synchronize_sched();
1759 for (i = 0; i < num; i++) {
1760 if (jps[i]->kp.addr)
1761 __unregister_kprobe_bottom(&jps[i]->kp);
1762 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001764EXPORT_SYMBOL_GPL(unregister_jprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001766#ifdef CONFIG_KRETPROBES
Adrian Bunke65cefe2006-02-03 03:03:42 -08001767/*
1768 * This kprobe pre_handler is registered with every kretprobe. When probe
1769 * hits it will set up the return probe.
1770 */
1771static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1772 struct pt_regs *regs)
1773{
1774 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001775 unsigned long hash, flags = 0;
1776 struct kretprobe_instance *ri;
Adrian Bunke65cefe2006-02-03 03:03:42 -08001777
1778 /*TODO: consider to only swap the RA after the last pre_handler fired */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001779 hash = hash_ptr(current, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001780 raw_spin_lock_irqsave(&rp->lock, flags);
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001781 if (!hlist_empty(&rp->free_instances)) {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001782 ri = hlist_entry(rp->free_instances.first,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001783 struct kretprobe_instance, hlist);
1784 hlist_del(&ri->hlist);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001785 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001786
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001787 ri->rp = rp;
1788 ri->task = current;
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001789
Jiang Liu55ca6142012-02-03 15:37:16 -08001790 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1791 raw_spin_lock_irqsave(&rp->lock, flags);
1792 hlist_add_head(&ri->hlist, &rp->free_instances);
1793 raw_spin_unlock_irqrestore(&rp->lock, flags);
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001794 return 0;
Jiang Liu55ca6142012-02-03 15:37:16 -08001795 }
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001796
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001797 arch_prepare_kretprobe(ri, regs);
1798
1799 /* XXX(hch): why is there no hlist_move_head? */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001800 INIT_HLIST_NODE(&ri->hlist);
1801 kretprobe_table_lock(hash, &flags);
1802 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1803 kretprobe_table_unlock(hash, &flags);
1804 } else {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001805 rp->nmissed++;
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001806 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001807 }
Adrian Bunke65cefe2006-02-03 03:03:42 -08001808 return 0;
1809}
1810
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001811int __kprobes register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001812{
1813 int ret = 0;
1814 struct kretprobe_instance *inst;
1815 int i;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001816 void *addr;
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001817
1818 if (kretprobe_blacklist_size) {
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001819 addr = kprobe_addr(&rp->kp);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001820 if (IS_ERR(addr))
1821 return PTR_ERR(addr);
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001822
1823 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1824 if (kretprobe_blacklist[i].addr == addr)
1825 return -EINVAL;
1826 }
1827 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001828
1829 rp->kp.pre_handler = pre_handler_kretprobe;
Ananth N Mavinakayanahalli7522a842006-04-20 02:43:11 -07001830 rp->kp.post_handler = NULL;
1831 rp->kp.fault_handler = NULL;
1832 rp->kp.break_handler = NULL;
Hien Nguyenb94cce92005-06-23 00:09:19 -07001833
1834 /* Pre-allocate memory for max kretprobe instances */
1835 if (rp->maxactive <= 0) {
1836#ifdef CONFIG_PREEMPT
Heiko Carstensc2ef6662009-12-21 13:02:24 +01001837 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
Hien Nguyenb94cce92005-06-23 00:09:19 -07001838#else
Ananth N Mavinakayanahalli4dae5602009-10-30 19:23:10 +05301839 rp->maxactive = num_possible_cpus();
Hien Nguyenb94cce92005-06-23 00:09:19 -07001840#endif
1841 }
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001842 raw_spin_lock_init(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001843 INIT_HLIST_HEAD(&rp->free_instances);
1844 for (i = 0; i < rp->maxactive; i++) {
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001845 inst = kmalloc(sizeof(struct kretprobe_instance) +
1846 rp->data_size, GFP_KERNEL);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001847 if (inst == NULL) {
1848 free_rp_inst(rp);
1849 return -ENOMEM;
1850 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001851 INIT_HLIST_NODE(&inst->hlist);
1852 hlist_add_head(&inst->hlist, &rp->free_instances);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001853 }
1854
1855 rp->nmissed = 0;
1856 /* Establish function entry probe point */
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001857 ret = register_kprobe(&rp->kp);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001858 if (ret != 0)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001859 free_rp_inst(rp);
1860 return ret;
1861}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001862EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001863
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001864int __kprobes register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001865{
1866 int ret = 0, i;
1867
1868 if (num <= 0)
1869 return -EINVAL;
1870 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001871 ret = register_kretprobe(rps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001872 if (ret < 0) {
1873 if (i > 0)
1874 unregister_kretprobes(rps, i);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001875 break;
1876 }
1877 }
1878 return ret;
1879}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001880EXPORT_SYMBOL_GPL(register_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001881
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001882void __kprobes unregister_kretprobe(struct kretprobe *rp)
1883{
1884 unregister_kretprobes(&rp, 1);
1885}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001886EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001887
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001888void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1889{
1890 int i;
1891
1892 if (num <= 0)
1893 return;
1894 mutex_lock(&kprobe_mutex);
1895 for (i = 0; i < num; i++)
1896 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1897 rps[i]->kp.addr = NULL;
1898 mutex_unlock(&kprobe_mutex);
1899
1900 synchronize_sched();
1901 for (i = 0; i < num; i++) {
1902 if (rps[i]->kp.addr) {
1903 __unregister_kprobe_bottom(&rps[i]->kp);
1904 cleanup_rp_inst(rps[i]);
1905 }
1906 }
1907}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001908EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001909
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001910#else /* CONFIG_KRETPROBES */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001911int __kprobes register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001912{
1913 return -ENOSYS;
1914}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001915EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001916
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001917int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1918{
1919 return -ENOSYS;
1920}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001921EXPORT_SYMBOL_GPL(register_kretprobes);
1922
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001923void __kprobes unregister_kretprobe(struct kretprobe *rp)
1924{
1925}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001926EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001927
1928void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1929{
1930}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001931EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001932
Srinivasa Ds346fd592007-02-20 13:57:54 -08001933static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1934 struct pt_regs *regs)
1935{
1936 return 0;
1937}
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001938
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001939#endif /* CONFIG_KRETPROBES */
Hien Nguyenb94cce92005-06-23 00:09:19 -07001940
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001941/* Set the kprobe gone and remove its instruction buffer. */
1942static void __kprobes kill_kprobe(struct kprobe *p)
1943{
1944 struct kprobe *kp;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001945
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001946 p->flags |= KPROBE_FLAG_GONE;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001947 if (kprobe_aggrprobe(p)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001948 /*
1949 * If this is an aggr_kprobe, we have to list all the
1950 * chained probes and mark them GONE.
1951 */
1952 list_for_each_entry_rcu(kp, &p->list, list)
1953 kp->flags |= KPROBE_FLAG_GONE;
1954 p->post_handler = NULL;
1955 p->break_handler = NULL;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001956 kill_optimized_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001957 }
1958 /*
1959 * Here, we can remove insn_slot safely, because no thread calls
1960 * the original probed function (which will be freed soon) any more.
1961 */
1962 arch_remove_kprobe(p);
1963}
1964
Masami Hiramatsuc0614822010-04-27 18:33:12 -04001965/* Disable one kprobe */
1966int __kprobes disable_kprobe(struct kprobe *kp)
1967{
1968 int ret = 0;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04001969
1970 mutex_lock(&kprobe_mutex);
1971
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001972 /* Disable this kprobe */
1973 if (__disable_kprobe(kp) == NULL)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04001974 ret = -EINVAL;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04001975
Masami Hiramatsuc0614822010-04-27 18:33:12 -04001976 mutex_unlock(&kprobe_mutex);
1977 return ret;
1978}
1979EXPORT_SYMBOL_GPL(disable_kprobe);
1980
1981/* Enable one kprobe */
1982int __kprobes enable_kprobe(struct kprobe *kp)
1983{
1984 int ret = 0;
1985 struct kprobe *p;
1986
1987 mutex_lock(&kprobe_mutex);
1988
1989 /* Check whether specified probe is valid. */
1990 p = __get_valid_kprobe(kp);
1991 if (unlikely(p == NULL)) {
1992 ret = -EINVAL;
1993 goto out;
1994 }
1995
1996 if (kprobe_gone(kp)) {
1997 /* This kprobe has gone, we couldn't enable it. */
1998 ret = -EINVAL;
1999 goto out;
2000 }
2001
2002 if (p != kp)
2003 kp->flags &= ~KPROBE_FLAG_DISABLED;
2004
2005 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2006 p->flags &= ~KPROBE_FLAG_DISABLED;
2007 arm_kprobe(p);
2008 }
2009out:
2010 mutex_unlock(&kprobe_mutex);
2011 return ret;
2012}
2013EXPORT_SYMBOL_GPL(enable_kprobe);
2014
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002015void __kprobes dump_kprobe(struct kprobe *kp)
2016{
2017 printk(KERN_WARNING "Dumping kprobe:\n");
2018 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
2019 kp->symbol_name, kp->addr, kp->offset);
2020}
2021
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002022/* Module notifier call back, checking kprobes on the module */
2023static int __kprobes kprobes_module_callback(struct notifier_block *nb,
2024 unsigned long val, void *data)
2025{
2026 struct module *mod = data;
2027 struct hlist_head *head;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002028 struct kprobe *p;
2029 unsigned int i;
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002030 int checkcore = (val == MODULE_STATE_GOING);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002031
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002032 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002033 return NOTIFY_DONE;
2034
2035 /*
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002036 * When MODULE_STATE_GOING was notified, both of module .text and
2037 * .init.text sections would be freed. When MODULE_STATE_LIVE was
2038 * notified, only .init.text section would be freed. We need to
2039 * disable kprobes which have been inserted in the sections.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002040 */
2041 mutex_lock(&kprobe_mutex);
2042 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2043 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002044 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002045 if (within_module_init((unsigned long)p->addr, mod) ||
2046 (checkcore &&
2047 within_module_core((unsigned long)p->addr, mod))) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002048 /*
2049 * The vaddr this probe is installed will soon
2050 * be vfreed buy not synced to disk. Hence,
2051 * disarming the breakpoint isn't needed.
2052 */
2053 kill_kprobe(p);
2054 }
2055 }
2056 mutex_unlock(&kprobe_mutex);
2057 return NOTIFY_DONE;
2058}
2059
2060static struct notifier_block kprobe_module_nb = {
2061 .notifier_call = kprobes_module_callback,
2062 .priority = 0
2063};
2064
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065static int __init init_kprobes(void)
2066{
2067 int i, err = 0;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07002068 unsigned long offset = 0, size = 0;
2069 char *modname, namebuf[128];
2070 const char *symbol_name;
2071 void *addr;
2072 struct kprobe_blackpoint *kb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002073
2074 /* FIXME allocate the probe table, currently defined statically */
2075 /* initialize all list heads */
Hien Nguyenb94cce92005-06-23 00:09:19 -07002076 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 INIT_HLIST_HEAD(&kprobe_table[i]);
Hien Nguyenb94cce92005-06-23 00:09:19 -07002078 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02002079 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
Hien Nguyenb94cce92005-06-23 00:09:19 -07002080 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07002082 /*
2083 * Lookup and populate the kprobe_blacklist.
2084 *
2085 * Unlike the kretprobe blacklist, we'll need to determine
2086 * the range of addresses that belong to the said functions,
2087 * since a kprobe need not necessarily be at the beginning
2088 * of a function.
2089 */
2090 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
2091 kprobe_lookup_name(kb->name, addr);
2092 if (!addr)
2093 continue;
2094
2095 kb->start_addr = (unsigned long)addr;
2096 symbol_name = kallsyms_lookup(kb->start_addr,
2097 &size, &offset, &modname, namebuf);
2098 if (!symbol_name)
2099 kb->range = 0;
2100 else
2101 kb->range = size;
2102 }
2103
Masami Hiramatsuf438d912007-10-16 01:27:49 -07002104 if (kretprobe_blacklist_size) {
2105 /* lookup the function address from its name */
2106 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2107 kprobe_lookup_name(kretprobe_blacklist[i].name,
2108 kretprobe_blacklist[i].addr);
2109 if (!kretprobe_blacklist[i].addr)
2110 printk("kretprobe: lookup failed: %s\n",
2111 kretprobe_blacklist[i].name);
2112 }
2113 }
2114
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002115#if defined(CONFIG_OPTPROBES)
2116#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002117 /* Init kprobe_optinsn_slots */
2118 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2119#endif
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002120 /* By default, kprobes can be optimized */
2121 kprobes_allow_optimization = true;
2122#endif
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002123
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002124 /* By default, kprobes are armed */
2125 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002126
Rusty Lynch67729262005-07-05 18:54:50 -07002127 err = arch_init_kprobes();
Rusty Lynch802eae72005-06-27 15:17:08 -07002128 if (!err)
2129 err = register_die_notifier(&kprobe_exceptions_nb);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002130 if (!err)
2131 err = register_module_notifier(&kprobe_module_nb);
2132
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07002133 kprobes_initialized = (err == 0);
Rusty Lynch802eae72005-06-27 15:17:08 -07002134
Ananth N Mavinakayanahalli8c1c9352008-01-30 13:32:53 +01002135 if (!err)
2136 init_test_probes();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137 return err;
2138}
2139
Srinivasa Ds346fd592007-02-20 13:57:54 -08002140#ifdef CONFIG_DEBUG_FS
2141static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002142 const char *sym, int offset, char *modname, struct kprobe *pp)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002143{
2144 char *kprobe_type;
2145
2146 if (p->pre_handler == pre_handler_kretprobe)
2147 kprobe_type = "r";
2148 else if (p->pre_handler == setjmp_pre_handler)
2149 kprobe_type = "j";
2150 else
2151 kprobe_type = "k";
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002152
Srinivasa Ds346fd592007-02-20 13:57:54 -08002153 if (sym)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002154 seq_printf(pi, "%p %s %s+0x%x %s ",
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002155 p->addr, kprobe_type, sym, offset,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002156 (modname ? modname : " "));
Srinivasa Ds346fd592007-02-20 13:57:54 -08002157 else
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002158 seq_printf(pi, "%p %s %p ",
2159 p->addr, kprobe_type, p->addr);
2160
2161 if (!pp)
2162 pp = p;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002163 seq_printf(pi, "%s%s%s%s\n",
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002164 (kprobe_gone(p) ? "[GONE]" : ""),
2165 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002166 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2167 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
Srinivasa Ds346fd592007-02-20 13:57:54 -08002168}
2169
2170static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2171{
2172 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2173}
2174
2175static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2176{
2177 (*pos)++;
2178 if (*pos >= KPROBE_TABLE_SIZE)
2179 return NULL;
2180 return pos;
2181}
2182
2183static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
2184{
2185 /* Nothing to do */
2186}
2187
2188static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
2189{
2190 struct hlist_head *head;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002191 struct kprobe *p, *kp;
2192 const char *sym = NULL;
2193 unsigned int i = *(loff_t *) v;
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002194 unsigned long offset = 0;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002195 char *modname, namebuf[128];
2196
2197 head = &kprobe_table[i];
2198 preempt_disable();
Sasha Levinb67bfe02013-02-27 17:06:00 -08002199 hlist_for_each_entry_rcu(p, head, hlist) {
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002200 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002201 &offset, &modname, namebuf);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002202 if (kprobe_aggrprobe(p)) {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002203 list_for_each_entry_rcu(kp, &p->list, list)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002204 report_probe(pi, kp, sym, offset, modname, p);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002205 } else
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002206 report_probe(pi, p, sym, offset, modname, NULL);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002207 }
2208 preempt_enable();
2209 return 0;
2210}
2211
James Morris88e9d342009-09-22 16:43:43 -07002212static const struct seq_operations kprobes_seq_ops = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002213 .start = kprobe_seq_start,
2214 .next = kprobe_seq_next,
2215 .stop = kprobe_seq_stop,
2216 .show = show_kprobe_addr
2217};
2218
2219static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
2220{
2221 return seq_open(filp, &kprobes_seq_ops);
2222}
2223
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002224static const struct file_operations debugfs_kprobes_operations = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002225 .open = kprobes_open,
2226 .read = seq_read,
2227 .llseek = seq_lseek,
2228 .release = seq_release,
2229};
2230
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002231static void __kprobes arm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002232{
2233 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002234 struct kprobe *p;
2235 unsigned int i;
2236
2237 mutex_lock(&kprobe_mutex);
2238
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002239 /* If kprobes are armed, just return */
2240 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002241 goto already_enabled;
2242
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002243 /* Arming kprobes doesn't optimize kprobe itself */
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002244 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2245 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002246 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002247 if (!kprobe_disabled(p))
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002248 arm_kprobe(p);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002249 }
2250
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002251 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002252 printk(KERN_INFO "Kprobes globally enabled\n");
2253
2254already_enabled:
2255 mutex_unlock(&kprobe_mutex);
2256 return;
2257}
2258
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002259static void __kprobes disarm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002260{
2261 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002262 struct kprobe *p;
2263 unsigned int i;
2264
2265 mutex_lock(&kprobe_mutex);
2266
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002267 /* If kprobes are already disarmed, just return */
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002268 if (kprobes_all_disarmed) {
2269 mutex_unlock(&kprobe_mutex);
2270 return;
2271 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002272
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002273 kprobes_all_disarmed = true;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002274 printk(KERN_INFO "Kprobes globally disabled\n");
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002275
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002276 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2277 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002278 hlist_for_each_entry_rcu(p, head, hlist) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002279 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002280 disarm_kprobe(p, false);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002281 }
2282 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002283 mutex_unlock(&kprobe_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002284
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002285 /* Wait for disarming all kprobes by optimizer */
2286 wait_for_kprobe_optimizer();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002287}
2288
2289/*
2290 * XXX: The debugfs bool file interface doesn't allow for callbacks
2291 * when the bool state is switched. We can reuse that facility when
2292 * available
2293 */
2294static ssize_t read_enabled_file_bool(struct file *file,
2295 char __user *user_buf, size_t count, loff_t *ppos)
2296{
2297 char buf[3];
2298
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002299 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002300 buf[0] = '1';
2301 else
2302 buf[0] = '0';
2303 buf[1] = '\n';
2304 buf[2] = 0x00;
2305 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2306}
2307
2308static ssize_t write_enabled_file_bool(struct file *file,
2309 const char __user *user_buf, size_t count, loff_t *ppos)
2310{
2311 char buf[32];
Stephen Boydefeb1562012-01-12 17:17:11 -08002312 size_t buf_size;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002313
2314 buf_size = min(count, (sizeof(buf)-1));
2315 if (copy_from_user(buf, user_buf, buf_size))
2316 return -EFAULT;
2317
2318 switch (buf[0]) {
2319 case 'y':
2320 case 'Y':
2321 case '1':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002322 arm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002323 break;
2324 case 'n':
2325 case 'N':
2326 case '0':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002327 disarm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002328 break;
2329 }
2330
2331 return count;
2332}
2333
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002334static const struct file_operations fops_kp = {
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002335 .read = read_enabled_file_bool,
2336 .write = write_enabled_file_bool,
Arnd Bergmann6038f372010-08-15 18:52:59 +02002337 .llseek = default_llseek,
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002338};
2339
Srinivasa Ds346fd592007-02-20 13:57:54 -08002340static int __kprobes debugfs_kprobe_init(void)
2341{
2342 struct dentry *dir, *file;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002343 unsigned int value = 1;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002344
2345 dir = debugfs_create_dir("kprobes", NULL);
2346 if (!dir)
2347 return -ENOMEM;
2348
Randy Dunlape3869792007-05-08 00:27:01 -07002349 file = debugfs_create_file("list", 0444, dir, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002350 &debugfs_kprobes_operations);
2351 if (!file) {
2352 debugfs_remove(dir);
2353 return -ENOMEM;
2354 }
2355
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002356 file = debugfs_create_file("enabled", 0600, dir,
2357 &value, &fops_kp);
2358 if (!file) {
2359 debugfs_remove(dir);
2360 return -ENOMEM;
2361 }
2362
Srinivasa Ds346fd592007-02-20 13:57:54 -08002363 return 0;
2364}
2365
2366late_initcall(debugfs_kprobe_init);
2367#endif /* CONFIG_DEBUG_FS */
2368
2369module_init(init_kprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002371/* defined in arch/.../kernel/kprobes.c */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372EXPORT_SYMBOL_GPL(jprobe_return);