| /* |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License as published by |
| * the Free Software Foundation; either version 2 of the License, or |
| * (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software |
| * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| * |
| * Copyright (C) 2007 Alan Stern |
| * Copyright (C) 2009 IBM Corporation |
| * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com> |
| */ |
| |
| /* |
| * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, |
| * using the CPU's debug registers. |
| */ |
| |
| #include <linux/perf_event.h> |
| #include <linux/hw_breakpoint.h> |
| #include <linux/irqflags.h> |
| #include <linux/notifier.h> |
| #include <linux/kallsyms.h> |
| #include <linux/kprobes.h> |
| #include <linux/percpu.h> |
| #include <linux/kdebug.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/sched.h> |
| #include <linux/init.h> |
| #include <linux/smp.h> |
| |
| #include <asm/hw_breakpoint.h> |
| #include <asm/processor.h> |
| #include <asm/debugreg.h> |
| |
| /* Per cpu debug control register value */ |
| DEFINE_PER_CPU(unsigned long, dr7); |
| |
| /* Per cpu debug address registers values */ |
| static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]); |
| |
| /* |
| * Stores the breakpoints currently in use on each breakpoint address |
| * register for each cpus |
| */ |
| static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]); |
| |
| |
| /* |
| * Encode the length, type, Exact, and Enable bits for a particular breakpoint |
| * as stored in debug register 7. |
| */ |
| unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type) |
| { |
| unsigned long bp_info; |
| |
| bp_info = (len | type) & 0xf; |
| bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE); |
| bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)) | |
| DR_GLOBAL_SLOWDOWN; |
| return bp_info; |
| } |
| |
| /* |
| * Decode the length and type bits for a particular breakpoint as |
| * stored in debug register 7. Return the "enabled" status. |
| */ |
| int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type) |
| { |
| int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE); |
| |
| *len = (bp_info & 0xc) | 0x40; |
| *type = (bp_info & 0x3) | 0x80; |
| |
| return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3; |
| } |
| |
| /* |
| * Install a perf counter breakpoint. |
| * |
| * We seek a free debug address register and use it for this |
| * breakpoint. Eventually we enable it in the debug control register. |
| * |
| * Atomic: we hold the counter->ctx->lock and we only handle variables |
| * and registers local to this cpu. |
| */ |
| int arch_install_hw_breakpoint(struct perf_event *bp) |
| { |
| struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
| unsigned long *dr7; |
| int i; |
| |
| for (i = 0; i < HBP_NUM; i++) { |
| struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]); |
| |
| if (!*slot) { |
| *slot = bp; |
| break; |
| } |
| } |
| |
| if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) |
| return -EBUSY; |
| |
| set_debugreg(info->address, i); |
| __get_cpu_var(cpu_debugreg[i]) = info->address; |
| |
| dr7 = &__get_cpu_var(dr7); |
| *dr7 |= encode_dr7(i, info->len, info->type); |
| |
| set_debugreg(*dr7, 7); |
| |
| return 0; |
| } |
| |
| /* |
| * Uninstall the breakpoint contained in the given counter. |
| * |
| * First we search the debug address register it uses and then we disable |
| * it. |
| * |
| * Atomic: we hold the counter->ctx->lock and we only handle variables |
| * and registers local to this cpu. |
| */ |
| void arch_uninstall_hw_breakpoint(struct perf_event *bp) |
| { |
| struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
| unsigned long *dr7; |
| int i; |
| |
| for (i = 0; i < HBP_NUM; i++) { |
| struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]); |
| |
| if (*slot == bp) { |
| *slot = NULL; |
| break; |
| } |
| } |
| |
| if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) |
| return; |
| |
| dr7 = &__get_cpu_var(dr7); |
| *dr7 &= ~encode_dr7(i, info->len, info->type); |
| |
| set_debugreg(*dr7, 7); |
| } |
| |
| static int get_hbp_len(u8 hbp_len) |
| { |
| unsigned int len_in_bytes = 0; |
| |
| switch (hbp_len) { |
| case X86_BREAKPOINT_LEN_1: |
| len_in_bytes = 1; |
| break; |
| case X86_BREAKPOINT_LEN_2: |
| len_in_bytes = 2; |
| break; |
| case X86_BREAKPOINT_LEN_4: |
| len_in_bytes = 4; |
| break; |
| #ifdef CONFIG_X86_64 |
| case X86_BREAKPOINT_LEN_8: |
| len_in_bytes = 8; |
| break; |
| #endif |
| } |
| return len_in_bytes; |
| } |
| |
| /* |
| * Check for virtual address in user space. |
| */ |
| int arch_check_va_in_userspace(unsigned long va, u8 hbp_len) |
| { |
| unsigned int len; |
| |
| len = get_hbp_len(hbp_len); |
| |
| return (va <= TASK_SIZE - len); |
| } |
| |
| /* |
| * Check for virtual address in kernel space. |
| */ |
| static int arch_check_va_in_kernelspace(unsigned long va, u8 hbp_len) |
| { |
| unsigned int len; |
| |
| len = get_hbp_len(hbp_len); |
| |
| return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); |
| } |
| |
| /* |
| * Store a breakpoint's encoded address, length, and type. |
| */ |
| static int arch_store_info(struct perf_event *bp) |
| { |
| struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
| /* |
| * For kernel-addresses, either the address or symbol name can be |
| * specified. |
| */ |
| if (info->name) |
| info->address = (unsigned long) |
| kallsyms_lookup_name(info->name); |
| if (info->address) |
| return 0; |
| |
| return -EINVAL; |
| } |
| |
| int arch_bp_generic_fields(int x86_len, int x86_type, |
| int *gen_len, int *gen_type) |
| { |
| /* Len */ |
| switch (x86_len) { |
| case X86_BREAKPOINT_LEN_1: |
| *gen_len = HW_BREAKPOINT_LEN_1; |
| break; |
| case X86_BREAKPOINT_LEN_2: |
| *gen_len = HW_BREAKPOINT_LEN_2; |
| break; |
| case X86_BREAKPOINT_LEN_4: |
| *gen_len = HW_BREAKPOINT_LEN_4; |
| break; |
| #ifdef CONFIG_X86_64 |
| case X86_BREAKPOINT_LEN_8: |
| *gen_len = HW_BREAKPOINT_LEN_8; |
| break; |
| #endif |
| default: |
| return -EINVAL; |
| } |
| |
| /* Type */ |
| switch (x86_type) { |
| case X86_BREAKPOINT_EXECUTE: |
| *gen_type = HW_BREAKPOINT_X; |
| break; |
| case X86_BREAKPOINT_WRITE: |
| *gen_type = HW_BREAKPOINT_W; |
| break; |
| case X86_BREAKPOINT_RW: |
| *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| |
| static int arch_build_bp_info(struct perf_event *bp) |
| { |
| struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
| |
| info->address = bp->attr.bp_addr; |
| |
| /* Len */ |
| switch (bp->attr.bp_len) { |
| case HW_BREAKPOINT_LEN_1: |
| info->len = X86_BREAKPOINT_LEN_1; |
| break; |
| case HW_BREAKPOINT_LEN_2: |
| info->len = X86_BREAKPOINT_LEN_2; |
| break; |
| case HW_BREAKPOINT_LEN_4: |
| info->len = X86_BREAKPOINT_LEN_4; |
| break; |
| #ifdef CONFIG_X86_64 |
| case HW_BREAKPOINT_LEN_8: |
| info->len = X86_BREAKPOINT_LEN_8; |
| break; |
| #endif |
| default: |
| return -EINVAL; |
| } |
| |
| /* Type */ |
| switch (bp->attr.bp_type) { |
| case HW_BREAKPOINT_W: |
| info->type = X86_BREAKPOINT_WRITE; |
| break; |
| case HW_BREAKPOINT_W | HW_BREAKPOINT_R: |
| info->type = X86_BREAKPOINT_RW; |
| break; |
| case HW_BREAKPOINT_X: |
| info->type = X86_BREAKPOINT_EXECUTE; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| /* |
| * Validate the arch-specific HW Breakpoint register settings |
| */ |
| int arch_validate_hwbkpt_settings(struct perf_event *bp, |
| struct task_struct *tsk) |
| { |
| struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
| unsigned int align; |
| int ret; |
| |
| |
| ret = arch_build_bp_info(bp); |
| if (ret) |
| return ret; |
| |
| ret = -EINVAL; |
| |
| if (info->type == X86_BREAKPOINT_EXECUTE) |
| /* |
| * Ptrace-refactoring code |
| * For now, we'll allow instruction breakpoint only for user-space |
| * addresses |
| */ |
| if ((!arch_check_va_in_userspace(info->address, info->len)) && |
| info->len != X86_BREAKPOINT_EXECUTE) |
| return ret; |
| |
| switch (info->len) { |
| case X86_BREAKPOINT_LEN_1: |
| align = 0; |
| break; |
| case X86_BREAKPOINT_LEN_2: |
| align = 1; |
| break; |
| case X86_BREAKPOINT_LEN_4: |
| align = 3; |
| break; |
| #ifdef CONFIG_X86_64 |
| case X86_BREAKPOINT_LEN_8: |
| align = 7; |
| break; |
| #endif |
| default: |
| return ret; |
| } |
| |
| if (bp->callback) |
| ret = arch_store_info(bp); |
| |
| if (ret < 0) |
| return ret; |
| /* |
| * Check that the low-order bits of the address are appropriate |
| * for the alignment implied by len. |
| */ |
| if (info->address & align) |
| return -EINVAL; |
| |
| /* Check that the virtual address is in the proper range */ |
| if (tsk) { |
| if (!arch_check_va_in_userspace(info->address, info->len)) |
| return -EFAULT; |
| } else { |
| if (!arch_check_va_in_kernelspace(info->address, info->len)) |
| return -EFAULT; |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Dump the debug register contents to the user. |
| * We can't dump our per cpu values because it |
| * may contain cpu wide breakpoint, something that |
| * doesn't belong to the current task. |
| * |
| * TODO: include non-ptrace user breakpoints (perf) |
| */ |
| void aout_dump_debugregs(struct user *dump) |
| { |
| int i; |
| int dr7 = 0; |
| struct perf_event *bp; |
| struct arch_hw_breakpoint *info; |
| struct thread_struct *thread = ¤t->thread; |
| |
| for (i = 0; i < HBP_NUM; i++) { |
| bp = thread->ptrace_bps[i]; |
| |
| if (bp && !bp->attr.disabled) { |
| dump->u_debugreg[i] = bp->attr.bp_addr; |
| info = counter_arch_bp(bp); |
| dr7 |= encode_dr7(i, info->len, info->type); |
| } else { |
| dump->u_debugreg[i] = 0; |
| } |
| } |
| |
| dump->u_debugreg[4] = 0; |
| dump->u_debugreg[5] = 0; |
| dump->u_debugreg[6] = current->thread.debugreg6; |
| |
| dump->u_debugreg[7] = dr7; |
| } |
| |
| /* |
| * Release the user breakpoints used by ptrace |
| */ |
| void flush_ptrace_hw_breakpoint(struct task_struct *tsk) |
| { |
| int i; |
| struct thread_struct *t = &tsk->thread; |
| |
| for (i = 0; i < HBP_NUM; i++) { |
| unregister_hw_breakpoint(t->ptrace_bps[i]); |
| t->ptrace_bps[i] = NULL; |
| } |
| } |
| |
| #ifdef CONFIG_KVM |
| void hw_breakpoint_restore(void) |
| { |
| set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0); |
| set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1); |
| set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2); |
| set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3); |
| set_debugreg(current->thread.debugreg6, 6); |
| set_debugreg(__get_cpu_var(dr7), 7); |
| } |
| EXPORT_SYMBOL_GPL(hw_breakpoint_restore); |
| #endif |
| |
| /* |
| * Handle debug exception notifications. |
| * |
| * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below. |
| * |
| * NOTIFY_DONE returned if one of the following conditions is true. |
| * i) When the causative address is from user-space and the exception |
| * is a valid one, i.e. not triggered as a result of lazy debug register |
| * switching |
| * ii) When there are more bits than trap<n> set in DR6 register (such |
| * as BD, BS or BT) indicating that more than one debug condition is |
| * met and requires some more action in do_debug(). |
| * |
| * NOTIFY_STOP returned for all other cases |
| * |
| */ |
| static int __kprobes hw_breakpoint_handler(struct die_args *args) |
| { |
| int i, cpu, rc = NOTIFY_STOP; |
| struct perf_event *bp; |
| unsigned long dr7, dr6; |
| unsigned long *dr6_p; |
| |
| /* The DR6 value is pointed by args->err */ |
| dr6_p = (unsigned long *)ERR_PTR(args->err); |
| dr6 = *dr6_p; |
| |
| /* Do an early return if no trap bits are set in DR6 */ |
| if ((dr6 & DR_TRAP_BITS) == 0) |
| return NOTIFY_DONE; |
| |
| get_debugreg(dr7, 7); |
| /* Disable breakpoints during exception handling */ |
| set_debugreg(0UL, 7); |
| /* |
| * Assert that local interrupts are disabled |
| * Reset the DRn bits in the virtualized register value. |
| * The ptrace trigger routine will add in whatever is needed. |
| */ |
| current->thread.debugreg6 &= ~DR_TRAP_BITS; |
| cpu = get_cpu(); |
| |
| /* Handle all the breakpoints that were triggered */ |
| for (i = 0; i < HBP_NUM; ++i) { |
| if (likely(!(dr6 & (DR_TRAP0 << i)))) |
| continue; |
| |
| /* |
| * The counter may be concurrently released but that can only |
| * occur from a call_rcu() path. We can then safely fetch |
| * the breakpoint, use its callback, touch its counter |
| * while we are in an rcu_read_lock() path. |
| */ |
| rcu_read_lock(); |
| |
| bp = per_cpu(bp_per_reg[i], cpu); |
| if (bp) |
| rc = NOTIFY_DONE; |
| /* |
| * Reset the 'i'th TRAP bit in dr6 to denote completion of |
| * exception handling |
| */ |
| (*dr6_p) &= ~(DR_TRAP0 << i); |
| /* |
| * bp can be NULL due to lazy debug register switching |
| * or due to concurrent perf counter removing. |
| */ |
| if (!bp) { |
| rcu_read_unlock(); |
| break; |
| } |
| |
| (bp->callback)(bp, args->regs); |
| |
| rcu_read_unlock(); |
| } |
| if (dr6 & (~DR_TRAP_BITS)) |
| rc = NOTIFY_DONE; |
| |
| set_debugreg(dr7, 7); |
| put_cpu(); |
| |
| return rc; |
| } |
| |
| /* |
| * Handle debug exception notifications. |
| */ |
| int __kprobes hw_breakpoint_exceptions_notify( |
| struct notifier_block *unused, unsigned long val, void *data) |
| { |
| if (val != DIE_DEBUG) |
| return NOTIFY_DONE; |
| |
| return hw_breakpoint_handler(data); |
| } |
| |
| void hw_breakpoint_pmu_read(struct perf_event *bp) |
| { |
| /* TODO */ |
| } |
| |
| void hw_breakpoint_pmu_unthrottle(struct perf_event *bp) |
| { |
| /* TODO */ |
| } |