blob: d8def8b3dba0b46a9d52b06d65262560a8c3cca2 [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_UACCESS_64_H
2#define _ASM_X86_UACCESS_64_H
Linus Torvalds1da177e2005-04-16 15:20:36 -07003
4/*
5 * User space memory access functions
6 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/compiler.h>
8#include <linux/errno.h>
Nick Piggin16dbc6c2008-10-02 14:50:12 -07009#include <linux/lockdep.h>
Jan Beulich1b1d9252009-12-18 16:12:56 +000010#include <asm/alternative.h>
11#include <asm/cpufeature.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <asm/page.h>
13
Linus Torvalds1da177e2005-04-16 15:20:36 -070014/*
15 * Copy To/From Userspace
16 */
17
18/* Handles exceptions in both to and from, but doesn't do access_ok */
Andi Kleen95912002006-09-26 10:52:39 +020019__must_check unsigned long
Fenghua Yu954e4822012-05-24 18:19:45 -070020copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
21__must_check unsigned long
Jan Beulich1b1d9252009-12-18 16:12:56 +000022copy_user_generic_string(void *to, const void *from, unsigned len);
23__must_check unsigned long
24copy_user_generic_unrolled(void *to, const void *from, unsigned len);
25
26static __always_inline __must_check unsigned long
27copy_user_generic(void *to, const void *from, unsigned len)
28{
29 unsigned ret;
30
Fenghua Yu954e4822012-05-24 18:19:45 -070031 /*
32 * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
33 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
34 * Otherwise, use copy_user_generic_unrolled.
35 */
36 alternative_call_2(copy_user_generic_unrolled,
Jan Beulich1b1d9252009-12-18 16:12:56 +000037 copy_user_generic_string,
38 X86_FEATURE_REP_GOOD,
Fenghua Yu954e4822012-05-24 18:19:45 -070039 copy_user_enhanced_fast_string,
40 X86_FEATURE_ERMS,
Jan Beulich1b1d9252009-12-18 16:12:56 +000041 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
42 "=d" (len)),
43 "1" (to), "2" (from), "3" (len)
44 : "memory", "rcx", "r8", "r9", "r10", "r11");
45 return ret;
46}
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
Andi Kleen95912002006-09-26 10:52:39 +020048__must_check unsigned long
Frederic Weisbecker3c93ca02009-11-16 15:42:18 +010049_copy_to_user(void __user *to, const void *from, unsigned len);
Andi Kleen95912002006-09-26 10:52:39 +020050__must_check unsigned long
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020051_copy_from_user(void *to, const void __user *from, unsigned len);
Andi Kleen95912002006-09-26 10:52:39 +020052__must_check unsigned long
53copy_in_user(void __user *to, const void __user *from, unsigned len);
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020055static inline unsigned long __must_check copy_from_user(void *to,
56 const void __user *from,
57 unsigned long n)
58{
59 int sz = __compiletime_object_size(to);
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020060
Frederic Weisbecker3c93ca02009-11-16 15:42:18 +010061 might_fault();
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020062 if (likely(sz == -1 || sz >= n))
Heiko Carstens409d02e2010-01-05 14:19:11 +010063 n = _copy_from_user(to, from, n);
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020064#ifdef CONFIG_DEBUG_VM
65 else
66 WARN(1, "Buffer overflow detected!\n");
67#endif
Heiko Carstens409d02e2010-01-05 14:19:11 +010068 return n;
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020069}
70
Frederic Weisbecker3c93ca02009-11-16 15:42:18 +010071static __always_inline __must_check
72int copy_to_user(void __user *dst, const void *src, unsigned size)
73{
74 might_fault();
75
76 return _copy_to_user(dst, src, size);
77}
Arjan van de Ven9f0cf4a2009-09-26 14:33:01 +020078
Andi Kleen95912002006-09-26 10:52:39 +020079static __always_inline __must_check
80int __copy_from_user(void *dst, const void __user *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -070081{
Andi Kleen383d0792006-09-26 10:52:40 +020082 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +010083
84 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -070086 return copy_user_generic(dst, (__force void *)src, size);
87 switch (size) {
88 case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
89 ret, "b", "b", "=q", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070091 case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
92 ret, "w", "w", "=r", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070094 case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
95 ret, "l", "k", "=r", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070097 case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
98 ret, "q", "", "=r", 8);
99 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 case 10:
Joe Perchesb8963132008-03-23 01:03:49 -0700101 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
Hiroshi Shimamoto20a4a232008-11-13 18:06:04 -0800102 ret, "q", "", "=r", 10);
Joe Perchesb8963132008-03-23 01:03:49 -0700103 if (unlikely(ret))
104 return ret;
105 __get_user_asm(*(u16 *)(8 + (char *)dst),
106 (u16 __user *)(8 + (char __user *)src),
107 ret, "w", "w", "=r", 2);
108 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 case 16:
Joe Perchesb8963132008-03-23 01:03:49 -0700110 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
111 ret, "q", "", "=r", 16);
112 if (unlikely(ret))
113 return ret;
114 __get_user_asm(*(u64 *)(8 + (char *)dst),
115 (u64 __user *)(8 + (char __user *)src),
116 ret, "q", "", "=r", 8);
117 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 default:
Joe Perchesb8963132008-03-23 01:03:49 -0700119 return copy_user_generic(dst, (__force void *)src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 }
Joe Perchesb8963132008-03-23 01:03:49 -0700121}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
Andi Kleen95912002006-09-26 10:52:39 +0200123static __always_inline __must_check
124int __copy_to_user(void __user *dst, const void *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -0700125{
Andi Kleen383d0792006-09-26 10:52:40 +0200126 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +0100127
128 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -0700130 return copy_user_generic((__force void *)dst, src, size);
131 switch (size) {
132 case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
133 ret, "b", "b", "iq", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -0700135 case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
136 ret, "w", "w", "ir", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -0700138 case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
139 ret, "l", "k", "ir", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -0700141 case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200142 ret, "q", "", "er", 8);
Joe Perchesb8963132008-03-23 01:03:49 -0700143 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 case 10:
Joe Perchesb8963132008-03-23 01:03:49 -0700145 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200146 ret, "q", "", "er", 10);
Joe Perchesb8963132008-03-23 01:03:49 -0700147 if (unlikely(ret))
148 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 asm("":::"memory");
Joe Perchesb8963132008-03-23 01:03:49 -0700150 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
151 ret, "w", "w", "ir", 2);
152 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 case 16:
Joe Perchesb8963132008-03-23 01:03:49 -0700154 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200155 ret, "q", "", "er", 16);
Joe Perchesb8963132008-03-23 01:03:49 -0700156 if (unlikely(ret))
157 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 asm("":::"memory");
Joe Perchesb8963132008-03-23 01:03:49 -0700159 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200160 ret, "q", "", "er", 8);
Joe Perchesb8963132008-03-23 01:03:49 -0700161 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 default:
Joe Perchesb8963132008-03-23 01:03:49 -0700163 return copy_user_generic((__force void *)dst, src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 }
Joe Perchesb8963132008-03-23 01:03:49 -0700165}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
Andi Kleen95912002006-09-26 10:52:39 +0200167static __always_inline __must_check
168int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -0700169{
Andi Kleen383d0792006-09-26 10:52:40 +0200170 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +0100171
172 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -0700174 return copy_user_generic((__force void *)dst,
175 (__force void *)src, size);
176 switch (size) {
177 case 1: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 u8 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700179 __get_user_asm(tmp, (u8 __user *)src,
180 ret, "b", "b", "=q", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700182 __put_user_asm(tmp, (u8 __user *)dst,
183 ret, "b", "b", "iq", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 return ret;
185 }
Joe Perchesb8963132008-03-23 01:03:49 -0700186 case 2: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 u16 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700188 __get_user_asm(tmp, (u16 __user *)src,
189 ret, "w", "w", "=r", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700191 __put_user_asm(tmp, (u16 __user *)dst,
192 ret, "w", "w", "ir", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 return ret;
194 }
195
Joe Perchesb8963132008-03-23 01:03:49 -0700196 case 4: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 u32 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700198 __get_user_asm(tmp, (u32 __user *)src,
199 ret, "l", "k", "=r", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700201 __put_user_asm(tmp, (u32 __user *)dst,
202 ret, "l", "k", "ir", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 return ret;
204 }
Joe Perchesb8963132008-03-23 01:03:49 -0700205 case 8: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 u64 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700207 __get_user_asm(tmp, (u64 __user *)src,
208 ret, "q", "", "=r", 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700210 __put_user_asm(tmp, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200211 ret, "q", "", "er", 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 return ret;
213 }
214 default:
Joe Perchesb8963132008-03-23 01:03:49 -0700215 return copy_user_generic((__force void *)dst,
216 (__force void *)src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 }
Joe Perchesb8963132008-03-23 01:03:49 -0700218}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
Andi Kleen95912002006-09-26 10:52:39 +0200220__must_check unsigned long clear_user(void __user *mem, unsigned long len);
221__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Jan Beulich14722482009-11-13 11:56:24 +0000223static __must_check __always_inline int
224__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
225{
226 return copy_user_generic(dst, (__force const void *)src, size);
227}
Andi Kleenb8858082006-09-30 01:47:55 +0200228
229static __must_check __always_inline int
230__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
231{
232 return copy_user_generic((__force void *)dst, src, size);
233}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234
Joe Perchesb8963132008-03-23 01:03:49 -0700235extern long __copy_user_nocache(void *dst, const void __user *src,
236 unsigned size, int zerorest);
Andi Kleen0812a572007-02-13 13:26:19 +0100237
Ingo Molnarf1800532009-03-02 11:00:57 +0100238static inline int
239__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
Andi Kleen0812a572007-02-13 13:26:19 +0100240{
241 might_sleep();
Ingo Molnarf1800532009-03-02 11:00:57 +0100242 return __copy_user_nocache(dst, src, size, 1);
Andi Kleen0812a572007-02-13 13:26:19 +0100243}
244
Ingo Molnarf1800532009-03-02 11:00:57 +0100245static inline int
246__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
247 unsigned size)
Andi Kleen0812a572007-02-13 13:26:19 +0100248{
Ingo Molnarf1800532009-03-02 11:00:57 +0100249 return __copy_user_nocache(dst, src, size, 0);
Andi Kleen0812a572007-02-13 13:26:19 +0100250}
251
Vitaly Mayatskikh11295852008-07-02 15:48:21 +0200252unsigned long
253copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);
254
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700255#endif /* _ASM_X86_UACCESS_64_H */