Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef __ASM_SYSTEM_H |
| 2 | #define __ASM_SYSTEM_H |
| 3 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | #include <linux/kernel.h> |
| 5 | #include <asm/segment.h> |
| 6 | #include <asm/cpufeature.h> |
| 7 | #include <linux/bitops.h> /* for LOCK_PREFIX */ |
| 8 | |
| 9 | #ifdef __KERNEL__ |
| 10 | |
| 11 | struct task_struct; /* one of the stranger aspects of C forward declarations.. */ |
| 12 | extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next)); |
| 13 | |
| 14 | #define switch_to(prev,next,last) do { \ |
| 15 | unsigned long esi,edi; \ |
Zachary Amsden | a520112 | 2005-09-03 15:56:44 -0700 | [diff] [blame] | 16 | asm volatile("pushl %%ebp\n\t" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | "movl %%esp,%0\n\t" /* save ESP */ \ |
| 18 | "movl %5,%%esp\n\t" /* restore ESP */ \ |
| 19 | "movl $1f,%1\n\t" /* save EIP */ \ |
| 20 | "pushl %6\n\t" /* restore EIP */ \ |
| 21 | "jmp __switch_to\n" \ |
| 22 | "1:\t" \ |
| 23 | "popl %%ebp\n\t" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ |
| 25 | "=a" (last),"=S" (esi),"=D" (edi) \ |
| 26 | :"m" (next->thread.esp),"m" (next->thread.eip), \ |
| 27 | "2" (prev), "d" (next)); \ |
| 28 | } while (0) |
| 29 | |
| 30 | #define _set_base(addr,base) do { unsigned long __pr; \ |
| 31 | __asm__ __volatile__ ("movw %%dx,%1\n\t" \ |
| 32 | "rorl $16,%%edx\n\t" \ |
| 33 | "movb %%dl,%2\n\t" \ |
| 34 | "movb %%dh,%3" \ |
| 35 | :"=&d" (__pr) \ |
| 36 | :"m" (*((addr)+2)), \ |
| 37 | "m" (*((addr)+4)), \ |
| 38 | "m" (*((addr)+7)), \ |
| 39 | "0" (base) \ |
| 40 | ); } while(0) |
| 41 | |
| 42 | #define _set_limit(addr,limit) do { unsigned long __lr; \ |
| 43 | __asm__ __volatile__ ("movw %%dx,%1\n\t" \ |
| 44 | "rorl $16,%%edx\n\t" \ |
| 45 | "movb %2,%%dh\n\t" \ |
| 46 | "andb $0xf0,%%dh\n\t" \ |
| 47 | "orb %%dh,%%dl\n\t" \ |
| 48 | "movb %%dl,%2" \ |
| 49 | :"=&d" (__lr) \ |
| 50 | :"m" (*(addr)), \ |
| 51 | "m" (*((addr)+6)), \ |
| 52 | "0" (limit) \ |
| 53 | ); } while(0) |
| 54 | |
| 55 | #define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) ) |
Zachary Amsden | 5fe9fe3c | 2006-01-06 00:11:55 -0800 | [diff] [blame] | 56 | #define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1) ) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | /* |
| 59 | * Load a segment. Fall back on loading the zero |
| 60 | * segment if something goes wrong.. |
| 61 | */ |
| 62 | #define loadsegment(seg,value) \ |
| 63 | asm volatile("\n" \ |
| 64 | "1:\t" \ |
H. J. Lu | fd51f66 | 2005-05-01 08:58:48 -0700 | [diff] [blame] | 65 | "mov %0,%%" #seg "\n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | "2:\n" \ |
| 67 | ".section .fixup,\"ax\"\n" \ |
| 68 | "3:\t" \ |
| 69 | "pushl $0\n\t" \ |
| 70 | "popl %%" #seg "\n\t" \ |
| 71 | "jmp 2b\n" \ |
| 72 | ".previous\n" \ |
| 73 | ".section __ex_table,\"a\"\n\t" \ |
| 74 | ".align 4\n\t" \ |
| 75 | ".long 1b,3b\n" \ |
| 76 | ".previous" \ |
Zachary Amsden | 4d37e7e | 2005-09-03 15:56:38 -0700 | [diff] [blame] | 77 | : :"rm" (value)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
| 79 | /* |
| 80 | * Save a segment register away |
| 81 | */ |
| 82 | #define savesegment(seg, value) \ |
Zachary Amsden | 4d37e7e | 2005-09-03 15:56:38 -0700 | [diff] [blame] | 83 | asm volatile("mov %%" #seg ",%0":"=rm" (value)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | |
| 85 | /* |
| 86 | * Clear and set 'TS' bit respectively |
| 87 | */ |
| 88 | #define clts() __asm__ __volatile__ ("clts") |
| 89 | #define read_cr0() ({ \ |
| 90 | unsigned int __dummy; \ |
Zachary Amsden | 4bb0d3e | 2005-09-03 15:56:36 -0700 | [diff] [blame] | 91 | __asm__ __volatile__( \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | "movl %%cr0,%0\n\t" \ |
| 93 | :"=r" (__dummy)); \ |
| 94 | __dummy; \ |
| 95 | }) |
| 96 | #define write_cr0(x) \ |
Zachary Amsden | 4bb0d3e | 2005-09-03 15:56:36 -0700 | [diff] [blame] | 97 | __asm__ __volatile__("movl %0,%%cr0": :"r" (x)); |
| 98 | |
| 99 | #define read_cr2() ({ \ |
| 100 | unsigned int __dummy; \ |
| 101 | __asm__ __volatile__( \ |
| 102 | "movl %%cr2,%0\n\t" \ |
| 103 | :"=r" (__dummy)); \ |
| 104 | __dummy; \ |
| 105 | }) |
| 106 | #define write_cr2(x) \ |
| 107 | __asm__ __volatile__("movl %0,%%cr2": :"r" (x)); |
| 108 | |
| 109 | #define read_cr3() ({ \ |
| 110 | unsigned int __dummy; \ |
| 111 | __asm__ ( \ |
| 112 | "movl %%cr3,%0\n\t" \ |
| 113 | :"=r" (__dummy)); \ |
| 114 | __dummy; \ |
| 115 | }) |
| 116 | #define write_cr3(x) \ |
| 117 | __asm__ __volatile__("movl %0,%%cr3": :"r" (x)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | |
| 119 | #define read_cr4() ({ \ |
| 120 | unsigned int __dummy; \ |
| 121 | __asm__( \ |
| 122 | "movl %%cr4,%0\n\t" \ |
| 123 | :"=r" (__dummy)); \ |
| 124 | __dummy; \ |
| 125 | }) |
Zachary Amsden | ff6e8c0 | 2006-01-06 00:11:50 -0800 | [diff] [blame] | 126 | |
| 127 | #define read_cr4_safe() ({ \ |
| 128 | unsigned int __dummy; \ |
| 129 | /* This could fault if %cr4 does not exist */ \ |
| 130 | __asm__("1: movl %%cr4, %0 \n" \ |
| 131 | "2: \n" \ |
| 132 | ".section __ex_table,\"a\" \n" \ |
| 133 | ".long 1b,2b \n" \ |
| 134 | ".previous \n" \ |
| 135 | : "=r" (__dummy): "0" (0)); \ |
| 136 | __dummy; \ |
| 137 | }) |
| 138 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | #define write_cr4(x) \ |
Zachary Amsden | 4bb0d3e | 2005-09-03 15:56:36 -0700 | [diff] [blame] | 140 | __asm__ __volatile__("movl %0,%%cr4": :"r" (x)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | #define stts() write_cr0(8 | read_cr0()) |
| 142 | |
| 143 | #endif /* __KERNEL__ */ |
| 144 | |
| 145 | #define wbinvd() \ |
| 146 | __asm__ __volatile__ ("wbinvd": : :"memory"); |
| 147 | |
| 148 | static inline unsigned long get_limit(unsigned long segment) |
| 149 | { |
| 150 | unsigned long __limit; |
| 151 | __asm__("lsll %1,%0" |
| 152 | :"=r" (__limit):"r" (segment)); |
| 153 | return __limit+1; |
| 154 | } |
| 155 | |
| 156 | #define nop() __asm__ __volatile__ ("nop") |
| 157 | |
| 158 | #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) |
| 159 | |
| 160 | #define tas(ptr) (xchg((ptr),1)) |
| 161 | |
| 162 | struct __xchg_dummy { unsigned long a[100]; }; |
| 163 | #define __xg(x) ((struct __xchg_dummy *)(x)) |
| 164 | |
| 165 | |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 166 | #ifdef CONFIG_X86_CMPXCHG64 |
| 167 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | /* |
| 169 | * The semantics of XCHGCMP8B are a bit strange, this is why |
| 170 | * there is a loop and the loading of %%eax and %%edx has to |
| 171 | * be inside. This inlines well in most cases, the cached |
| 172 | * cost is around ~38 cycles. (in the future we might want |
| 173 | * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that |
| 174 | * might have an implicit FPU-save as a cost, so it's not |
| 175 | * clear which path to go.) |
| 176 | * |
| 177 | * cmpxchg8b must be used with the lock prefix here to allow |
| 178 | * the instruction to be executed atomically, see page 3-102 |
| 179 | * of the instruction set reference 24319102.pdf. We need |
| 180 | * the reader side to see the coherent 64bit value. |
| 181 | */ |
| 182 | static inline void __set_64bit (unsigned long long * ptr, |
| 183 | unsigned int low, unsigned int high) |
| 184 | { |
| 185 | __asm__ __volatile__ ( |
| 186 | "\n1:\t" |
| 187 | "movl (%0), %%eax\n\t" |
| 188 | "movl 4(%0), %%edx\n\t" |
| 189 | "lock cmpxchg8b (%0)\n\t" |
| 190 | "jnz 1b" |
| 191 | : /* no outputs */ |
| 192 | : "D"(ptr), |
| 193 | "b"(low), |
| 194 | "c"(high) |
| 195 | : "ax","dx","memory"); |
| 196 | } |
| 197 | |
| 198 | static inline void __set_64bit_constant (unsigned long long *ptr, |
| 199 | unsigned long long value) |
| 200 | { |
| 201 | __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL)); |
| 202 | } |
| 203 | #define ll_low(x) *(((unsigned int*)&(x))+0) |
| 204 | #define ll_high(x) *(((unsigned int*)&(x))+1) |
| 205 | |
| 206 | static inline void __set_64bit_var (unsigned long long *ptr, |
| 207 | unsigned long long value) |
| 208 | { |
| 209 | __set_64bit(ptr,ll_low(value), ll_high(value)); |
| 210 | } |
| 211 | |
| 212 | #define set_64bit(ptr,value) \ |
| 213 | (__builtin_constant_p(value) ? \ |
| 214 | __set_64bit_constant(ptr, value) : \ |
| 215 | __set_64bit_var(ptr, value) ) |
| 216 | |
| 217 | #define _set_64bit(ptr,value) \ |
| 218 | (__builtin_constant_p(value) ? \ |
| 219 | __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \ |
| 220 | __set_64bit(ptr, ll_low(value), ll_high(value)) ) |
| 221 | |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 222 | #endif |
| 223 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | /* |
| 225 | * Note: no "lock" prefix even on SMP: xchg always implies lock anyway |
| 226 | * Note 2: xchg has side effect, so that attribute volatile is necessary, |
| 227 | * but generally the primitive is invalid, *ptr is output argument. --ANK |
| 228 | */ |
| 229 | static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) |
| 230 | { |
| 231 | switch (size) { |
| 232 | case 1: |
| 233 | __asm__ __volatile__("xchgb %b0,%1" |
| 234 | :"=q" (x) |
| 235 | :"m" (*__xg(ptr)), "0" (x) |
| 236 | :"memory"); |
| 237 | break; |
| 238 | case 2: |
| 239 | __asm__ __volatile__("xchgw %w0,%1" |
| 240 | :"=r" (x) |
| 241 | :"m" (*__xg(ptr)), "0" (x) |
| 242 | :"memory"); |
| 243 | break; |
| 244 | case 4: |
| 245 | __asm__ __volatile__("xchgl %0,%1" |
| 246 | :"=r" (x) |
| 247 | :"m" (*__xg(ptr)), "0" (x) |
| 248 | :"memory"); |
| 249 | break; |
| 250 | } |
| 251 | return x; |
| 252 | } |
| 253 | |
| 254 | /* |
| 255 | * Atomic compare and exchange. Compare OLD with MEM, if identical, |
| 256 | * store NEW in MEM. Return the initial value in MEM. Success is |
| 257 | * indicated by comparing RETURN with OLD. |
| 258 | */ |
| 259 | |
| 260 | #ifdef CONFIG_X86_CMPXCHG |
| 261 | #define __HAVE_ARCH_CMPXCHG 1 |
Nick Piggin | 53e86b9 | 2005-11-13 16:07:23 -0800 | [diff] [blame] | 262 | #define cmpxchg(ptr,o,n)\ |
| 263 | ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ |
| 264 | (unsigned long)(n),sizeof(*(ptr)))) |
| 265 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | |
| 267 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, |
| 268 | unsigned long new, int size) |
| 269 | { |
| 270 | unsigned long prev; |
| 271 | switch (size) { |
| 272 | case 1: |
| 273 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" |
| 274 | : "=a"(prev) |
| 275 | : "q"(new), "m"(*__xg(ptr)), "0"(old) |
| 276 | : "memory"); |
| 277 | return prev; |
| 278 | case 2: |
| 279 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" |
| 280 | : "=a"(prev) |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 281 | : "r"(new), "m"(*__xg(ptr)), "0"(old) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | : "memory"); |
| 283 | return prev; |
| 284 | case 4: |
| 285 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" |
| 286 | : "=a"(prev) |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 287 | : "r"(new), "m"(*__xg(ptr)), "0"(old) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | : "memory"); |
| 289 | return prev; |
| 290 | } |
| 291 | return old; |
| 292 | } |
| 293 | |
Nick Piggin | 53e86b9 | 2005-11-13 16:07:23 -0800 | [diff] [blame] | 294 | #ifndef CONFIG_X86_CMPXCHG |
| 295 | /* |
| 296 | * Building a kernel capable running on 80386. It may be necessary to |
| 297 | * simulate the cmpxchg on the 80386 CPU. For that purpose we define |
| 298 | * a function for each of the sizes we support. |
| 299 | */ |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 300 | |
Nick Piggin | 53e86b9 | 2005-11-13 16:07:23 -0800 | [diff] [blame] | 301 | extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8); |
| 302 | extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); |
| 303 | extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); |
| 304 | |
| 305 | static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, |
| 306 | unsigned long new, int size) |
| 307 | { |
| 308 | switch (size) { |
| 309 | case 1: |
| 310 | return cmpxchg_386_u8(ptr, old, new); |
| 311 | case 2: |
| 312 | return cmpxchg_386_u16(ptr, old, new); |
| 313 | case 4: |
| 314 | return cmpxchg_386_u32(ptr, old, new); |
| 315 | } |
| 316 | return old; |
| 317 | } |
| 318 | |
| 319 | #define cmpxchg(ptr,o,n) \ |
| 320 | ({ \ |
| 321 | __typeof__(*(ptr)) __ret; \ |
| 322 | if (likely(boot_cpu_data.x86 > 3)) \ |
| 323 | __ret = __cmpxchg((ptr), (unsigned long)(o), \ |
| 324 | (unsigned long)(n), sizeof(*(ptr))); \ |
| 325 | else \ |
| 326 | __ret = cmpxchg_386((ptr), (unsigned long)(o), \ |
| 327 | (unsigned long)(n), sizeof(*(ptr))); \ |
| 328 | __ret; \ |
| 329 | }) |
Jan Beulich | 8896fab | 2005-10-30 14:59:27 -0800 | [diff] [blame] | 330 | #endif |
| 331 | |
| 332 | #ifdef CONFIG_X86_CMPXCHG64 |
| 333 | |
| 334 | static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old, |
| 335 | unsigned long long new) |
| 336 | { |
| 337 | unsigned long long prev; |
| 338 | __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3" |
| 339 | : "=A"(prev) |
| 340 | : "b"((unsigned long)new), |
| 341 | "c"((unsigned long)(new >> 32)), |
| 342 | "m"(*__xg(ptr)), |
| 343 | "0"(old) |
| 344 | : "memory"); |
| 345 | return prev; |
| 346 | } |
| 347 | |
| 348 | #define cmpxchg64(ptr,o,n)\ |
| 349 | ((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\ |
| 350 | (unsigned long long)(n))) |
| 351 | |
| 352 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 353 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 354 | /* |
| 355 | * Force strict CPU ordering. |
| 356 | * And yes, this is required on UP too when we're talking |
| 357 | * to devices. |
| 358 | * |
| 359 | * For now, "wmb()" doesn't actually do anything, as all |
| 360 | * Intel CPU's follow what Intel calls a *Processor Order*, |
| 361 | * in which all writes are seen in the program order even |
| 362 | * outside the CPU. |
| 363 | * |
| 364 | * I expect future Intel CPU's to have a weaker ordering, |
| 365 | * but I'd also expect them to finally get their act together |
| 366 | * and add some real memory barriers if so. |
| 367 | * |
| 368 | * Some non intel clones support out of order store. wmb() ceases to be a |
| 369 | * nop for these. |
| 370 | */ |
| 371 | |
| 372 | |
| 373 | /* |
| 374 | * Actually only lfence would be needed for mb() because all stores done |
| 375 | * by the kernel should be already ordered. But keep a full barrier for now. |
| 376 | */ |
| 377 | |
| 378 | #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) |
| 379 | #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2) |
| 380 | |
| 381 | /** |
| 382 | * read_barrier_depends - Flush all pending reads that subsequents reads |
| 383 | * depend on. |
| 384 | * |
| 385 | * No data-dependent reads from memory-like regions are ever reordered |
| 386 | * over this barrier. All reads preceding this primitive are guaranteed |
| 387 | * to access memory (but not necessarily other CPUs' caches) before any |
| 388 | * reads following this primitive that depend on the data return by |
| 389 | * any of the preceding reads. This primitive is much lighter weight than |
| 390 | * rmb() on most CPUs, and is never heavier weight than is |
| 391 | * rmb(). |
| 392 | * |
| 393 | * These ordering constraints are respected by both the local CPU |
| 394 | * and the compiler. |
| 395 | * |
| 396 | * Ordering is not guaranteed by anything other than these primitives, |
| 397 | * not even by data dependencies. See the documentation for |
| 398 | * memory_barrier() for examples and URLs to more information. |
| 399 | * |
| 400 | * For example, the following code would force ordering (the initial |
| 401 | * value of "a" is zero, "b" is one, and "p" is "&a"): |
| 402 | * |
| 403 | * <programlisting> |
| 404 | * CPU 0 CPU 1 |
| 405 | * |
| 406 | * b = 2; |
| 407 | * memory_barrier(); |
| 408 | * p = &b; q = p; |
| 409 | * read_barrier_depends(); |
| 410 | * d = *q; |
| 411 | * </programlisting> |
| 412 | * |
| 413 | * because the read of "*q" depends on the read of "p" and these |
| 414 | * two reads are separated by a read_barrier_depends(). However, |
| 415 | * the following code, with the same initial values for "a" and "b": |
| 416 | * |
| 417 | * <programlisting> |
| 418 | * CPU 0 CPU 1 |
| 419 | * |
| 420 | * a = 2; |
| 421 | * memory_barrier(); |
| 422 | * b = 3; y = b; |
| 423 | * read_barrier_depends(); |
| 424 | * x = a; |
| 425 | * </programlisting> |
| 426 | * |
| 427 | * does not enforce ordering, since there is no data dependency between |
| 428 | * the read of "a" and the read of "b". Therefore, on some CPUs, such |
| 429 | * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() |
| 430 | * in cases like thiswhere there are no data dependencies. |
| 431 | **/ |
| 432 | |
| 433 | #define read_barrier_depends() do { } while(0) |
| 434 | |
| 435 | #ifdef CONFIG_X86_OOSTORE |
| 436 | /* Actually there are no OOO store capable CPUs for now that do SSE, |
| 437 | but make it already an possibility. */ |
| 438 | #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM) |
| 439 | #else |
| 440 | #define wmb() __asm__ __volatile__ ("": : :"memory") |
| 441 | #endif |
| 442 | |
| 443 | #ifdef CONFIG_SMP |
| 444 | #define smp_mb() mb() |
| 445 | #define smp_rmb() rmb() |
| 446 | #define smp_wmb() wmb() |
| 447 | #define smp_read_barrier_depends() read_barrier_depends() |
Takashi Iwai | 911b0ad | 2006-02-04 23:28:05 -0800 | [diff] [blame] | 448 | #define set_mb(var, value) do { (void) xchg(&var, value); } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | #else |
| 450 | #define smp_mb() barrier() |
| 451 | #define smp_rmb() barrier() |
| 452 | #define smp_wmb() barrier() |
| 453 | #define smp_read_barrier_depends() do { } while(0) |
| 454 | #define set_mb(var, value) do { var = value; barrier(); } while (0) |
| 455 | #endif |
| 456 | |
| 457 | #define set_wmb(var, value) do { var = value; wmb(); } while (0) |
| 458 | |
| 459 | /* interrupt control.. */ |
| 460 | #define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */); } while (0) |
| 461 | #define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0) |
| 462 | #define local_irq_disable() __asm__ __volatile__("cli": : :"memory") |
| 463 | #define local_irq_enable() __asm__ __volatile__("sti": : :"memory") |
| 464 | /* used in the idle loop; sti takes one instruction cycle to complete */ |
| 465 | #define safe_halt() __asm__ __volatile__("sti; hlt": : :"memory") |
Zachary Amsden | 4bb0d3e | 2005-09-03 15:56:36 -0700 | [diff] [blame] | 466 | /* used when interrupts are already enabled or to shutdown the processor */ |
| 467 | #define halt() __asm__ __volatile__("hlt": : :"memory") |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | |
| 469 | #define irqs_disabled() \ |
| 470 | ({ \ |
| 471 | unsigned long flags; \ |
| 472 | local_save_flags(flags); \ |
| 473 | !(flags & (1<<9)); \ |
| 474 | }) |
| 475 | |
| 476 | /* For spinlocks etc */ |
| 477 | #define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") |
| 478 | |
| 479 | /* |
| 480 | * disable hlt during certain critical i/o operations |
| 481 | */ |
| 482 | #define HAVE_DISABLE_HLT |
| 483 | void disable_hlt(void); |
| 484 | void enable_hlt(void); |
| 485 | |
| 486 | extern int es7000_plat; |
| 487 | void cpu_idle_wait(void); |
| 488 | |
Ingo Molnar | 4dc7a0b | 2006-01-12 01:05:27 -0800 | [diff] [blame] | 489 | /* |
| 490 | * On SMP systems, when the scheduler does migration-cost autodetection, |
| 491 | * it needs a way to flush as much of the CPU's caches as possible: |
| 492 | */ |
| 493 | static inline void sched_cacheflush(void) |
| 494 | { |
| 495 | wbinvd(); |
| 496 | } |
| 497 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 498 | extern unsigned long arch_align_stack(unsigned long sp); |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 499 | extern void free_init_pages(char *what, unsigned long begin, unsigned long end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 500 | |
Adrian Bunk | cdb0452 | 2006-03-24 03:15:57 -0800 | [diff] [blame] | 501 | void default_idle(void); |
| 502 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 503 | #endif |