blob: f77fc53db654655ccf5477e16c3750012a1a1230 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
2 because MTRRs can span upto 40 bits (36bits on most modern x86) */
3#include <linux/init.h>
4#include <linux/slab.h>
5#include <linux/mm.h>
Jan Beulich365bff82006-12-07 02:14:09 +01006#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <asm/io.h>
8#include <asm/mtrr.h>
9#include <asm/msr.h>
10#include <asm/system.h>
11#include <asm/cpufeature.h>
12#include <asm/tlbflush.h>
13#include "mtrr.h"
14
15struct mtrr_state {
16 struct mtrr_var_range *var_ranges;
17 mtrr_type fixed_ranges[NUM_FIXED_RANGES];
18 unsigned char enabled;
Jan Beulich365bff82006-12-07 02:14:09 +010019 unsigned char have_fixed;
Linus Torvalds1da177e2005-04-16 15:20:36 -070020 mtrr_type def_type;
21};
22
23static unsigned long smp_changes_mask;
24static struct mtrr_state mtrr_state = {};
25
Jan Beulich365bff82006-12-07 02:14:09 +010026#undef MODULE_PARAM_PREFIX
27#define MODULE_PARAM_PREFIX "mtrr."
28
29static __initdata int mtrr_show;
30module_param_named(show, mtrr_show, bool, 0);
31
Linus Torvalds1da177e2005-04-16 15:20:36 -070032/* Get the MSR pair relating to a var range */
33static void __init
34get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
35{
36 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
37 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
38}
39
40static void __init
41get_fixed_ranges(mtrr_type * frs)
42{
43 unsigned int *p = (unsigned int *) frs;
44 int i;
45
46 rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]);
47
48 for (i = 0; i < 2; i++)
49 rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]);
50 for (i = 0; i < 8; i++)
51 rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
52}
53
Jan Beulich365bff82006-12-07 02:14:09 +010054static void __init print_fixed(unsigned base, unsigned step, const mtrr_type*types)
55{
56 unsigned i;
57
58 for (i = 0; i < 8; ++i, ++types, base += step)
59 printk(KERN_INFO "MTRR %05X-%05X %s\n", base, base + step - 1, mtrr_attrib_to_str(*types));
60}
61
Linus Torvalds1da177e2005-04-16 15:20:36 -070062/* Grab all of the MTRR state for this CPU into *state */
63void __init get_mtrr_state(void)
64{
65 unsigned int i;
66 struct mtrr_var_range *vrs;
67 unsigned lo, dummy;
68
69 if (!mtrr_state.var_ranges) {
70 mtrr_state.var_ranges = kmalloc(num_var_ranges * sizeof (struct mtrr_var_range),
71 GFP_KERNEL);
72 if (!mtrr_state.var_ranges)
73 return;
74 }
75 vrs = mtrr_state.var_ranges;
76
Jan Beulich365bff82006-12-07 02:14:09 +010077 rdmsr(MTRRcap_MSR, lo, dummy);
78 mtrr_state.have_fixed = (lo >> 8) & 1;
79
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 for (i = 0; i < num_var_ranges; i++)
81 get_mtrr_var_range(i, &vrs[i]);
Jan Beulich365bff82006-12-07 02:14:09 +010082 if (mtrr_state.have_fixed)
83 get_fixed_ranges(mtrr_state.fixed_ranges);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
85 rdmsr(MTRRdefType_MSR, lo, dummy);
86 mtrr_state.def_type = (lo & 0xff);
87 mtrr_state.enabled = (lo & 0xc00) >> 10;
Jan Beulich365bff82006-12-07 02:14:09 +010088
89 if (mtrr_show) {
90 int high_width;
91
92 printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type));
93 if (mtrr_state.have_fixed) {
94 printk(KERN_INFO "MTRR fixed ranges %sabled:\n",
95 mtrr_state.enabled & 1 ? "en" : "dis");
96 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
97 for (i = 0; i < 2; ++i)
98 print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
99 for (i = 0; i < 8; ++i)
100 print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
101 }
102 printk(KERN_INFO "MTRR variable ranges %sabled:\n",
103 mtrr_state.enabled & 2 ? "en" : "dis");
104 high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4;
105 for (i = 0; i < num_var_ranges; ++i) {
106 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
107 printk(KERN_INFO "MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n",
108 i,
109 high_width,
110 mtrr_state.var_ranges[i].base_hi,
111 mtrr_state.var_ranges[i].base_lo >> 12,
112 high_width,
113 mtrr_state.var_ranges[i].mask_hi,
114 mtrr_state.var_ranges[i].mask_lo >> 12,
115 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
116 else
117 printk(KERN_INFO "MTRR %u disabled\n", i);
118 }
119 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120}
121
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122/* Some BIOS's are fucked and don't set all MTRRs the same! */
123void __init mtrr_state_warn(void)
124{
125 unsigned long mask = smp_changes_mask;
126
127 if (!mask)
128 return;
129 if (mask & MTRR_CHANGE_MASK_FIXED)
130 printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
131 if (mask & MTRR_CHANGE_MASK_VARIABLE)
132 printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
133 if (mask & MTRR_CHANGE_MASK_DEFTYPE)
134 printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
135 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
136 printk(KERN_INFO "mtrr: corrected configuration.\n");
137}
138
139/* Doesn't attempt to pass an error out to MTRR users
140 because it's quite complicated in some cases and probably not
141 worth it because the best error handling is to ignore it. */
142void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
143{
144 if (wrmsr_safe(msr, a, b) < 0)
145 printk(KERN_ERR
146 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
147 smp_processor_id(), msr, a, b);
148}
149
Jan Beulich365bff82006-12-07 02:14:09 +0100150int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151/* [SUMMARY] Get a free MTRR.
152 <base> The starting (base) address of the region.
153 <size> The size (in bytes) of the region.
154 [RETURNS] The index of the region on success, else -1 on error.
155*/
156{
157 int i, max;
158 mtrr_type ltype;
Jan Beulich365bff82006-12-07 02:14:09 +0100159 unsigned long lbase, lsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160
161 max = num_var_ranges;
Jan Beulich365bff82006-12-07 02:14:09 +0100162 if (replace_reg >= 0 && replace_reg < max)
163 return replace_reg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 for (i = 0; i < max; ++i) {
165 mtrr_if->get(i, &lbase, &lsize, &ltype);
166 if (lsize == 0)
167 return i;
168 }
169 return -ENOSPC;
170}
171
Adrian Bunk408b6642005-05-01 08:59:29 -0700172static void generic_get_mtrr(unsigned int reg, unsigned long *base,
Jan Beulich365bff82006-12-07 02:14:09 +0100173 unsigned long *size, mtrr_type *type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174{
175 unsigned int mask_lo, mask_hi, base_lo, base_hi;
176
177 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
178 if ((mask_lo & 0x800) == 0) {
179 /* Invalid (i.e. free) range */
180 *base = 0;
181 *size = 0;
182 *type = 0;
183 return;
184 }
185
186 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
187
188 /* Work out the shifted address mask. */
189 mask_lo = size_or_mask | mask_hi << (32 - PAGE_SHIFT)
190 | mask_lo >> PAGE_SHIFT;
191
192 /* This works correctly if size is a power of two, i.e. a
193 contiguous range. */
194 *size = -mask_lo;
195 *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
196 *type = base_lo & 0xff;
197}
198
199static int set_fixed_ranges(mtrr_type * frs)
200{
201 unsigned int *p = (unsigned int *) frs;
202 int changed = FALSE;
203 int i;
204 unsigned int lo, hi;
205
206 rdmsr(MTRRfix64K_00000_MSR, lo, hi);
207 if (p[0] != lo || p[1] != hi) {
208 mtrr_wrmsr(MTRRfix64K_00000_MSR, p[0], p[1]);
209 changed = TRUE;
210 }
211
212 for (i = 0; i < 2; i++) {
213 rdmsr(MTRRfix16K_80000_MSR + i, lo, hi);
214 if (p[2 + i * 2] != lo || p[3 + i * 2] != hi) {
215 mtrr_wrmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2],
216 p[3 + i * 2]);
217 changed = TRUE;
218 }
219 }
220
221 for (i = 0; i < 8; i++) {
222 rdmsr(MTRRfix4K_C0000_MSR + i, lo, hi);
223 if (p[6 + i * 2] != lo || p[7 + i * 2] != hi) {
224 mtrr_wrmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2],
225 p[7 + i * 2]);
226 changed = TRUE;
227 }
228 }
229 return changed;
230}
231
232/* Set the MSR pair relating to a var range. Returns TRUE if
233 changes are made */
234static int set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
235{
236 unsigned int lo, hi;
237 int changed = FALSE;
238
239 rdmsr(MTRRphysBase_MSR(index), lo, hi);
240 if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
Siddha, Suresh Bcf94b622005-04-16 15:25:11 -0700241 || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
242 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
244 changed = TRUE;
245 }
246
247 rdmsr(MTRRphysMask_MSR(index), lo, hi);
248
249 if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
Siddha, Suresh Bcf94b622005-04-16 15:25:11 -0700250 || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
251 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
253 changed = TRUE;
254 }
255 return changed;
256}
257
Jan Beulich365bff82006-12-07 02:14:09 +0100258static u32 deftype_lo, deftype_hi;
259
260static unsigned long set_mtrr_state(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261/* [SUMMARY] Set the MTRR state for this CPU.
262 <state> The MTRR state information to read.
263 <ctxt> Some relevant CPU context.
264 [NOTE] The CPU must already be in a safe state for MTRR changes.
265 [RETURNS] 0 if no changes made, else a mask indication what was changed.
266*/
267{
268 unsigned int i;
269 unsigned long change_mask = 0;
270
271 for (i = 0; i < num_var_ranges; i++)
272 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
273 change_mask |= MTRR_CHANGE_MASK_VARIABLE;
274
Jan Beulich365bff82006-12-07 02:14:09 +0100275 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 change_mask |= MTRR_CHANGE_MASK_FIXED;
277
278 /* Set_mtrr_restore restores the old value of MTRRdefType,
279 so to set it we fiddle with the saved value */
280 if ((deftype_lo & 0xff) != mtrr_state.def_type
281 || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
Jan Beulich365bff82006-12-07 02:14:09 +0100282 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
284 }
285
286 return change_mask;
287}
288
289
290static unsigned long cr4 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291static DEFINE_SPINLOCK(set_atomicity_lock);
292
293/*
294 * Since we are disabling the cache don't allow any interrupts - they
295 * would run extremely slow and would only increase the pain. The caller must
296 * ensure that local interrupts are disabled and are reenabled after post_set()
297 * has been called.
298 */
299
Josh Triplett182daa52006-09-25 23:32:36 -0700300static void prepare_set(void) __acquires(set_atomicity_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301{
302 unsigned long cr0;
303
304 /* Note that this is not ideal, since the cache is only flushed/disabled
305 for this CPU while the MTRRs are changed, but changing this requires
306 more invasive changes to the way the kernel boots */
307
308 spin_lock(&set_atomicity_lock);
309
310 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
311 cr0 = read_cr0() | 0x40000000; /* set CD flag */
312 write_cr0(cr0);
313 wbinvd();
314
315 /* Save value of CR4 and clear Page Global Enable (bit 7) */
316 if ( cpu_has_pge ) {
317 cr4 = read_cr4();
318 write_cr4(cr4 & ~X86_CR4_PGE);
319 }
320
321 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
322 __flush_tlb();
323
324 /* Save MTRR state */
325 rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
326
327 /* Disable MTRRs, and set the default type to uncached */
Jan Beulich365bff82006-12-07 02:14:09 +0100328 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329}
330
Josh Triplett182daa52006-09-25 23:32:36 -0700331static void post_set(void) __releases(set_atomicity_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
333 /* Flush TLBs (no need to flush caches - they are disabled) */
334 __flush_tlb();
335
336 /* Intel (P6) standard MTRRs */
337 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
338
339 /* Enable caches */
340 write_cr0(read_cr0() & 0xbfffffff);
341
342 /* Restore value of CR4 */
343 if ( cpu_has_pge )
344 write_cr4(cr4);
345 spin_unlock(&set_atomicity_lock);
346}
347
348static void generic_set_all(void)
349{
350 unsigned long mask, count;
351 unsigned long flags;
352
353 local_irq_save(flags);
354 prepare_set();
355
356 /* Actually set the state */
Jan Beulich365bff82006-12-07 02:14:09 +0100357 mask = set_mtrr_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358
359 post_set();
360 local_irq_restore(flags);
361
362 /* Use the atomic bitops to update the global mask */
363 for (count = 0; count < sizeof mask * 8; ++count) {
364 if (mask & 0x01)
365 set_bit(count, &smp_changes_mask);
366 mask >>= 1;
367 }
368
369}
370
371static void generic_set_mtrr(unsigned int reg, unsigned long base,
372 unsigned long size, mtrr_type type)
373/* [SUMMARY] Set variable MTRR register on the local CPU.
374 <reg> The register to set.
375 <base> The base address of the region.
376 <size> The size of the region. If this is 0 the region is disabled.
377 <type> The type of the region.
378 <do_safe> If TRUE, do the change safely. If FALSE, safety measures should
379 be done externally.
380 [RETURNS] Nothing.
381*/
382{
383 unsigned long flags;
Shaohua Li3b520b22005-07-07 17:56:38 -0700384 struct mtrr_var_range *vr;
385
386 vr = &mtrr_state.var_ranges[reg];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
388 local_irq_save(flags);
389 prepare_set();
390
391 if (size == 0) {
392 /* The invalid bit is kept in the mask, so we simply clear the
393 relevant mask register to disable a range. */
394 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
Shaohua Li3b520b22005-07-07 17:56:38 -0700395 memset(vr, 0, sizeof(struct mtrr_var_range));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 } else {
Shaohua Li3b520b22005-07-07 17:56:38 -0700397 vr->base_lo = base << PAGE_SHIFT | type;
398 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
399 vr->mask_lo = -size << PAGE_SHIFT | 0x800;
400 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
401
402 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
403 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 }
405
406 post_set();
407 local_irq_restore(flags);
408}
409
410int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
411{
412 unsigned long lbase, last;
413
414 /* For Intel PPro stepping <= 7, must be 4 MiB aligned
415 and not touch 0x70000000->0x7003FFFF */
416 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
417 boot_cpu_data.x86_model == 1 &&
418 boot_cpu_data.x86_mask <= 7) {
419 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
420 printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
421 return -EINVAL;
422 }
Andreas Mohr9b483412006-12-07 02:14:00 +0100423 if (!(base + size < 0x70000 || base > 0x7003F) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 (type == MTRR_TYPE_WRCOMB
425 || type == MTRR_TYPE_WRBACK)) {
426 printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
427 return -EINVAL;
428 }
429 }
430
Jan Beulich365bff82006-12-07 02:14:09 +0100431 if (base + size < 0x100) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 printk(KERN_WARNING "mtrr: cannot set region below 1 MiB (0x%lx000,0x%lx000)\n",
433 base, size);
434 return -EINVAL;
435 }
436 /* Check upper bits of base and last are equal and lower bits are 0
437 for base and 1 for last */
438 last = base + size - 1;
439 for (lbase = base; !(lbase & 1) && (last & 1);
440 lbase = lbase >> 1, last = last >> 1) ;
441 if (lbase != last) {
442 printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
443 base, size);
444 return -EINVAL;
445 }
446 return 0;
447}
448
449
450static int generic_have_wrcomb(void)
451{
452 unsigned long config, dummy;
453 rdmsr(MTRRcap_MSR, config, dummy);
454 return (config & (1 << 10));
455}
456
457int positive_have_wrcomb(void)
458{
459 return 1;
460}
461
462/* generic structure...
463 */
464struct mtrr_ops generic_mtrr_ops = {
465 .use_intel_if = 1,
466 .set_all = generic_set_all,
467 .get = generic_get_mtrr,
468 .get_free_region = generic_get_free_region,
469 .set = generic_set_mtrr,
470 .validate_add_page = generic_validate_add_page,
471 .have_wrcomb = generic_have_wrcomb,
472};