blob: fb6156fee6f79dcc07559a9765f3702d7365506d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Intel specific MCE features.
3 * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
Andi Kleen88ccbed2009-02-12 13:49:36 +01004 * Copyright (C) 2008, 2009 Intel Corporation
5 * Author: Andi Kleen
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 */
7
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09008#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/interrupt.h>
10#include <linux/percpu.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040011#include <linux/sched.h>
H. Peter Anvin1bf7b312009-06-17 08:31:15 -070012#include <asm/apic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <asm/processor.h>
14#include <asm/msr.h>
15#include <asm/mce.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
Chen Gong55babd82012-08-09 11:44:51 -070017#include "mce-internal.h"
18
Andi Kleen88ccbed2009-02-12 13:49:36 +010019/*
20 * Support for Intel Correct Machine Check Interrupts. This allows
21 * the CPU to raise an interrupt when a corrected machine check happened.
22 * Normally we pick those up using a regular polling timer.
23 * Also supports reliable discovery of shared banks.
24 */
25
Naveen N. Rao06444142013-06-25 23:58:59 +053026/*
27 * CMCI can be delivered to multiple cpus that share a machine check bank
28 * so we need to designate a single cpu to process errors logged in each bank
29 * in the interrupt handler (otherwise we would have many races and potential
30 * double reporting of the same error).
31 * Note that this can change when a cpu is offlined or brought online since
32 * some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
33 * disables CMCI on all banks owned by the cpu and clears this bitfield. At
34 * this point, cmci_rediscover() kicks in and a different cpu may end up
35 * taking ownership of some of the shared MCA banks that were previously
36 * owned by the offlined cpu.
37 */
Andi Kleen88ccbed2009-02-12 13:49:36 +010038static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
39
40/*
41 * cmci_discover_lock protects against parallel discovery attempts
42 * which could race against each other.
43 */
Thomas Gleixner59d958d2010-07-15 14:28:02 +020044static DEFINE_RAW_SPINLOCK(cmci_discover_lock);
Andi Kleen88ccbed2009-02-12 13:49:36 +010045
Chen Gong55babd82012-08-09 11:44:51 -070046#define CMCI_THRESHOLD 1
47#define CMCI_POLL_INTERVAL (30 * HZ)
48#define CMCI_STORM_INTERVAL (1 * HZ)
49#define CMCI_STORM_THRESHOLD 15
50
51static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
52static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
53static DEFINE_PER_CPU(unsigned int, cmci_storm_state);
54
55enum {
56 CMCI_STORM_NONE,
57 CMCI_STORM_ACTIVE,
58 CMCI_STORM_SUBSIDED,
59};
60
61static atomic_t cmci_storm_on_cpus;
Andi Kleen88ccbed2009-02-12 13:49:36 +010062
H. Peter Anvindf20e2e2009-02-24 13:19:02 -080063static int cmci_supported(int *banks)
Andi Kleen88ccbed2009-02-12 13:49:36 +010064{
65 u64 cap;
66
Borislav Petkov7af19e42012-10-15 20:25:17 +020067 if (mca_cfg.cmci_disabled || mca_cfg.ignore_ce)
Hidetoshi Seto62fdac52009-06-11 16:06:07 +090068 return 0;
69
Andi Kleen88ccbed2009-02-12 13:49:36 +010070 /*
71 * Vendor check is not strictly needed, but the initial
72 * initialization is vendor keyed and this
73 * makes sure none of the backdoors are entered otherwise.
74 */
75 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
76 return 0;
77 if (!cpu_has_apic || lapic_get_maxlvt() < 6)
78 return 0;
79 rdmsrl(MSR_IA32_MCG_CAP, cap);
80 *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
81 return !!(cap & MCG_CMCI_P);
82}
83
Chen Gong55babd82012-08-09 11:44:51 -070084void mce_intel_cmci_poll(void)
85{
86 if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
87 return;
88 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
89}
90
91void mce_intel_hcpu_update(unsigned long cpu)
92{
93 if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
94 atomic_dec(&cmci_storm_on_cpus);
95
96 per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
97}
98
99unsigned long mce_intel_adjust_timer(unsigned long interval)
100{
101 int r;
102
103 if (interval < CMCI_POLL_INTERVAL)
104 return interval;
105
106 switch (__this_cpu_read(cmci_storm_state)) {
107 case CMCI_STORM_ACTIVE:
108 /*
109 * We switch back to interrupt mode once the poll timer has
110 * silenced itself. That means no events recorded and the
111 * timer interval is back to our poll interval.
112 */
113 __this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
114 r = atomic_sub_return(1, &cmci_storm_on_cpus);
115 if (r == 0)
116 pr_notice("CMCI storm subsided: switching to interrupt mode\n");
117 /* FALLTHROUGH */
118
119 case CMCI_STORM_SUBSIDED:
120 /*
121 * We wait for all cpus to go back to SUBSIDED
122 * state. When that happens we switch back to
123 * interrupt mode.
124 */
125 if (!atomic_read(&cmci_storm_on_cpus)) {
126 __this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
127 cmci_reenable();
128 cmci_recheck();
129 }
130 return CMCI_POLL_INTERVAL;
131 default:
132 /*
133 * We have shiny weather. Let the poll do whatever it
134 * thinks.
135 */
136 return interval;
137 }
138}
139
140static bool cmci_storm_detect(void)
141{
142 unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
143 unsigned long ts = __this_cpu_read(cmci_time_stamp);
144 unsigned long now = jiffies;
145 int r;
146
147 if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
148 return true;
149
150 if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
151 cnt++;
152 } else {
153 cnt = 1;
154 __this_cpu_write(cmci_time_stamp, now);
155 }
156 __this_cpu_write(cmci_storm_cnt, cnt);
157
158 if (cnt <= CMCI_STORM_THRESHOLD)
159 return false;
160
161 cmci_clear();
162 __this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
163 r = atomic_add_return(1, &cmci_storm_on_cpus);
164 mce_timer_kick(CMCI_POLL_INTERVAL);
165
166 if (r == 1)
167 pr_notice("CMCI storm detected: switching to poll mode\n");
168 return true;
169}
170
Andi Kleen88ccbed2009-02-12 13:49:36 +0100171/*
172 * The interrupt handler. This is called on every event.
173 * Just call the poller directly to log any events.
174 * This could in theory increase the threshold under high load,
175 * but doesn't for now.
176 */
177static void intel_threshold_interrupt(void)
178{
Chen Gong55babd82012-08-09 11:44:51 -0700179 if (cmci_storm_detect())
180 return;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100181 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
Andi Kleen9ff36ee2009-05-27 21:56:58 +0200182 mce_notify_irq();
Andi Kleen88ccbed2009-02-12 13:49:36 +0100183}
184
Andi Kleen88ccbed2009-02-12 13:49:36 +0100185/*
186 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
187 * on this CPU. Use the algorithm recommended in the SDM to discover shared
188 * banks.
189 */
Tony Luck4670a302012-08-09 10:59:21 -0700190static void cmci_discover(int banks)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100191{
192 unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
Hidetoshi Setoe5299922009-05-08 17:28:40 +0900193 unsigned long flags;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100194 int i;
Naveen N. Rao450cc202012-09-27 10:08:00 -0700195 int bios_wrong_thresh = 0;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100196
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200197 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100198 for (i = 0; i < banks; i++) {
199 u64 val;
Naveen N. Rao450cc202012-09-27 10:08:00 -0700200 int bios_zero_thresh = 0;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100201
202 if (test_bit(i, owned))
203 continue;
204
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530205 /* Skip banks in firmware first mode */
206 if (test_bit(i, mce_banks_ce_disabled))
207 continue;
208
Andi Kleena2d32bc2009-07-09 00:31:44 +0200209 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100210
211 /* Already owned by someone else? */
Huang Ying1f9a0bd2010-06-08 14:09:08 +0800212 if (val & MCI_CTL2_CMCI_EN) {
Tony Luck4670a302012-08-09 10:59:21 -0700213 clear_bit(i, owned);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100214 __clear_bit(i, __get_cpu_var(mce_poll_banks));
215 continue;
216 }
217
Borislav Petkov14625942012-10-17 12:05:33 +0200218 if (!mca_cfg.bios_cmci_threshold) {
Naveen N. Rao450cc202012-09-27 10:08:00 -0700219 val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
220 val |= CMCI_THRESHOLD;
221 } else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
222 /*
223 * If bios_cmci_threshold boot option was specified
224 * but the threshold is zero, we'll try to initialize
225 * it to 1.
226 */
227 bios_zero_thresh = 1;
228 val |= CMCI_THRESHOLD;
229 }
230
231 val |= MCI_CTL2_CMCI_EN;
Andi Kleena2d32bc2009-07-09 00:31:44 +0200232 wrmsrl(MSR_IA32_MCx_CTL2(i), val);
233 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100234
235 /* Did the enable bit stick? -- the bank supports CMCI */
Huang Ying1f9a0bd2010-06-08 14:09:08 +0800236 if (val & MCI_CTL2_CMCI_EN) {
Tony Luck4670a302012-08-09 10:59:21 -0700237 set_bit(i, owned);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100238 __clear_bit(i, __get_cpu_var(mce_poll_banks));
Naveen N. Rao450cc202012-09-27 10:08:00 -0700239 /*
240 * We are able to set thresholds for some banks that
241 * had a threshold of 0. This means the BIOS has not
242 * set the thresholds properly or does not work with
243 * this boot option. Note down now and report later.
244 */
Borislav Petkov14625942012-10-17 12:05:33 +0200245 if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
Naveen N. Rao450cc202012-09-27 10:08:00 -0700246 (val & MCI_CTL2_CMCI_THRESHOLD_MASK))
247 bios_wrong_thresh = 1;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100248 } else {
249 WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
250 }
251 }
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200252 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
Borislav Petkov14625942012-10-17 12:05:33 +0200253 if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
Naveen N. Rao450cc202012-09-27 10:08:00 -0700254 pr_info_once(
255 "bios_cmci_threshold: Some banks do not have valid thresholds set\n");
256 pr_info_once(
257 "bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
258 }
Andi Kleen88ccbed2009-02-12 13:49:36 +0100259}
260
261/*
262 * Just in case we missed an event during initialization check
263 * all the CMCI owned banks.
264 */
H. Peter Anvindf20e2e2009-02-24 13:19:02 -0800265void cmci_recheck(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100266{
267 unsigned long flags;
268 int banks;
269
Tejun Heo7b543a52010-12-18 16:30:05 +0100270 if (!mce_available(__this_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
Andi Kleen88ccbed2009-02-12 13:49:36 +0100271 return;
272 local_irq_save(flags);
273 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
274 local_irq_restore(flags);
275}
276
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530277/* Caller must hold the lock on cmci_discover_lock */
278static void __cmci_disable_bank(int bank)
279{
280 u64 val;
281
282 if (!test_bit(bank, __get_cpu_var(mce_banks_owned)))
283 return;
284 rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
285 val &= ~MCI_CTL2_CMCI_EN;
286 wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
287 __clear_bit(bank, __get_cpu_var(mce_banks_owned));
288}
289
Andi Kleen88ccbed2009-02-12 13:49:36 +0100290/*
291 * Disable CMCI on this CPU for all banks it owns when it goes down.
292 * This allows other CPUs to claim the banks on rediscovery.
293 */
H. Peter Anvindf20e2e2009-02-24 13:19:02 -0800294void cmci_clear(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100295{
Hidetoshi Setoe5299922009-05-08 17:28:40 +0900296 unsigned long flags;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100297 int i;
298 int banks;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100299
300 if (!cmci_supported(&banks))
301 return;
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200302 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530303 for (i = 0; i < banks; i++)
304 __cmci_disable_bank(i);
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200305 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100306}
307
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530308static void cmci_rediscover_work_func(void *arg)
Tang Chen85b97632012-10-29 11:01:50 +0800309{
310 int banks;
311
312 /* Recheck banks in case CPUs don't all have the same */
313 if (cmci_supported(&banks))
314 cmci_discover(banks);
Tang Chen85b97632012-10-29 11:01:50 +0800315}
316
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530317/* After a CPU went down cycle through all the others and rediscover */
318void cmci_rediscover(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100319{
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530320 int banks;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100321
322 if (!cmci_supported(&banks))
323 return;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100324
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530325 on_each_cpu(cmci_rediscover_work_func, NULL, 1);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100326}
327
328/*
329 * Reenable CMCI on this CPU in case a CPU down failed.
330 */
331void cmci_reenable(void)
332{
333 int banks;
334 if (cmci_supported(&banks))
Tony Luck4670a302012-08-09 10:59:21 -0700335 cmci_discover(banks);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100336}
337
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530338void cmci_disable_bank(int bank)
339{
340 int banks;
341 unsigned long flags;
342
343 if (!cmci_supported(&banks))
344 return;
345
346 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
347 __cmci_disable_bank(bank);
348 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
349}
350
Hidetoshi Seto514ec492009-03-16 17:07:33 +0900351static void intel_init_cmci(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100352{
353 int banks;
354
355 if (!cmci_supported(&banks))
356 return;
357
358 mce_threshold_vector = intel_threshold_interrupt;
Tony Luck4670a302012-08-09 10:59:21 -0700359 cmci_discover(banks);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100360 /*
361 * For CPU #0 this runs with still disabled APIC, but that's
362 * ok because only the vector is set up. We still do another
363 * check for the banks later for CPU #0 just to make sure
364 * to not miss any events.
365 */
366 apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
367 cmci_recheck();
368}
369
H. Peter Anvincc3ca222009-02-20 23:35:51 -0800370void mce_intel_feature_init(struct cpuinfo_x86 *c)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371{
372 intel_init_thermal(c);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100373 intel_init_cmci();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374}