blob: 3bdb95ae8c430fa8bacc76a9f644c4abce8ec35e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Intel specific MCE features.
3 * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
Andi Kleen88ccbed2009-02-12 13:49:36 +01004 * Copyright (C) 2008, 2009 Intel Corporation
5 * Author: Andi Kleen
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 */
7
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09008#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/interrupt.h>
10#include <linux/percpu.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040011#include <linux/sched.h>
Chen, Gong27f6c572014-03-27 21:24:36 -040012#include <linux/cpumask.h>
H. Peter Anvin1bf7b312009-06-17 08:31:15 -070013#include <asm/apic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <asm/processor.h>
15#include <asm/msr.h>
16#include <asm/mce.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017
Chen Gong55babd82012-08-09 11:44:51 -070018#include "mce-internal.h"
19
Andi Kleen88ccbed2009-02-12 13:49:36 +010020/*
21 * Support for Intel Correct Machine Check Interrupts. This allows
22 * the CPU to raise an interrupt when a corrected machine check happened.
23 * Normally we pick those up using a regular polling timer.
24 * Also supports reliable discovery of shared banks.
25 */
26
Naveen N. Rao06444142013-06-25 23:58:59 +053027/*
28 * CMCI can be delivered to multiple cpus that share a machine check bank
29 * so we need to designate a single cpu to process errors logged in each bank
30 * in the interrupt handler (otherwise we would have many races and potential
31 * double reporting of the same error).
32 * Note that this can change when a cpu is offlined or brought online since
33 * some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
34 * disables CMCI on all banks owned by the cpu and clears this bitfield. At
35 * this point, cmci_rediscover() kicks in and a different cpu may end up
36 * taking ownership of some of the shared MCA banks that were previously
37 * owned by the offlined cpu.
38 */
Andi Kleen88ccbed2009-02-12 13:49:36 +010039static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
40
41/*
42 * cmci_discover_lock protects against parallel discovery attempts
43 * which could race against each other.
44 */
Thomas Gleixner59d958d2010-07-15 14:28:02 +020045static DEFINE_RAW_SPINLOCK(cmci_discover_lock);
Andi Kleen88ccbed2009-02-12 13:49:36 +010046
Chen Gong55babd82012-08-09 11:44:51 -070047#define CMCI_THRESHOLD 1
48#define CMCI_POLL_INTERVAL (30 * HZ)
49#define CMCI_STORM_INTERVAL (1 * HZ)
50#define CMCI_STORM_THRESHOLD 15
51
52static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
53static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
54static DEFINE_PER_CPU(unsigned int, cmci_storm_state);
55
56enum {
57 CMCI_STORM_NONE,
58 CMCI_STORM_ACTIVE,
59 CMCI_STORM_SUBSIDED,
60};
61
62static atomic_t cmci_storm_on_cpus;
Andi Kleen88ccbed2009-02-12 13:49:36 +010063
H. Peter Anvindf20e2e2009-02-24 13:19:02 -080064static int cmci_supported(int *banks)
Andi Kleen88ccbed2009-02-12 13:49:36 +010065{
66 u64 cap;
67
Borislav Petkov7af19e42012-10-15 20:25:17 +020068 if (mca_cfg.cmci_disabled || mca_cfg.ignore_ce)
Hidetoshi Seto62fdac52009-06-11 16:06:07 +090069 return 0;
70
Andi Kleen88ccbed2009-02-12 13:49:36 +010071 /*
72 * Vendor check is not strictly needed, but the initial
73 * initialization is vendor keyed and this
74 * makes sure none of the backdoors are entered otherwise.
75 */
76 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
77 return 0;
78 if (!cpu_has_apic || lapic_get_maxlvt() < 6)
79 return 0;
80 rdmsrl(MSR_IA32_MCG_CAP, cap);
81 *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
82 return !!(cap & MCG_CMCI_P);
83}
84
Chen Gong55babd82012-08-09 11:44:51 -070085void mce_intel_cmci_poll(void)
86{
87 if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
88 return;
89 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
90}
91
92void mce_intel_hcpu_update(unsigned long cpu)
93{
94 if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
95 atomic_dec(&cmci_storm_on_cpus);
96
97 per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
98}
99
100unsigned long mce_intel_adjust_timer(unsigned long interval)
101{
102 int r;
103
104 if (interval < CMCI_POLL_INTERVAL)
105 return interval;
106
107 switch (__this_cpu_read(cmci_storm_state)) {
108 case CMCI_STORM_ACTIVE:
109 /*
110 * We switch back to interrupt mode once the poll timer has
111 * silenced itself. That means no events recorded and the
112 * timer interval is back to our poll interval.
113 */
114 __this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
115 r = atomic_sub_return(1, &cmci_storm_on_cpus);
116 if (r == 0)
117 pr_notice("CMCI storm subsided: switching to interrupt mode\n");
118 /* FALLTHROUGH */
119
120 case CMCI_STORM_SUBSIDED:
121 /*
122 * We wait for all cpus to go back to SUBSIDED
123 * state. When that happens we switch back to
124 * interrupt mode.
125 */
126 if (!atomic_read(&cmci_storm_on_cpus)) {
127 __this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
128 cmci_reenable();
129 cmci_recheck();
130 }
131 return CMCI_POLL_INTERVAL;
132 default:
133 /*
134 * We have shiny weather. Let the poll do whatever it
135 * thinks.
136 */
137 return interval;
138 }
139}
140
Chen, Gong27f6c572014-03-27 21:24:36 -0400141static void cmci_storm_disable_banks(void)
142{
143 unsigned long flags, *owned;
144 int bank;
145 u64 val;
146
147 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
148 owned = __get_cpu_var(mce_banks_owned);
149 for_each_set_bit(bank, owned, MAX_NR_BANKS) {
150 rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
151 val &= ~MCI_CTL2_CMCI_EN;
152 wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
153 }
154 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
155}
156
Chen Gong55babd82012-08-09 11:44:51 -0700157static bool cmci_storm_detect(void)
158{
159 unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
160 unsigned long ts = __this_cpu_read(cmci_time_stamp);
161 unsigned long now = jiffies;
162 int r;
163
164 if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
165 return true;
166
167 if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
168 cnt++;
169 } else {
170 cnt = 1;
171 __this_cpu_write(cmci_time_stamp, now);
172 }
173 __this_cpu_write(cmci_storm_cnt, cnt);
174
175 if (cnt <= CMCI_STORM_THRESHOLD)
176 return false;
177
Chen, Gong27f6c572014-03-27 21:24:36 -0400178 cmci_storm_disable_banks();
Chen Gong55babd82012-08-09 11:44:51 -0700179 __this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
180 r = atomic_add_return(1, &cmci_storm_on_cpus);
181 mce_timer_kick(CMCI_POLL_INTERVAL);
182
183 if (r == 1)
184 pr_notice("CMCI storm detected: switching to poll mode\n");
185 return true;
186}
187
Andi Kleen88ccbed2009-02-12 13:49:36 +0100188/*
189 * The interrupt handler. This is called on every event.
190 * Just call the poller directly to log any events.
191 * This could in theory increase the threshold under high load,
192 * but doesn't for now.
193 */
194static void intel_threshold_interrupt(void)
195{
Chen Gong55babd82012-08-09 11:44:51 -0700196 if (cmci_storm_detect())
197 return;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100198 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
Andi Kleen9ff36ee2009-05-27 21:56:58 +0200199 mce_notify_irq();
Andi Kleen88ccbed2009-02-12 13:49:36 +0100200}
201
Andi Kleen88ccbed2009-02-12 13:49:36 +0100202/*
203 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
204 * on this CPU. Use the algorithm recommended in the SDM to discover shared
205 * banks.
206 */
Tony Luck4670a302012-08-09 10:59:21 -0700207static void cmci_discover(int banks)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100208{
209 unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
Hidetoshi Setoe5299922009-05-08 17:28:40 +0900210 unsigned long flags;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100211 int i;
Naveen N. Rao450cc202012-09-27 10:08:00 -0700212 int bios_wrong_thresh = 0;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100213
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200214 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100215 for (i = 0; i < banks; i++) {
216 u64 val;
Naveen N. Rao450cc202012-09-27 10:08:00 -0700217 int bios_zero_thresh = 0;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100218
219 if (test_bit(i, owned))
220 continue;
221
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530222 /* Skip banks in firmware first mode */
223 if (test_bit(i, mce_banks_ce_disabled))
224 continue;
225
Andi Kleena2d32bc2009-07-09 00:31:44 +0200226 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100227
228 /* Already owned by someone else? */
Huang Ying1f9a0bd2010-06-08 14:09:08 +0800229 if (val & MCI_CTL2_CMCI_EN) {
Tony Luck4670a302012-08-09 10:59:21 -0700230 clear_bit(i, owned);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100231 __clear_bit(i, __get_cpu_var(mce_poll_banks));
232 continue;
233 }
234
Borislav Petkov14625942012-10-17 12:05:33 +0200235 if (!mca_cfg.bios_cmci_threshold) {
Naveen N. Rao450cc202012-09-27 10:08:00 -0700236 val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
237 val |= CMCI_THRESHOLD;
238 } else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
239 /*
240 * If bios_cmci_threshold boot option was specified
241 * but the threshold is zero, we'll try to initialize
242 * it to 1.
243 */
244 bios_zero_thresh = 1;
245 val |= CMCI_THRESHOLD;
246 }
247
248 val |= MCI_CTL2_CMCI_EN;
Andi Kleena2d32bc2009-07-09 00:31:44 +0200249 wrmsrl(MSR_IA32_MCx_CTL2(i), val);
250 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100251
252 /* Did the enable bit stick? -- the bank supports CMCI */
Huang Ying1f9a0bd2010-06-08 14:09:08 +0800253 if (val & MCI_CTL2_CMCI_EN) {
Tony Luck4670a302012-08-09 10:59:21 -0700254 set_bit(i, owned);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100255 __clear_bit(i, __get_cpu_var(mce_poll_banks));
Naveen N. Rao450cc202012-09-27 10:08:00 -0700256 /*
257 * We are able to set thresholds for some banks that
258 * had a threshold of 0. This means the BIOS has not
259 * set the thresholds properly or does not work with
260 * this boot option. Note down now and report later.
261 */
Borislav Petkov14625942012-10-17 12:05:33 +0200262 if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
Naveen N. Rao450cc202012-09-27 10:08:00 -0700263 (val & MCI_CTL2_CMCI_THRESHOLD_MASK))
264 bios_wrong_thresh = 1;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100265 } else {
266 WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
267 }
268 }
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200269 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
Borislav Petkov14625942012-10-17 12:05:33 +0200270 if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
Naveen N. Rao450cc202012-09-27 10:08:00 -0700271 pr_info_once(
272 "bios_cmci_threshold: Some banks do not have valid thresholds set\n");
273 pr_info_once(
274 "bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
275 }
Andi Kleen88ccbed2009-02-12 13:49:36 +0100276}
277
278/*
279 * Just in case we missed an event during initialization check
280 * all the CMCI owned banks.
281 */
H. Peter Anvindf20e2e2009-02-24 13:19:02 -0800282void cmci_recheck(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100283{
284 unsigned long flags;
285 int banks;
286
Tejun Heo7b543a52010-12-18 16:30:05 +0100287 if (!mce_available(__this_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
Andi Kleen88ccbed2009-02-12 13:49:36 +0100288 return;
289 local_irq_save(flags);
290 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
291 local_irq_restore(flags);
292}
293
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530294/* Caller must hold the lock on cmci_discover_lock */
295static void __cmci_disable_bank(int bank)
296{
297 u64 val;
298
299 if (!test_bit(bank, __get_cpu_var(mce_banks_owned)))
300 return;
301 rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
302 val &= ~MCI_CTL2_CMCI_EN;
303 wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
304 __clear_bit(bank, __get_cpu_var(mce_banks_owned));
305}
306
Andi Kleen88ccbed2009-02-12 13:49:36 +0100307/*
308 * Disable CMCI on this CPU for all banks it owns when it goes down.
309 * This allows other CPUs to claim the banks on rediscovery.
310 */
H. Peter Anvindf20e2e2009-02-24 13:19:02 -0800311void cmci_clear(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100312{
Hidetoshi Setoe5299922009-05-08 17:28:40 +0900313 unsigned long flags;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100314 int i;
315 int banks;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100316
317 if (!cmci_supported(&banks))
318 return;
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200319 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530320 for (i = 0; i < banks; i++)
321 __cmci_disable_bank(i);
Thomas Gleixner59d958d2010-07-15 14:28:02 +0200322 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100323}
324
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530325static void cmci_rediscover_work_func(void *arg)
Tang Chen85b97632012-10-29 11:01:50 +0800326{
327 int banks;
328
329 /* Recheck banks in case CPUs don't all have the same */
330 if (cmci_supported(&banks))
331 cmci_discover(banks);
Tang Chen85b97632012-10-29 11:01:50 +0800332}
333
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530334/* After a CPU went down cycle through all the others and rediscover */
335void cmci_rediscover(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100336{
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530337 int banks;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100338
339 if (!cmci_supported(&banks))
340 return;
Andi Kleen88ccbed2009-02-12 13:49:36 +0100341
Srivatsa S. Bhat7a0c8192013-03-20 15:31:29 +0530342 on_each_cpu(cmci_rediscover_work_func, NULL, 1);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100343}
344
345/*
346 * Reenable CMCI on this CPU in case a CPU down failed.
347 */
348void cmci_reenable(void)
349{
350 int banks;
351 if (cmci_supported(&banks))
Tony Luck4670a302012-08-09 10:59:21 -0700352 cmci_discover(banks);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100353}
354
Naveen N. Raoc3d1fb52013-07-01 21:08:47 +0530355void cmci_disable_bank(int bank)
356{
357 int banks;
358 unsigned long flags;
359
360 if (!cmci_supported(&banks))
361 return;
362
363 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
364 __cmci_disable_bank(bank);
365 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
366}
367
Hidetoshi Seto514ec492009-03-16 17:07:33 +0900368static void intel_init_cmci(void)
Andi Kleen88ccbed2009-02-12 13:49:36 +0100369{
370 int banks;
371
372 if (!cmci_supported(&banks))
373 return;
374
375 mce_threshold_vector = intel_threshold_interrupt;
Tony Luck4670a302012-08-09 10:59:21 -0700376 cmci_discover(banks);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100377 /*
378 * For CPU #0 this runs with still disabled APIC, but that's
379 * ok because only the vector is set up. We still do another
380 * check for the banks later for CPU #0 just to make sure
381 * to not miss any events.
382 */
383 apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
384 cmci_recheck();
385}
386
H. Peter Anvincc3ca222009-02-20 23:35:51 -0800387void mce_intel_feature_init(struct cpuinfo_x86 *c)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388{
389 intel_init_thermal(c);
Andi Kleen88ccbed2009-02-12 13:49:36 +0100390 intel_init_cmci();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391}