?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
amd64_edac: Cleanup old defines cruft
[linux-2.6/libata-dev.git] / drivers / edac / amd64_edac.c
blob729d9f1aecb9c4d2e158e3d2fb064d02783dd4de
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21 static atomic_t drv_instances = ATOMIC_INIT(0);
22
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
26
27 /*
28 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and
29 * later.
30 */
31 static int ddr2_dbam_revCG[] = {
32 [0] = 32,
33 [1] = 64,
34 [2] = 128,
35 [3] = 256,
36 [4] = 512,
37 [5] = 1024,
38 [6] = 2048,
39 };
40
41 static int ddr2_dbam_revD[] = {
42 [0] = 32,
43 [1] = 64,
44 [2 ... 3] = 128,
45 [4] = 256,
46 [5] = 512,
47 [6] = 256,
48 [7] = 512,
49 [8 ... 9] = 1024,
50 [10] = 2048,
51 };
52
53 static int ddr2_dbam[] = { [0] = 128,
54 [1] = 256,
55 [2 ... 4] = 512,
56 [5 ... 6] = 1024,
57 [7 ... 8] = 2048,
58 [9 ... 10] = 4096,
59 [11] = 8192,
60 };
61
62 static int ddr3_dbam[] = { [0] = -1,
63 [1] = 256,
64 [2] = 512,
65 [3 ... 4] = -1,
66 [5 ... 6] = 1024,
67 [7 ... 8] = 2048,
68 [9 ... 10] = 4096,
69 [11] = 8192,
70 };
71
72 /*
73 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
74 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
75 * or higher value'.
76 *
77 *FIXME: Produce a better mapping/linearisation.
78 */
79
80
81 struct scrubrate {
82 u32 scrubval; /* bit pattern for scrub rate */
83 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
84 } scrubrates[] = {
85 { 0x01, 1600000000UL},
86 { 0x02, 800000000UL},
87 { 0x03, 400000000UL},
88 { 0x04, 200000000UL},
89 { 0x05, 100000000UL},
90 { 0x06, 50000000UL},
91 { 0x07, 25000000UL},
92 { 0x08, 12284069UL},
93 { 0x09, 6274509UL},
94 { 0x0A, 3121951UL},
95 { 0x0B, 1560975UL},
96 { 0x0C, 781440UL},
97 { 0x0D, 390720UL},
98 { 0x0E, 195300UL},
99 { 0x0F, 97650UL},
100 { 0x10, 48854UL},
101 { 0x11, 24427UL},
102 { 0x12, 12213UL},
103 { 0x13, 6101UL},
104 { 0x14, 3051UL},
105 { 0x15, 1523UL},
106 { 0x16, 761UL},
107 { 0x00, 0UL}, /* scrubbing off */
108 };
109
110 static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
111 u32 *val, const char *func)
112 {
113 int err = 0;
114
115 err = pci_read_config_dword(pdev, offset, val);
116 if (err)
117 amd64_warn("%s: error reading F%dx%03x.\n",
118 func, PCI_FUNC(pdev->devfn), offset);
119
120 return err;
121 }
122
123 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
124 u32 val, const char *func)
125 {
126 int err = 0;
127
128 err = pci_write_config_dword(pdev, offset, val);
129 if (err)
130 amd64_warn("%s: error writing to F%dx%03x.\n",
131 func, PCI_FUNC(pdev->devfn), offset);
132
133 return err;
134 }
135
136 /*
137 *
138 * Depending on the family, F2 DCT reads need special handling:
139 *
140 * K8: has a single DCT only
141 *
142 * F10h: each DCT has its own set of regs
143 * DCT0 -> F2x040..
144 * DCT1 -> F2x140..
145 *
146 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
147 *
148 */
149 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
150 const char *func)
151 {
152 if (addr >= 0x100)
153 return -EINVAL;
154
155 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
156 }
157
158 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
159 const char *func)
160 {
161 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
162 }
163
164 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
165 const char *func)
166 {
167 u32 reg = 0;
168 u8 dct = 0;
169
170 if (addr >= 0x140 && addr <= 0x1a0) {
171 dct = 1;
172 addr -= 0x100;
173 }
174
175 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
176 reg &= 0xfffffffe;
177 reg |= dct;
178 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
179
180 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
181 }
182
183 /*
184 * Memory scrubber control interface. For K8, memory scrubbing is handled by
185 * hardware and can involve L2 cache, dcache as well as the main memory. With
186 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
187 * functionality.
188 *
189 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
190 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
191 * bytes/sec for the setting.
192 *
193 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
194 * other archs, we might not have access to the caches directly.
195 */
196
197 /*
198 * scan the scrub rate mapping table for a close or matching bandwidth value to
199 * issue. If requested is too big, then use last maximum value found.
200 */
201 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
202 {
203 u32 scrubval;
204 int i;
205
206 /*
207 * map the configured rate (new_bw) to a value specific to the AMD64
208 * memory controller and apply to register. Search for the first
209 * bandwidth entry that is greater or equal than the setting requested
210 * and program that. If at last entry, turn off DRAM scrubbing.
211 */
212 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
213 /*
214 * skip scrub rates which aren't recommended
215 * (see F10 BKDG, F3x58)
216 */
217 if (scrubrates[i].scrubval < min_rate)
218 continue;
219
220 if (scrubrates[i].bandwidth <= new_bw)
221 break;
222
223 /*
224 * if no suitable bandwidth found, turn off DRAM scrubbing
225 * entirely by falling back to the last element in the
226 * scrubrates array.
227 */
228 }
229
230 scrubval = scrubrates[i].scrubval;
231
232 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
233
234 if (scrubval)
235 return scrubrates[i].bandwidth;
236
237 return 0;
238 }
239
240 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
241 {
242 struct amd64_pvt *pvt = mci->pvt_info;
243
244 return __amd64_set_scrub_rate(pvt->F3, bw, pvt->min_scrubrate);
245 }
246
247 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
248 {
249 struct amd64_pvt *pvt = mci->pvt_info;
250 u32 scrubval = 0;
251 int i, retval = -EINVAL;
252
253 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
254
255 scrubval = scrubval & 0x001F;
256
257 amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval);
258
259 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
260 if (scrubrates[i].scrubval == scrubval) {
261 retval = scrubrates[i].bandwidth;
262 break;
263 }
264 }
265 return retval;
266 }
267
268 /*
269 * returns true if the SysAddr given by sys_addr matches the
270 * DRAM base/limit associated with node_id
271 */
272 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, int nid)
273 {
274 u64 addr;
275
276 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
277 * all ones if the most significant implemented address bit is 1.
278 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
279 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
280 * Application Programming.
281 */
282 addr = sys_addr & 0x000000ffffffffffull;
283
284 return ((addr >= get_dram_base(pvt, nid)) &&
285 (addr <= get_dram_limit(pvt, nid)));
286 }
287
288 /*
289 * Attempt to map a SysAddr to a node. On success, return a pointer to the
290 * mem_ctl_info structure for the node that the SysAddr maps to.
291 *
292 * On failure, return NULL.
293 */
294 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
295 u64 sys_addr)
296 {
297 struct amd64_pvt *pvt;
298 int node_id;
299 u32 intlv_en, bits;
300
301 /*
302 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
303 * 3.4.4.2) registers to map the SysAddr to a node ID.
304 */
305 pvt = mci->pvt_info;
306
307 /*
308 * The value of this field should be the same for all DRAM Base
309 * registers. Therefore we arbitrarily choose to read it from the
310 * register for node 0.
311 */
312 intlv_en = dram_intlv_en(pvt, 0);
313
314 if (intlv_en == 0) {
315 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
316 if (amd64_base_limit_match(pvt, sys_addr, node_id))
317 goto found;
318 }
319 goto err_no_match;
320 }
321
322 if (unlikely((intlv_en != 0x01) &&
323 (intlv_en != 0x03) &&
324 (intlv_en != 0x07))) {
325 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
326 return NULL;
327 }
328
329 bits = (((u32) sys_addr) >> 12) & intlv_en;
330
331 for (node_id = 0; ; ) {
332 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
333 break; /* intlv_sel field matches */
334
335 if (++node_id >= DRAM_RANGES)
336 goto err_no_match;
337 }
338
339 /* sanity test for sys_addr */
340 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
341 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
342 "range for node %d with node interleaving enabled.\n",
343 __func__, sys_addr, node_id);
344 return NULL;
345 }
346
347 found:
348 return edac_mc_find(node_id);
349
350 err_no_match:
351 debugf2("sys_addr 0x%lx doesn't match any node\n",
352 (unsigned long)sys_addr);
353
354 return NULL;
355 }
356
357 /*
358 * compute the CS base address of the @csrow on the DRAM controller @dct.
359 * For details see F2x[5C:40] in the processor's BKDG
360 */
361 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
362 u64 *base, u64 *mask)
363 {
364 u64 csbase, csmask, base_bits, mask_bits;
365 u8 addr_shift;
366
367 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
368 csbase = pvt->csels[dct].csbases[csrow];
369 csmask = pvt->csels[dct].csmasks[csrow];
370 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
371 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
372 addr_shift = 4;
373 } else {
374 csbase = pvt->csels[dct].csbases[csrow];
375 csmask = pvt->csels[dct].csmasks[csrow >> 1];
376 addr_shift = 8;
377
378 if (boot_cpu_data.x86 == 0x15)
379 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
380 else
381 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
382 }
383
384 *base = (csbase & base_bits) << addr_shift;
385
386 *mask = ~0ULL;
387 /* poke holes for the csmask */
388 *mask &= ~(mask_bits << addr_shift);
389 /* OR them in */
390 *mask |= (csmask & mask_bits) << addr_shift;
391 }
392
393 #define for_each_chip_select(i, dct, pvt) \
394 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
395
396 #define for_each_chip_select_mask(i, dct, pvt) \
397 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
398
399 /*
400 * @input_addr is an InputAddr associated with the node given by mci. Return the
401 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
402 */
403 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
404 {
405 struct amd64_pvt *pvt;
406 int csrow;
407 u64 base, mask;
408
409 pvt = mci->pvt_info;
410
411 for_each_chip_select(csrow, 0, pvt) {
412 if (!csrow_enabled(csrow, 0, pvt))
413 continue;
414
415 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
416
417 mask = ~mask;
418
419 if ((input_addr & mask) == (base & mask)) {
420 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
421 (unsigned long)input_addr, csrow,
422 pvt->mc_node_id);
423
424 return csrow;
425 }
426 }
427 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
428 (unsigned long)input_addr, pvt->mc_node_id);
429
430 return -1;
431 }
432
433 /*
434 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
435 * for the node represented by mci. Info is passed back in *hole_base,
436 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
437 * info is invalid. Info may be invalid for either of the following reasons:
438 *
439 * - The revision of the node is not E or greater. In this case, the DRAM Hole
440 * Address Register does not exist.
441 *
442 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
443 * indicating that its contents are not valid.
444 *
445 * The values passed back in *hole_base, *hole_offset, and *hole_size are
446 * complete 32-bit values despite the fact that the bitfields in the DHAR
447 * only represent bits 31-24 of the base and offset values.
448 */
449 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
450 u64 *hole_offset, u64 *hole_size)
451 {
452 struct amd64_pvt *pvt = mci->pvt_info;
453 u64 base;
454
455 /* only revE and later have the DRAM Hole Address Register */
456 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
457 debugf1(" revision %d for node %d does not support DHAR\n",
458 pvt->ext_model, pvt->mc_node_id);
459 return 1;
460 }
461
462 /* valid for Fam10h and above */
463 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
464 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
465 return 1;
466 }
467
468 if (!dhar_valid(pvt)) {
469 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
470 pvt->mc_node_id);
471 return 1;
472 }
473
474 /* This node has Memory Hoisting */
475
476 /* +------------------+--------------------+--------------------+-----
477 * | memory | DRAM hole | relocated |
478 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
479 * | | | DRAM hole |
480 * | | | [0x100000000, |
481 * | | | (0x100000000+ |
482 * | | | (0xffffffff-x))] |
483 * +------------------+--------------------+--------------------+-----
484 *
485 * Above is a diagram of physical memory showing the DRAM hole and the
486 * relocated addresses from the DRAM hole. As shown, the DRAM hole
487 * starts at address x (the base address) and extends through address
488 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
489 * addresses in the hole so that they start at 0x100000000.
490 */
491
492 base = dhar_base(pvt);
493
494 *hole_base = base;
495 *hole_size = (0x1ull << 32) - base;
496
497 if (boot_cpu_data.x86 > 0xf)
498 *hole_offset = f10_dhar_offset(pvt);
499 else
500 *hole_offset = k8_dhar_offset(pvt);
501
502 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
503 pvt->mc_node_id, (unsigned long)*hole_base,
504 (unsigned long)*hole_offset, (unsigned long)*hole_size);
505
506 return 0;
507 }
508 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
509
510 /*
511 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
512 * assumed that sys_addr maps to the node given by mci.
513 *
514 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
515 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
516 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
517 * then it is also involved in translating a SysAddr to a DramAddr. Sections
518 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
519 * These parts of the documentation are unclear. I interpret them as follows:
520 *
521 * When node n receives a SysAddr, it processes the SysAddr as follows:
522 *
523 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
524 * Limit registers for node n. If the SysAddr is not within the range
525 * specified by the base and limit values, then node n ignores the Sysaddr
526 * (since it does not map to node n). Otherwise continue to step 2 below.
527 *
528 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
529 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
530 * the range of relocated addresses (starting at 0x100000000) from the DRAM
531 * hole. If not, skip to step 3 below. Else get the value of the
532 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
533 * offset defined by this value from the SysAddr.
534 *
535 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
536 * Base register for node n. To obtain the DramAddr, subtract the base
537 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
538 */
539 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
540 {
541 struct amd64_pvt *pvt = mci->pvt_info;
542 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
543 int ret = 0;
544
545 dram_base = get_dram_base(pvt, pvt->mc_node_id);
546
547 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
548 &hole_size);
549 if (!ret) {
550 if ((sys_addr >= (1ull << 32)) &&
551 (sys_addr < ((1ull << 32) + hole_size))) {
552 /* use DHAR to translate SysAddr to DramAddr */
553 dram_addr = sys_addr - hole_offset;
554
555 debugf2("using DHAR to translate SysAddr 0x%lx to "
556 "DramAddr 0x%lx\n",
557 (unsigned long)sys_addr,
558 (unsigned long)dram_addr);
559
560 return dram_addr;
561 }
562 }
563
564 /*
565 * Translate the SysAddr to a DramAddr as shown near the start of
566 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
567 * only deals with 40-bit values. Therefore we discard bits 63-40 of
568 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
569 * discard are all 1s. Otherwise the bits we discard are all 0s. See
570 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
571 * Programmer's Manual Volume 1 Application Programming.
572 */
573 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
574
575 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
576 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
577 (unsigned long)dram_addr);
578 return dram_addr;
579 }
580
581 /*
582 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
583 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
584 * for node interleaving.
585 */
586 static int num_node_interleave_bits(unsigned intlv_en)
587 {
588 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
589 int n;
590
591 BUG_ON(intlv_en > 7);
592 n = intlv_shift_table[intlv_en];
593 return n;
594 }
595
596 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
597 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
598 {
599 struct amd64_pvt *pvt;
600 int intlv_shift;
601 u64 input_addr;
602
603 pvt = mci->pvt_info;
604
605 /*
606 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
607 * concerning translating a DramAddr to an InputAddr.
608 */
609 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
610 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
611 (dram_addr & 0xfff);
612
613 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
614 intlv_shift, (unsigned long)dram_addr,
615 (unsigned long)input_addr);
616
617 return input_addr;
618 }
619
620 /*
621 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
622 * assumed that @sys_addr maps to the node given by mci.
623 */
624 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
625 {
626 u64 input_addr;
627
628 input_addr =
629 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
630
631 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
632 (unsigned long)sys_addr, (unsigned long)input_addr);
633
634 return input_addr;
635 }
636
637
638 /*
639 * @input_addr is an InputAddr associated with the node represented by mci.
640 * Translate @input_addr to a DramAddr and return the result.
641 */
642 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
643 {
644 struct amd64_pvt *pvt;
645 int node_id, intlv_shift;
646 u64 bits, dram_addr;
647 u32 intlv_sel;
648
649 /*
650 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
651 * shows how to translate a DramAddr to an InputAddr. Here we reverse
652 * this procedure. When translating from a DramAddr to an InputAddr, the
653 * bits used for node interleaving are discarded. Here we recover these
654 * bits from the IntlvSel field of the DRAM Limit register (section
655 * 3.4.4.2) for the node that input_addr is associated with.
656 */
657 pvt = mci->pvt_info;
658 node_id = pvt->mc_node_id;
659 BUG_ON((node_id < 0) || (node_id > 7));
660
661 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
662
663 if (intlv_shift == 0) {
664 debugf1(" InputAddr 0x%lx translates to DramAddr of "
665 "same value\n", (unsigned long)input_addr);
666
667 return input_addr;
668 }
669
670 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
671 (input_addr & 0xfff);
672
673 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
674 dram_addr = bits + (intlv_sel << 12);
675
676 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
677 "(%d node interleave bits)\n", (unsigned long)input_addr,
678 (unsigned long)dram_addr, intlv_shift);
679
680 return dram_addr;
681 }
682
683 /*
684 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
685 * @dram_addr to a SysAddr.
686 */
687 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
688 {
689 struct amd64_pvt *pvt = mci->pvt_info;
690 u64 hole_base, hole_offset, hole_size, base, sys_addr;
691 int ret = 0;
692
693 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
694 &hole_size);
695 if (!ret) {
696 if ((dram_addr >= hole_base) &&
697 (dram_addr < (hole_base + hole_size))) {
698 sys_addr = dram_addr + hole_offset;
699
700 debugf1("using DHAR to translate DramAddr 0x%lx to "
701 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
702 (unsigned long)sys_addr);
703
704 return sys_addr;
705 }
706 }
707
708 base = get_dram_base(pvt, pvt->mc_node_id);
709 sys_addr = dram_addr + base;
710
711 /*
712 * The sys_addr we have computed up to this point is a 40-bit value
713 * because the k8 deals with 40-bit values. However, the value we are
714 * supposed to return is a full 64-bit physical address. The AMD
715 * x86-64 architecture specifies that the most significant implemented
716 * address bit through bit 63 of a physical address must be either all
717 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
718 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
719 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
720 * Programming.
721 */
722 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
723
724 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
725 pvt->mc_node_id, (unsigned long)dram_addr,
726 (unsigned long)sys_addr);
727
728 return sys_addr;
729 }
730
731 /*
732 * @input_addr is an InputAddr associated with the node given by mci. Translate
733 * @input_addr to a SysAddr.
734 */
735 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
736 u64 input_addr)
737 {
738 return dram_addr_to_sys_addr(mci,
739 input_addr_to_dram_addr(mci, input_addr));
740 }
741
742 /*
743 * Find the minimum and maximum InputAddr values that map to the given @csrow.
744 * Pass back these values in *input_addr_min and *input_addr_max.
745 */
746 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
747 u64 *input_addr_min, u64 *input_addr_max)
748 {
749 struct amd64_pvt *pvt;
750 u64 base, mask;
751
752 pvt = mci->pvt_info;
753 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
754
755 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
756
757 *input_addr_min = base & ~mask;
758 *input_addr_max = base | mask;
759 }
760
761 /* Map the Error address to a PAGE and PAGE OFFSET. */
762 static inline void error_address_to_page_and_offset(u64 error_address,
763 u32 *page, u32 *offset)
764 {
765 *page = (u32) (error_address >> PAGE_SHIFT);
766 *offset = ((u32) error_address) & ~PAGE_MASK;
767 }
768
769 /*
770 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
771 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
772 * of a node that detected an ECC memory error. mci represents the node that
773 * the error address maps to (possibly different from the node that detected
774 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
775 * error.
776 */
777 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
778 {
779 int csrow;
780
781 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
782
783 if (csrow == -1)
784 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
785 "address 0x%lx\n", (unsigned long)sys_addr);
786 return csrow;
787 }
788
789 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
790
791 static u16 extract_syndrome(struct err_regs *err)
792 {
793 return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00);
794 }
795
796 /*
797 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
798 * are ECC capable.
799 */
800 static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
801 {
802 u8 bit;
803 enum dev_type edac_cap = EDAC_FLAG_NONE;
804
805 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
806 ? 19
807 : 17;
808
809 if (pvt->dclr0 & BIT(bit))
810 edac_cap = EDAC_FLAG_SECDED;
811
812 return edac_cap;
813 }
814
815
816 static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt);
817
818 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
819 {
820 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
821
822 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
823 (dclr & BIT(16)) ? "un" : "",
824 (dclr & BIT(19)) ? "yes" : "no");
825
826 debugf1(" PAR/ERR parity: %s\n",
827 (dclr & BIT(8)) ? "enabled" : "disabled");
828
829 if (boot_cpu_data.x86 == 0x10)
830 debugf1(" DCT 128bit mode width: %s\n",
831 (dclr & BIT(11)) ? "128b" : "64b");
832
833 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
834 (dclr & BIT(12)) ? "yes" : "no",
835 (dclr & BIT(13)) ? "yes" : "no",
836 (dclr & BIT(14)) ? "yes" : "no",
837 (dclr & BIT(15)) ? "yes" : "no");
838 }
839
840 /* Display and decode various NB registers for debug purposes. */
841 static void dump_misc_regs(struct amd64_pvt *pvt)
842 {
843 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
844
845 debugf1(" NB two channel DRAM capable: %s\n",
846 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
847
848 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
849 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
850 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
851
852 amd64_dump_dramcfg_low(pvt->dclr0, 0);
853
854 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
855
856 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
857 "offset: 0x%08x\n",
858 pvt->dhar, dhar_base(pvt),
859 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
860 : f10_dhar_offset(pvt));
861
862 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
863
864 amd64_debug_display_dimm_sizes(0, pvt);
865
866 /* everything below this point is Fam10h and above */
867 if (boot_cpu_data.x86 == 0xf)
868 return;
869
870 amd64_debug_display_dimm_sizes(1, pvt);
871
872 amd64_info("using %s syndromes.\n", ((pvt->syn_type == 8) ? "x8" : "x4"));
873
874 /* Only if NOT ganged does dclr1 have valid info */
875 if (!dct_ganging_enabled(pvt))
876 amd64_dump_dramcfg_low(pvt->dclr1, 1);
877 }
878
879 /*
880 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
881 */
882 static void prep_chip_selects(struct amd64_pvt *pvt)
883 {
884 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
885 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
886 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
887 } else {
888 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
889 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
890 }
891 }
892
893 /*
894 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
895 */
896 static void read_dct_base_mask(struct amd64_pvt *pvt)
897 {
898 int cs;
899
900 prep_chip_selects(pvt);
901
902 for_each_chip_select(cs, 0, pvt) {
903 u32 reg0 = DCSB0 + (cs * 4);
904 u32 reg1 = DCSB1 + (cs * 4);
905 u32 *base0 = &pvt->csels[0].csbases[cs];
906 u32 *base1 = &pvt->csels[1].csbases[cs];
907
908 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
909 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
910 cs, *base0, reg0);
911
912 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
913 continue;
914
915 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
916 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
917 cs, *base1, reg1);
918 }
919
920 for_each_chip_select_mask(cs, 0, pvt) {
921 u32 reg0 = DCSM0 + (cs * 4);
922 u32 reg1 = DCSM1 + (cs * 4);
923 u32 *mask0 = &pvt->csels[0].csmasks[cs];
924 u32 *mask1 = &pvt->csels[1].csmasks[cs];
925
926 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
927 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
928 cs, *mask0, reg0);
929
930 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
931 continue;
932
933 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
934 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
935 cs, *mask1, reg1);
936 }
937 }
938
939 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
940 {
941 enum mem_type type;
942
943 /* F15h supports only DDR3 */
944 if (boot_cpu_data.x86 >= 0x15)
945 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
946 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
947 if (pvt->dchr0 & DDR3_MODE)
948 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
949 else
950 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
951 } else {
952 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
953 }
954
955 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
956
957 return type;
958 }
959
960 /* Get the number of DCT channels the memory controller is using. */
961 static int k8_early_channel_count(struct amd64_pvt *pvt)
962 {
963 int flag;
964
965 if (pvt->ext_model >= K8_REV_F)
966 /* RevF (NPT) and later */
967 flag = pvt->dclr0 & F10_WIDTH_128;
968 else
969 /* RevE and earlier */
970 flag = pvt->dclr0 & REVE_WIDTH_128;
971
972 /* not used */
973 pvt->dclr1 = 0;
974
975 return (flag) ? 2 : 1;
976 }
977
978 /* Extract the ERROR ADDRESS for the K8 CPUs */
979 static u64 k8_get_error_address(struct mem_ctl_info *mci,
980 struct err_regs *info)
981 {
982 return (((u64) (info->nbeah & 0xff)) << 32) +
983 (info->nbeal & ~0x03);
984 }
985
986 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
987 {
988 u32 off = range << 3;
989
990 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
991 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
992
993 if (boot_cpu_data.x86 == 0xf)
994 return;
995
996 if (!dram_rw(pvt, range))
997 return;
998
999 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1000 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1001 }
1002
1003 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
1004 struct err_regs *err_info, u64 sys_addr)
1005 {
1006 struct mem_ctl_info *src_mci;
1007 int channel, csrow;
1008 u32 page, offset;
1009 u16 syndrome;
1010
1011 syndrome = extract_syndrome(err_info);
1012
1013 /* CHIPKILL enabled */
1014 if (err_info->nbcfg & NBCFG_CHIPKILL) {
1015 channel = get_channel_from_ecc_syndrome(mci, syndrome);
1016 if (channel < 0) {
1017 /*
1018 * Syndrome didn't map, so we don't know which of the
1019 * 2 DIMMs is in error. So we need to ID 'both' of them
1020 * as suspect.
1021 */
1022 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1023 "error reporting race\n", syndrome);
1024 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1025 return;
1026 }
1027 } else {
1028 /*
1029 * non-chipkill ecc mode
1030 *
1031 * The k8 documentation is unclear about how to determine the
1032 * channel number when using non-chipkill memory. This method
1033 * was obtained from email communication with someone at AMD.
1034 * (Wish the email was placed in this comment - norsk)
1035 */
1036 channel = ((sys_addr & BIT(3)) != 0);
1037 }
1038
1039 /*
1040 * Find out which node the error address belongs to. This may be
1041 * different from the node that detected the error.
1042 */
1043 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1044 if (!src_mci) {
1045 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1046 (unsigned long)sys_addr);
1047 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1048 return;
1049 }
1050
1051 /* Now map the sys_addr to a CSROW */
1052 csrow = sys_addr_to_csrow(src_mci, sys_addr);
1053 if (csrow < 0) {
1054 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1055 } else {
1056 error_address_to_page_and_offset(sys_addr, &page, &offset);
1057
1058 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1059 channel, EDAC_MOD_STR);
1060 }
1061 }
1062
1063 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
1064 {
1065 int *dbam_map;
1066
1067 if (pvt->ext_model >= K8_REV_F)
1068 dbam_map = ddr2_dbam;
1069 else if (pvt->ext_model >= K8_REV_D)
1070 dbam_map = ddr2_dbam_revD;
1071 else
1072 dbam_map = ddr2_dbam_revCG;
1073
1074 return dbam_map[cs_mode];
1075 }
1076
1077 /*
1078 * Get the number of DCT channels in use.
1079 *
1080 * Return:
1081 * number of Memory Channels in operation
1082 * Pass back:
1083 * contents of the DCL0_LOW register
1084 */
1085 static int f10_early_channel_count(struct amd64_pvt *pvt)
1086 {
1087 int i, j, channels = 0;
1088
1089 /* If we are in 128 bit mode, then we are using 2 channels */
1090 if (pvt->dclr0 & F10_WIDTH_128) {
1091 channels = 2;
1092 return channels;
1093 }
1094
1095 /*
1096 * Need to check if in unganged mode: In such, there are 2 channels,
1097 * but they are not in 128 bit mode and thus the above 'dclr0' status
1098 * bit will be OFF.
1099 *
1100 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1101 * their CSEnable bit on. If so, then SINGLE DIMM case.
1102 */
1103 debugf0("Data width is not 128 bits - need more decoding\n");
1104
1105 /*
1106 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1107 * is more than just one DIMM present in unganged mode. Need to check
1108 * both controllers since DIMMs can be placed in either one.
1109 */
1110 for (i = 0; i < 2; i++) {
1111 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1112
1113 for (j = 0; j < 4; j++) {
1114 if (DBAM_DIMM(j, dbam) > 0) {
1115 channels++;
1116 break;
1117 }
1118 }
1119 }
1120
1121 if (channels > 2)
1122 channels = 2;
1123
1124 amd64_info("MCT channel count: %d\n", channels);
1125
1126 return channels;
1127 }
1128
1129 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
1130 {
1131 int *dbam_map;
1132
1133 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1134 dbam_map = ddr3_dbam;
1135 else
1136 dbam_map = ddr2_dbam;
1137
1138 return dbam_map[cs_mode];
1139 }
1140
1141 static u64 f10_get_error_address(struct mem_ctl_info *mci,
1142 struct err_regs *info)
1143 {
1144 return (((u64) (info->nbeah & 0xffff)) << 32) +
1145 (info->nbeal & ~0x01);
1146 }
1147
1148 static void f10_read_dram_ctl_register(struct amd64_pvt *pvt)
1149 {
1150
1151 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1152 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1153 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1154
1155 debugf0(" mode: %s, All DCTs on: %s\n",
1156 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"),
1157 (dct_dram_enabled(pvt) ? "yes" : "no"));
1158
1159 if (!dct_ganging_enabled(pvt))
1160 debugf0(" Address range split per DCT: %s\n",
1161 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1162
1163 debugf0(" data interleave for ECC: %s, "
1164 "DRAM cleared since last warm reset: %s\n",
1165 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1166 (dct_memory_cleared(pvt) ? "yes" : "no"));
1167
1168 debugf0(" channel interleave: %s, "
1169 "interleave bits selector: 0x%x\n",
1170 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1171 dct_sel_interleave_addr(pvt));
1172 }
1173
1174 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1175 }
1176
1177 /*
1178 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1179 * Interleaving Modes.
1180 */
1181 static u8 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1182 bool hi_range_sel, u8 intlv_en)
1183 {
1184 u32 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1185
1186 if (dct_ganging_enabled(pvt))
1187 return 0;
1188
1189 if (hi_range_sel)
1190 return dct_sel_high;
1191
1192 /*
1193 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1194 */
1195 if (dct_interleave_enabled(pvt)) {
1196 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1197
1198 /* return DCT select function: 0=DCT0, 1=DCT1 */
1199 if (!intlv_addr)
1200 return sys_addr >> 6 & 1;
1201
1202 if (intlv_addr & 0x2) {
1203 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1204 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1205
1206 return ((sys_addr >> shift) & 1) ^ temp;
1207 }
1208
1209 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1210 }
1211
1212 if (dct_high_range_enabled(pvt))
1213 return ~dct_sel_high & 1;
1214
1215 return 0;
1216 }
1217
1218 /* Convert the sys_addr to the normalized DCT address */
1219 static u64 f10_get_norm_dct_addr(struct amd64_pvt *pvt, int range,
1220 u64 sys_addr, bool hi_rng,
1221 u32 dct_sel_base_addr)
1222 {
1223 u64 chan_off;
1224 u64 dram_base = get_dram_base(pvt, range);
1225 u64 hole_off = f10_dhar_offset(pvt);
1226 u32 hole_valid = dhar_valid(pvt);
1227 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1228
1229 if (hi_rng) {
1230 /*
1231 * if
1232 * base address of high range is below 4Gb
1233 * (bits [47:27] at [31:11])
1234 * DRAM address space on this DCT is hoisted above 4Gb &&
1235 * sys_addr > 4Gb
1236 *
1237 * remove hole offset from sys_addr
1238 * else
1239 * remove high range offset from sys_addr
1240 */
1241 if ((!(dct_sel_base_addr >> 16) ||
1242 dct_sel_base_addr < dhar_base(pvt)) &&
1243 hole_valid &&
1244 (sys_addr >= BIT_64(32)))
1245 chan_off = hole_off;
1246 else
1247 chan_off = dct_sel_base_off;
1248 } else {
1249 /*
1250 * if
1251 * we have a valid hole &&
1252 * sys_addr > 4Gb
1253 *
1254 * remove hole
1255 * else
1256 * remove dram base to normalize to DCT address
1257 */
1258 if (hole_valid && (sys_addr >= BIT_64(32)))
1259 chan_off = hole_off;
1260 else
1261 chan_off = dram_base;
1262 }
1263
1264 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1265 }
1266
1267 /* Hack for the time being - Can we get this from BIOS?? */
1268 #define CH0SPARE_RANK 0
1269 #define CH1SPARE_RANK 1
1270
1271 /*
1272 * checks if the csrow passed in is marked as SPARED, if so returns the new
1273 * spare row
1274 */
1275 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1276 {
1277 u32 swap_done;
1278 u32 bad_dram_cs;
1279
1280 /* Depending on channel, isolate respective SPARING info */
1281 if (dct) {
1282 swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare);
1283 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare);
1284 if (swap_done && (csrow == bad_dram_cs))
1285 csrow = CH1SPARE_RANK;
1286 } else {
1287 swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare);
1288 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare);
1289 if (swap_done && (csrow == bad_dram_cs))
1290 csrow = CH0SPARE_RANK;
1291 }
1292 return csrow;
1293 }
1294
1295 /*
1296 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1297 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1298 *
1299 * Return:
1300 * -EINVAL: NOT FOUND
1301 * 0..csrow = Chip-Select Row
1302 */
1303 static int f10_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
1304 {
1305 struct mem_ctl_info *mci;
1306 struct amd64_pvt *pvt;
1307 u64 cs_base, cs_mask;
1308 int cs_found = -EINVAL;
1309 int csrow;
1310
1311 mci = mcis[nid];
1312 if (!mci)
1313 return cs_found;
1314
1315 pvt = mci->pvt_info;
1316
1317 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1318
1319 for_each_chip_select(csrow, dct, pvt) {
1320 if (!csrow_enabled(csrow, dct, pvt))
1321 continue;
1322
1323 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1324
1325 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1326 csrow, cs_base, cs_mask);
1327
1328 cs_mask = ~cs_mask;
1329
1330 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1331 "(CSBase & ~CSMask)=0x%llx\n",
1332 (in_addr & cs_mask), (cs_base & cs_mask));
1333
1334 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1335 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1336
1337 debugf1(" MATCH csrow=%d\n", cs_found);
1338 break;
1339 }
1340 }
1341 return cs_found;
1342 }
1343
1344 /* For a given @dram_range, check if @sys_addr falls within it. */
1345 static int f10_match_to_this_node(struct amd64_pvt *pvt, int range,
1346 u64 sys_addr, int *nid, int *chan_sel)
1347 {
1348 int cs_found = -EINVAL;
1349 u64 chan_addr;
1350 u32 tmp, dct_sel_base;
1351 u8 channel;
1352 bool high_range = false;
1353
1354 u8 node_id = dram_dst_node(pvt, range);
1355 u8 intlv_en = dram_intlv_en(pvt, range);
1356 u32 intlv_sel = dram_intlv_sel(pvt, range);
1357
1358 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1359 range, sys_addr, get_dram_limit(pvt, range));
1360
1361 if (intlv_en &&
1362 (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1363 return -EINVAL;
1364
1365 dct_sel_base = dct_sel_baseaddr(pvt);
1366
1367 /*
1368 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1369 * select between DCT0 and DCT1.
1370 */
1371 if (dct_high_range_enabled(pvt) &&
1372 !dct_ganging_enabled(pvt) &&
1373 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1374 high_range = true;
1375
1376 channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en);
1377
1378 chan_addr = f10_get_norm_dct_addr(pvt, range, sys_addr,
1379 high_range, dct_sel_base);
1380
1381 /* remove Node ID (in case of node interleaving) */
1382 tmp = chan_addr & 0xFC0;
1383
1384 chan_addr = ((chan_addr >> hweight8(intlv_en)) & GENMASK(12, 47)) | tmp;
1385
1386 /* remove channel interleave and hash */
1387 if (dct_interleave_enabled(pvt) &&
1388 !dct_high_range_enabled(pvt) &&
1389 !dct_ganging_enabled(pvt)) {
1390 if (dct_sel_interleave_addr(pvt) != 1)
1391 chan_addr = (chan_addr >> 1) & GENMASK(6, 63);
1392 else {
1393 tmp = chan_addr & 0xFC0;
1394 chan_addr = ((chan_addr & GENMASK(14, 63)) >> 1) | tmp;
1395 }
1396 }
1397
1398 debugf1(" (ChannelAddrLong=0x%llx)\n", chan_addr);
1399
1400 cs_found = f10_lookup_addr_in_dct(chan_addr, node_id, channel);
1401
1402 if (cs_found >= 0) {
1403 *nid = node_id;
1404 *chan_sel = channel;
1405 }
1406 return cs_found;
1407 }
1408
1409 static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1410 int *node, int *chan_sel)
1411 {
1412 int range, cs_found = -EINVAL;
1413
1414 for (range = 0; range < DRAM_RANGES; range++) {
1415
1416 if (!dram_rw(pvt, range))
1417 continue;
1418
1419 if ((get_dram_base(pvt, range) <= sys_addr) &&
1420 (get_dram_limit(pvt, range) >= sys_addr)) {
1421
1422 cs_found = f10_match_to_this_node(pvt, range,
1423 sys_addr, node,
1424 chan_sel);
1425 if (cs_found >= 0)
1426 break;
1427 }
1428 }
1429 return cs_found;
1430 }
1431
1432 /*
1433 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1434 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1435 *
1436 * The @sys_addr is usually an error address received from the hardware
1437 * (MCX_ADDR).
1438 */
1439 static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
1440 struct err_regs *err_info,
1441 u64 sys_addr)
1442 {
1443 struct amd64_pvt *pvt = mci->pvt_info;
1444 u32 page, offset;
1445 int nid, csrow, chan = 0;
1446 u16 syndrome;
1447
1448 csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1449
1450 if (csrow < 0) {
1451 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1452 return;
1453 }
1454
1455 error_address_to_page_and_offset(sys_addr, &page, &offset);
1456
1457 syndrome = extract_syndrome(err_info);
1458
1459 /*
1460 * We need the syndromes for channel detection only when we're
1461 * ganged. Otherwise @chan should already contain the channel at
1462 * this point.
1463 */
1464 if (dct_ganging_enabled(pvt))
1465 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1466
1467 if (chan >= 0)
1468 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1469 EDAC_MOD_STR);
1470 else
1471 /*
1472 * Channel unknown, report all channels on this CSROW as failed.
1473 */
1474 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1475 edac_mc_handle_ce(mci, page, offset, syndrome,
1476 csrow, chan, EDAC_MOD_STR);
1477 }
1478
1479 /*
1480 * debug routine to display the memory sizes of all logical DIMMs and its
1481 * CSROWs
1482 */
1483 static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt)
1484 {
1485 int dimm, size0, size1, factor = 0;
1486 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1487 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1488
1489 if (boot_cpu_data.x86 == 0xf) {
1490 if (pvt->dclr0 & F10_WIDTH_128)
1491 factor = 1;
1492
1493 /* K8 families < revF not supported yet */
1494 if (pvt->ext_model < K8_REV_F)
1495 return;
1496 else
1497 WARN_ON(ctrl != 0);
1498 }
1499
1500 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1501 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1502 : pvt->csels[0].csbases;
1503
1504 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1505
1506 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1507
1508 /* Dump memory sizes for DIMM and its CSROWs */
1509 for (dimm = 0; dimm < 4; dimm++) {
1510
1511 size0 = 0;
1512 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1513 size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
1514
1515 size1 = 0;
1516 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1517 size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
1518
1519 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1520 dimm * 2, size0 << factor,
1521 dimm * 2 + 1, size1 << factor);
1522 }
1523 }
1524
1525 static struct amd64_family_type amd64_family_types[] = {
1526 [K8_CPUS] = {
1527 .ctl_name = "K8",
1528 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1529 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1530 .ops = {
1531 .early_channel_count = k8_early_channel_count,
1532 .get_error_address = k8_get_error_address,
1533 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1534 .dbam_to_cs = k8_dbam_to_chip_select,
1535 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1536 }
1537 },
1538 [F10_CPUS] = {
1539 .ctl_name = "F10h",
1540 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1541 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1542 .ops = {
1543 .early_channel_count = f10_early_channel_count,
1544 .get_error_address = f10_get_error_address,
1545 .read_dram_ctl_register = f10_read_dram_ctl_register,
1546 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
1547 .dbam_to_cs = f10_dbam_to_chip_select,
1548 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1549 }
1550 },
1551 [F15_CPUS] = {
1552 .ctl_name = "F15h",
1553 .ops = {
1554 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1555 }
1556 },
1557 };
1558
1559 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1560 unsigned int device,
1561 struct pci_dev *related)
1562 {
1563 struct pci_dev *dev = NULL;
1564
1565 dev = pci_get_device(vendor, device, dev);
1566 while (dev) {
1567 if ((dev->bus->number == related->bus->number) &&
1568 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1569 break;
1570 dev = pci_get_device(vendor, device, dev);
1571 }
1572
1573 return dev;
1574 }
1575
1576 /*
1577 * These are tables of eigenvectors (one per line) which can be used for the
1578 * construction of the syndrome tables. The modified syndrome search algorithm
1579 * uses those to find the symbol in error and thus the DIMM.
1580 *
1581 * Algorithm courtesy of Ross LaFetra from AMD.
1582 */
1583 static u16 x4_vectors[] = {
1584 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1585 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1586 0x0001, 0x0002, 0x0004, 0x0008,
1587 0x1013, 0x3032, 0x4044, 0x8088,
1588 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1589 0x4857, 0xc4fe, 0x13cc, 0x3288,
1590 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1591 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1592 0x15c1, 0x2a42, 0x89ac, 0x4758,
1593 0x2b03, 0x1602, 0x4f0c, 0xca08,
1594 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1595 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1596 0x2b87, 0x164e, 0x642c, 0xdc18,
1597 0x40b9, 0x80de, 0x1094, 0x20e8,
1598 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1599 0x11c1, 0x2242, 0x84ac, 0x4c58,
1600 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1601 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1602 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1603 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1604 0x16b3, 0x3d62, 0x4f34, 0x8518,
1605 0x1e2f, 0x391a, 0x5cac, 0xf858,
1606 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1607 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1608 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1609 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1610 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1611 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1612 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1613 0x185d, 0x2ca6, 0x7914, 0x9e28,
1614 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1615 0x4199, 0x82ee, 0x19f4, 0x2e58,
1616 0x4807, 0xc40e, 0x130c, 0x3208,
1617 0x1905, 0x2e0a, 0x5804, 0xac08,
1618 0x213f, 0x132a, 0xadfc, 0x5ba8,
1619 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1620 };
1621
1622 static u16 x8_vectors[] = {
1623 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1624 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1625 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1626 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1627 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1628 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1629 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1630 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1631 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1632 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1633 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1634 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1635 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1636 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1637 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1638 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1639 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1640 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1641 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1642 };
1643
1644 static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs,
1645 int v_dim)
1646 {
1647 unsigned int i, err_sym;
1648
1649 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1650 u16 s = syndrome;
1651 int v_idx = err_sym * v_dim;
1652 int v_end = (err_sym + 1) * v_dim;
1653
1654 /* walk over all 16 bits of the syndrome */
1655 for (i = 1; i < (1U << 16); i <<= 1) {
1656
1657 /* if bit is set in that eigenvector... */
1658 if (v_idx < v_end && vectors[v_idx] & i) {
1659 u16 ev_comp = vectors[v_idx++];
1660
1661 /* ... and bit set in the modified syndrome, */
1662 if (s & i) {
1663 /* remove it. */
1664 s ^= ev_comp;
1665
1666 if (!s)
1667 return err_sym;
1668 }
1669
1670 } else if (s & i)
1671 /* can't get to zero, move to next symbol */
1672 break;
1673 }
1674 }
1675
1676 debugf0("syndrome(%x) not found\n", syndrome);
1677 return -1;
1678 }
1679
1680 static int map_err_sym_to_channel(int err_sym, int sym_size)
1681 {
1682 if (sym_size == 4)
1683 switch (err_sym) {
1684 case 0x20:
1685 case 0x21:
1686 return 0;
1687 break;
1688 case 0x22:
1689 case 0x23:
1690 return 1;
1691 break;
1692 default:
1693 return err_sym >> 4;
1694 break;
1695 }
1696 /* x8 symbols */
1697 else
1698 switch (err_sym) {
1699 /* imaginary bits not in a DIMM */
1700 case 0x10:
1701 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1702 err_sym);
1703 return -1;
1704 break;
1705
1706 case 0x11:
1707 return 0;
1708 break;
1709 case 0x12:
1710 return 1;
1711 break;
1712 default:
1713 return err_sym >> 3;
1714 break;
1715 }
1716 return -1;
1717 }
1718
1719 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1720 {
1721 struct amd64_pvt *pvt = mci->pvt_info;
1722 int err_sym = -1;
1723
1724 if (pvt->syn_type == 8)
1725 err_sym = decode_syndrome(syndrome, x8_vectors,
1726 ARRAY_SIZE(x8_vectors),
1727 pvt->syn_type);
1728 else if (pvt->syn_type == 4)
1729 err_sym = decode_syndrome(syndrome, x4_vectors,
1730 ARRAY_SIZE(x4_vectors),
1731 pvt->syn_type);
1732 else {
1733 amd64_warn("Illegal syndrome type: %u\n", pvt->syn_type);
1734 return err_sym;
1735 }
1736
1737 return map_err_sym_to_channel(err_sym, pvt->syn_type);
1738 }
1739
1740 /*
1741 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1742 * ADDRESS and process.
1743 */
1744 static void amd64_handle_ce(struct mem_ctl_info *mci,
1745 struct err_regs *info)
1746 {
1747 struct amd64_pvt *pvt = mci->pvt_info;
1748 u64 sys_addr;
1749
1750 /* Ensure that the Error Address is VALID */
1751 if (!(info->nbsh & NBSH_VALID_ERROR_ADDR)) {
1752 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1753 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1754 return;
1755 }
1756
1757 sys_addr = pvt->ops->get_error_address(mci, info);
1758
1759 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1760
1761 pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr);
1762 }
1763
1764 /* Handle any Un-correctable Errors (UEs) */
1765 static void amd64_handle_ue(struct mem_ctl_info *mci,
1766 struct err_regs *info)
1767 {
1768 struct amd64_pvt *pvt = mci->pvt_info;
1769 struct mem_ctl_info *log_mci, *src_mci = NULL;
1770 int csrow;
1771 u64 sys_addr;
1772 u32 page, offset;
1773
1774 log_mci = mci;
1775
1776 if (!(info->nbsh & NBSH_VALID_ERROR_ADDR)) {
1777 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1778 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1779 return;
1780 }
1781
1782 sys_addr = pvt->ops->get_error_address(mci, info);
1783
1784 /*
1785 * Find out which node the error address belongs to. This may be
1786 * different from the node that detected the error.
1787 */
1788 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1789 if (!src_mci) {
1790 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1791 (unsigned long)sys_addr);
1792 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1793 return;
1794 }
1795
1796 log_mci = src_mci;
1797
1798 csrow = sys_addr_to_csrow(log_mci, sys_addr);
1799 if (csrow < 0) {
1800 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1801 (unsigned long)sys_addr);
1802 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1803 } else {
1804 error_address_to_page_and_offset(sys_addr, &page, &offset);
1805 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1806 }
1807 }
1808
1809 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1810 struct err_regs *info)
1811 {
1812 u16 ec = EC(info->nbsl);
1813 u8 xec = XEC(info->nbsl, 0x1f);
1814 int ecc_type = (info->nbsh >> 13) & 0x3;
1815
1816 /* Bail early out if this was an 'observed' error */
1817 if (PP(ec) == NBSL_PP_OBS)
1818 return;
1819
1820 /* Do only ECC errors */
1821 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1822 return;
1823
1824 if (ecc_type == 2)
1825 amd64_handle_ce(mci, info);
1826 else if (ecc_type == 1)
1827 amd64_handle_ue(mci, info);
1828 }
1829
1830 void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
1831 {
1832 struct mem_ctl_info *mci = mcis[node_id];
1833 struct err_regs regs;
1834
1835 regs.nbsl = (u32) m->status;
1836 regs.nbsh = (u32)(m->status >> 32);
1837 regs.nbeal = (u32) m->addr;
1838 regs.nbeah = (u32)(m->addr >> 32);
1839 regs.nbcfg = nbcfg;
1840
1841 __amd64_decode_bus_error(mci, &regs);
1842 }
1843
1844 /*
1845 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1846 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1847 */
1848 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1849 {
1850 /* Reserve the ADDRESS MAP Device */
1851 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1852 if (!pvt->F1) {
1853 amd64_err("error address map device not found: "
1854 "vendor %x device 0x%x (broken BIOS?)\n",
1855 PCI_VENDOR_ID_AMD, f1_id);
1856 return -ENODEV;
1857 }
1858
1859 /* Reserve the MISC Device */
1860 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1861 if (!pvt->F3) {
1862 pci_dev_put(pvt->F1);
1863 pvt->F1 = NULL;
1864
1865 amd64_err("error F3 device not found: "
1866 "vendor %x device 0x%x (broken BIOS?)\n",
1867 PCI_VENDOR_ID_AMD, f3_id);
1868
1869 return -ENODEV;
1870 }
1871 debugf1("F1: %s\n", pci_name(pvt->F1));
1872 debugf1("F2: %s\n", pci_name(pvt->F2));
1873 debugf1("F3: %s\n", pci_name(pvt->F3));
1874
1875 return 0;
1876 }
1877
1878 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
1879 {
1880 pci_dev_put(pvt->F1);
1881 pci_dev_put(pvt->F3);
1882 }
1883
1884 /*
1885 * Retrieve the hardware registers of the memory controller (this includes the
1886 * 'Address Map' and 'Misc' device regs)
1887 */
1888 static void read_mc_regs(struct amd64_pvt *pvt)
1889 {
1890 u64 msr_val;
1891 u32 tmp;
1892 int range;
1893
1894 /*
1895 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1896 * those are Read-As-Zero
1897 */
1898 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1899 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
1900
1901 /* check first whether TOP_MEM2 is enabled */
1902 rdmsrl(MSR_K8_SYSCFG, msr_val);
1903 if (msr_val & (1U << 21)) {
1904 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1905 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
1906 } else
1907 debugf0(" TOP_MEM2 disabled.\n");
1908
1909 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
1910
1911 if (pvt->ops->read_dram_ctl_register)
1912 pvt->ops->read_dram_ctl_register(pvt);
1913
1914 for (range = 0; range < DRAM_RANGES; range++) {
1915 u8 rw;
1916
1917 /* read settings for this DRAM range */
1918 read_dram_base_limit_regs(pvt, range);
1919
1920 rw = dram_rw(pvt, range);
1921 if (!rw)
1922 continue;
1923
1924 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1925 range,
1926 get_dram_base(pvt, range),
1927 get_dram_limit(pvt, range));
1928
1929 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1930 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1931 (rw & 0x1) ? "R" : "-",
1932 (rw & 0x2) ? "W" : "-",
1933 dram_intlv_sel(pvt, range),
1934 dram_dst_node(pvt, range));
1935 }
1936
1937 read_dct_base_mask(pvt);
1938
1939 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
1940 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
1941
1942 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
1943
1944 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1945 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
1946
1947 if (!dct_ganging_enabled(pvt)) {
1948 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
1949 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
1950 }
1951
1952 if (boot_cpu_data.x86 >= 0x10) {
1953 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
1954 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
1955 }
1956
1957 if (boot_cpu_data.x86 == 0x10 &&
1958 boot_cpu_data.x86_model > 7 &&
1959 /* F3x180[EccSymbolSize]=1 => x8 symbols */
1960 tmp & BIT(25))
1961 pvt->syn_type = 8;
1962 else
1963 pvt->syn_type = 4;
1964
1965 dump_misc_regs(pvt);
1966 }
1967
1968 /*
1969 * NOTE: CPU Revision Dependent code
1970 *
1971 * Input:
1972 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
1973 * k8 private pointer to -->
1974 * DRAM Bank Address mapping register
1975 * node_id
1976 * DCL register where dual_channel_active is
1977 *
1978 * The DBAM register consists of 4 sets of 4 bits each definitions:
1979 *
1980 * Bits: CSROWs
1981 * 0-3 CSROWs 0 and 1
1982 * 4-7 CSROWs 2 and 3
1983 * 8-11 CSROWs 4 and 5
1984 * 12-15 CSROWs 6 and 7
1985 *
1986 * Values range from: 0 to 15
1987 * The meaning of the values depends on CPU revision and dual-channel state,
1988 * see relevant BKDG more info.
1989 *
1990 * The memory controller provides for total of only 8 CSROWs in its current
1991 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
1992 * single channel or two (2) DIMMs in dual channel mode.
1993 *
1994 * The following code logic collapses the various tables for CSROW based on CPU
1995 * revision.
1996 *
1997 * Returns:
1998 * The number of PAGE_SIZE pages on the specified CSROW number it
1999 * encompasses
2000 *
2001 */
2002 static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt)
2003 {
2004 u32 cs_mode, nr_pages;
2005
2006 /*
2007 * The math on this doesn't look right on the surface because x/2*4 can
2008 * be simplified to x*2 but this expression makes use of the fact that
2009 * it is integral math where 1/2=0. This intermediate value becomes the
2010 * number of bits to shift the DBAM register to extract the proper CSROW
2011 * field.
2012 */
2013 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2014
2015 nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT);
2016
2017 /*
2018 * If dual channel then double the memory size of single channel.
2019 * Channel count is 1 or 2
2020 */
2021 nr_pages <<= (pvt->channel_count - 1);
2022
2023 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2024 debugf0(" nr_pages= %u channel-count = %d\n",
2025 nr_pages, pvt->channel_count);
2026
2027 return nr_pages;
2028 }
2029
2030 /*
2031 * Initialize the array of csrow attribute instances, based on the values
2032 * from pci config hardware registers.
2033 */
2034 static int init_csrows(struct mem_ctl_info *mci)
2035 {
2036 struct csrow_info *csrow;
2037 struct amd64_pvt *pvt = mci->pvt_info;
2038 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
2039 u32 val;
2040 int i, empty = 1;
2041
2042 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2043
2044 pvt->nbcfg = val;
2045
2046 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2047 pvt->mc_node_id, val,
2048 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2049
2050 for_each_chip_select(i, 0, pvt) {
2051 csrow = &mci->csrows[i];
2052
2053 if (!csrow_enabled(i, 0, pvt)) {
2054 debugf1("----CSROW %d EMPTY for node %d\n", i,
2055 pvt->mc_node_id);
2056 continue;
2057 }
2058
2059 debugf1("----CSROW %d VALID for MC node %d\n",
2060 i, pvt->mc_node_id);
2061
2062 empty = 0;
2063 csrow->nr_pages = amd64_csrow_nr_pages(i, pvt);
2064 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2065 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2066 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2067 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2068 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2069
2070 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2071 csrow->page_mask = ~mask;
2072 /* 8 bytes of resolution */
2073
2074 csrow->mtype = amd64_determine_memory_type(pvt, i);
2075
2076 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2077 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2078 (unsigned long)input_addr_min,
2079 (unsigned long)input_addr_max);
2080 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2081 (unsigned long)sys_addr, csrow->page_mask);
2082 debugf1(" nr_pages: %u first_page: 0x%lx "
2083 "last_page: 0x%lx\n",
2084 (unsigned)csrow->nr_pages,
2085 csrow->first_page, csrow->last_page);
2086
2087 /*
2088 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2089 */
2090 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2091 csrow->edac_mode =
2092 (pvt->nbcfg & NBCFG_CHIPKILL) ?
2093 EDAC_S4ECD4ED : EDAC_SECDED;
2094 else
2095 csrow->edac_mode = EDAC_NONE;
2096 }
2097
2098 return empty;
2099 }
2100
2101 /* get all cores on this DCT */
2102 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid)
2103 {
2104 int cpu;
2105
2106 for_each_online_cpu(cpu)
2107 if (amd_get_nb_id(cpu) == nid)
2108 cpumask_set_cpu(cpu, mask);
2109 }
2110
2111 /* check MCG_CTL on all the cpus on this node */
2112 static bool amd64_nb_mce_bank_enabled_on_node(int nid)
2113 {
2114 cpumask_var_t mask;
2115 int cpu, nbe;
2116 bool ret = false;
2117
2118 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2119 amd64_warn("%s: Error allocating mask\n", __func__);
2120 return false;
2121 }
2122
2123 get_cpus_on_this_dct_cpumask(mask, nid);
2124
2125 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2126
2127 for_each_cpu(cpu, mask) {
2128 struct msr *reg = per_cpu_ptr(msrs, cpu);
2129 nbe = reg->l & MSR_MCGCTL_NBE;
2130
2131 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2132 cpu, reg->q,
2133 (nbe ? "enabled" : "disabled"));
2134
2135 if (!nbe)
2136 goto out;
2137 }
2138 ret = true;
2139
2140 out:
2141 free_cpumask_var(mask);
2142 return ret;
2143 }
2144
2145 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2146 {
2147 cpumask_var_t cmask;
2148 int cpu;
2149
2150 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2151 amd64_warn("%s: error allocating mask\n", __func__);
2152 return false;
2153 }
2154
2155 get_cpus_on_this_dct_cpumask(cmask, nid);
2156
2157 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2158
2159 for_each_cpu(cpu, cmask) {
2160
2161 struct msr *reg = per_cpu_ptr(msrs, cpu);
2162
2163 if (on) {
2164 if (reg->l & MSR_MCGCTL_NBE)
2165 s->flags.nb_mce_enable = 1;
2166
2167 reg->l |= MSR_MCGCTL_NBE;
2168 } else {
2169 /*
2170 * Turn off NB MCE reporting only when it was off before
2171 */
2172 if (!s->flags.nb_mce_enable)
2173 reg->l &= ~MSR_MCGCTL_NBE;
2174 }
2175 }
2176 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2177
2178 free_cpumask_var(cmask);
2179
2180 return 0;
2181 }
2182
2183 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2184 struct pci_dev *F3)
2185 {
2186 bool ret = true;
2187 u32 value, mask = 0x3; /* UECC/CECC enable */
2188
2189 if (toggle_ecc_err_reporting(s, nid, ON)) {
2190 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2191 return false;
2192 }
2193
2194 amd64_read_pci_cfg(F3, NBCTL, &value);
2195
2196 s->old_nbctl = value & mask;
2197 s->nbctl_valid = true;
2198
2199 value |= mask;
2200 amd64_write_pci_cfg(F3, NBCTL, value);
2201
2202 amd64_read_pci_cfg(F3, NBCFG, &value);
2203
2204 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2205 nid, value, !!(value & NBCFG_ECC_ENABLE));
2206
2207 if (!(value & NBCFG_ECC_ENABLE)) {
2208 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2209
2210 s->flags.nb_ecc_prev = 0;
2211
2212 /* Attempt to turn on DRAM ECC Enable */
2213 value |= NBCFG_ECC_ENABLE;
2214 amd64_write_pci_cfg(F3, NBCFG, value);
2215
2216 amd64_read_pci_cfg(F3, NBCFG, &value);
2217
2218 if (!(value & NBCFG_ECC_ENABLE)) {
2219 amd64_warn("Hardware rejected DRAM ECC enable,"
2220 "check memory DIMM configuration.\n");
2221 ret = false;
2222 } else {
2223 amd64_info("Hardware accepted DRAM ECC Enable\n");
2224 }
2225 } else {
2226 s->flags.nb_ecc_prev = 1;
2227 }
2228
2229 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2230 nid, value, !!(value & NBCFG_ECC_ENABLE));
2231
2232 return ret;
2233 }
2234
2235 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2236 struct pci_dev *F3)
2237 {
2238 u32 value, mask = 0x3; /* UECC/CECC enable */
2239
2240
2241 if (!s->nbctl_valid)
2242 return;
2243
2244 amd64_read_pci_cfg(F3, NBCTL, &value);
2245 value &= ~mask;
2246 value |= s->old_nbctl;
2247
2248 amd64_write_pci_cfg(F3, NBCTL, value);
2249
2250 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2251 if (!s->flags.nb_ecc_prev) {
2252 amd64_read_pci_cfg(F3, NBCFG, &value);
2253 value &= ~NBCFG_ECC_ENABLE;
2254 amd64_write_pci_cfg(F3, NBCFG, value);
2255 }
2256
2257 /* restore the NB Enable MCGCTL bit */
2258 if (toggle_ecc_err_reporting(s, nid, OFF))
2259 amd64_warn("Error restoring NB MCGCTL settings!\n");
2260 }
2261
2262 /*
2263 * EDAC requires that the BIOS have ECC enabled before
2264 * taking over the processing of ECC errors. A command line
2265 * option allows to force-enable hardware ECC later in
2266 * enable_ecc_error_reporting().
2267 */
2268 static const char *ecc_msg =
2269 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2270 " Either enable ECC checking or force module loading by setting "
2271 "'ecc_enable_override'.\n"
2272 " (Note that use of the override may cause unknown side effects.)\n";
2273
2274 static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2275 {
2276 u32 value;
2277 u8 ecc_en = 0;
2278 bool nb_mce_en = false;
2279
2280 amd64_read_pci_cfg(F3, NBCFG, &value);
2281
2282 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2283 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2284
2285 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2286 if (!nb_mce_en)
2287 amd64_notice("NB MCE bank disabled, set MSR "
2288 "0x%08x[4] on node %d to enable.\n",
2289 MSR_IA32_MCG_CTL, nid);
2290
2291 if (!ecc_en || !nb_mce_en) {
2292 amd64_notice("%s", ecc_msg);
2293 return false;
2294 }
2295 return true;
2296 }
2297
2298 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2299 ARRAY_SIZE(amd64_inj_attrs) +
2300 1];
2301
2302 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2303
2304 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2305 {
2306 unsigned int i = 0, j = 0;
2307
2308 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2309 sysfs_attrs[i] = amd64_dbg_attrs[i];
2310
2311 if (boot_cpu_data.x86 >= 0x10)
2312 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2313 sysfs_attrs[i] = amd64_inj_attrs[j];
2314
2315 sysfs_attrs[i] = terminator;
2316
2317 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2318 }
2319
2320 static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
2321 {
2322 struct amd64_pvt *pvt = mci->pvt_info;
2323
2324 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2325 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2326
2327 if (pvt->nbcap & NBCAP_SECDED)
2328 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2329
2330 if (pvt->nbcap & NBCAP_CHIPKILL)
2331 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2332
2333 mci->edac_cap = amd64_determine_edac_cap(pvt);
2334 mci->mod_name = EDAC_MOD_STR;
2335 mci->mod_ver = EDAC_AMD64_VERSION;
2336 mci->ctl_name = pvt->ctl_name;
2337 mci->dev_name = pci_name(pvt->F2);
2338 mci->ctl_page_to_phys = NULL;
2339
2340 /* memory scrubber interface */
2341 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2342 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2343 }
2344
2345 /*
2346 * returns a pointer to the family descriptor on success, NULL otherwise.
2347 */
2348 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2349 {
2350 u8 fam = boot_cpu_data.x86;
2351 struct amd64_family_type *fam_type = NULL;
2352
2353 switch (fam) {
2354 case 0xf:
2355 fam_type = &amd64_family_types[K8_CPUS];
2356 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2357 pvt->ctl_name = fam_type->ctl_name;
2358 pvt->min_scrubrate = K8_MIN_SCRUB_RATE_BITS;
2359 break;
2360 case 0x10:
2361 fam_type = &amd64_family_types[F10_CPUS];
2362 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2363 pvt->ctl_name = fam_type->ctl_name;
2364 pvt->min_scrubrate = F10_MIN_SCRUB_RATE_BITS;
2365 break;
2366
2367 default:
2368 amd64_err("Unsupported family!\n");
2369 return NULL;
2370 }
2371
2372 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2373
2374 amd64_info("%s %sdetected (node %d).\n", pvt->ctl_name,
2375 (fam == 0xf ?
2376 (pvt->ext_model >= K8_REV_F ? "revF or later "
2377 : "revE or earlier ")
2378 : ""), pvt->mc_node_id);
2379 return fam_type;
2380 }
2381
2382 static int amd64_init_one_instance(struct pci_dev *F2)
2383 {
2384 struct amd64_pvt *pvt = NULL;
2385 struct amd64_family_type *fam_type = NULL;
2386 struct mem_ctl_info *mci = NULL;
2387 int err = 0, ret;
2388 u8 nid = get_node_id(F2);
2389
2390 ret = -ENOMEM;
2391 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2392 if (!pvt)
2393 goto err_ret;
2394
2395 pvt->mc_node_id = nid;
2396 pvt->F2 = F2;
2397
2398 ret = -EINVAL;
2399 fam_type = amd64_per_family_init(pvt);
2400 if (!fam_type)
2401 goto err_free;
2402
2403 ret = -ENODEV;
2404 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2405 if (err)
2406 goto err_free;
2407
2408 read_mc_regs(pvt);
2409
2410 /*
2411 * We need to determine how many memory channels there are. Then use
2412 * that information for calculating the size of the dynamic instance
2413 * tables in the 'mci' structure.
2414 */
2415 ret = -EINVAL;
2416 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2417 if (pvt->channel_count < 0)
2418 goto err_siblings;
2419
2420 ret = -ENOMEM;
2421 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
2422 if (!mci)
2423 goto err_siblings;
2424
2425 mci->pvt_info = pvt;
2426 mci->dev = &pvt->F2->dev;
2427
2428 setup_mci_misc_attrs(mci);
2429
2430 if (init_csrows(mci))
2431 mci->edac_cap = EDAC_FLAG_NONE;
2432
2433 set_mc_sysfs_attrs(mci);
2434
2435 ret = -ENODEV;
2436 if (edac_mc_add_mc(mci)) {
2437 debugf1("failed edac_mc_add_mc()\n");
2438 goto err_add_mc;
2439 }
2440
2441 /* register stuff with EDAC MCE */
2442 if (report_gart_errors)
2443 amd_report_gart_errors(true);
2444
2445 amd_register_ecc_decoder(amd64_decode_bus_error);
2446
2447 mcis[nid] = mci;
2448
2449 atomic_inc(&drv_instances);
2450
2451 return 0;
2452
2453 err_add_mc:
2454 edac_mc_free(mci);
2455
2456 err_siblings:
2457 free_mc_sibling_devs(pvt);
2458
2459 err_free:
2460 kfree(pvt);
2461
2462 err_ret:
2463 return ret;
2464 }
2465
2466 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2467 const struct pci_device_id *mc_type)
2468 {
2469 u8 nid = get_node_id(pdev);
2470 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2471 struct ecc_settings *s;
2472 int ret = 0;
2473
2474 ret = pci_enable_device(pdev);
2475 if (ret < 0) {
2476 debugf0("ret=%d\n", ret);
2477 return -EIO;
2478 }
2479
2480 ret = -ENOMEM;
2481 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2482 if (!s)
2483 goto err_out;
2484
2485 ecc_stngs[nid] = s;
2486
2487 if (!ecc_enabled(F3, nid)) {
2488 ret = -ENODEV;
2489
2490 if (!ecc_enable_override)
2491 goto err_enable;
2492
2493 amd64_warn("Forcing ECC on!\n");
2494
2495 if (!enable_ecc_error_reporting(s, nid, F3))
2496 goto err_enable;
2497 }
2498
2499 ret = amd64_init_one_instance(pdev);
2500 if (ret < 0) {
2501 amd64_err("Error probing instance: %d\n", nid);
2502 restore_ecc_error_reporting(s, nid, F3);
2503 }
2504
2505 return ret;
2506
2507 err_enable:
2508 kfree(s);
2509 ecc_stngs[nid] = NULL;
2510
2511 err_out:
2512 return ret;
2513 }
2514
2515 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2516 {
2517 struct mem_ctl_info *mci;
2518 struct amd64_pvt *pvt;
2519 u8 nid = get_node_id(pdev);
2520 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2521 struct ecc_settings *s = ecc_stngs[nid];
2522
2523 /* Remove from EDAC CORE tracking list */
2524 mci = edac_mc_del_mc(&pdev->dev);
2525 if (!mci)
2526 return;
2527
2528 pvt = mci->pvt_info;
2529
2530 restore_ecc_error_reporting(s, nid, F3);
2531
2532 free_mc_sibling_devs(pvt);
2533
2534 /* unregister from EDAC MCE */
2535 amd_report_gart_errors(false);
2536 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2537
2538 kfree(ecc_stngs[nid]);
2539 ecc_stngs[nid] = NULL;
2540
2541 /* Free the EDAC CORE resources */
2542 mci->pvt_info = NULL;
2543 mcis[nid] = NULL;
2544
2545 kfree(pvt);
2546 edac_mc_free(mci);
2547 }
2548
2549 /*
2550 * This table is part of the interface for loading drivers for PCI devices. The
2551 * PCI core identifies what devices are on a system during boot, and then
2552 * inquiry this table to see if this driver is for a given device found.
2553 */
2554 static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2555 {
2556 .vendor = PCI_VENDOR_ID_AMD,
2557 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2558 .subvendor = PCI_ANY_ID,
2559 .subdevice = PCI_ANY_ID,
2560 .class = 0,
2561 .class_mask = 0,
2562 },
2563 {
2564 .vendor = PCI_VENDOR_ID_AMD,
2565 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2566 .subvendor = PCI_ANY_ID,
2567 .subdevice = PCI_ANY_ID,
2568 .class = 0,
2569 .class_mask = 0,
2570 },
2571 {0, }
2572 };
2573 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2574
2575 static struct pci_driver amd64_pci_driver = {
2576 .name = EDAC_MOD_STR,
2577 .probe = amd64_probe_one_instance,
2578 .remove = __devexit_p(amd64_remove_one_instance),
2579 .id_table = amd64_pci_table,
2580 };
2581
2582 static void setup_pci_device(void)
2583 {
2584 struct mem_ctl_info *mci;
2585 struct amd64_pvt *pvt;
2586
2587 if (amd64_ctl_pci)
2588 return;
2589
2590 mci = mcis[0];
2591 if (mci) {
2592
2593 pvt = mci->pvt_info;
2594 amd64_ctl_pci =
2595 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2596
2597 if (!amd64_ctl_pci) {
2598 pr_warning("%s(): Unable to create PCI control\n",
2599 __func__);
2600
2601 pr_warning("%s(): PCI error report via EDAC not set\n",
2602 __func__);
2603 }
2604 }
2605 }
2606
2607 static int __init amd64_edac_init(void)
2608 {
2609 int err = -ENODEV;
2610
2611 edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n");
2612
2613 opstate_init();
2614
2615 if (amd_cache_northbridges() < 0)
2616 goto err_ret;
2617
2618 err = -ENOMEM;
2619 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2620 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2621 if (!(mcis && ecc_stngs))
2622 goto err_ret;
2623
2624 msrs = msrs_alloc();
2625 if (!msrs)
2626 goto err_free;
2627
2628 err = pci_register_driver(&amd64_pci_driver);
2629 if (err)
2630 goto err_pci;
2631
2632 err = -ENODEV;
2633 if (!atomic_read(&drv_instances))
2634 goto err_no_instances;
2635
2636 setup_pci_device();
2637 return 0;
2638
2639 err_no_instances:
2640 pci_unregister_driver(&amd64_pci_driver);
2641
2642 err_pci:
2643 msrs_free(msrs);
2644 msrs = NULL;
2645
2646 err_free:
2647 kfree(mcis);
2648 mcis = NULL;
2649
2650 kfree(ecc_stngs);
2651 ecc_stngs = NULL;
2652
2653 err_ret:
2654 return err;
2655 }
2656
2657 static void __exit amd64_edac_exit(void)
2658 {
2659 if (amd64_ctl_pci)
2660 edac_pci_release_generic_ctl(amd64_ctl_pci);
2661
2662 pci_unregister_driver(&amd64_pci_driver);
2663
2664 kfree(ecc_stngs);
2665 ecc_stngs = NULL;
2666
2667 kfree(mcis);
2668 mcis = NULL;
2669
2670 msrs_free(msrs);
2671 msrs = NULL;
2672 }
2673
2674 module_init(amd64_edac_init);
2675 module_exit(amd64_edac_exit);
2676
2677 MODULE_LICENSE("GPL");
2678 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2679 "Dave Peterson, Thayne Harbaugh");
2680 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2681 EDAC_AMD64_VERSION);
2682
2683 module_param(edac_op_state, int, 0444);
2684 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
Linux 2.6 libata-dev (mirror)