1 // SPDX-License-Identifier: GPL-2.0-only
3 #include "amd64_edac.h"
4 #include <asm/amd_nb.h>
6 static struct edac_pci_ctl_info
*pci_ctl
;
9 * Set by command line parameter. If BIOS has enabled the ECC, this override is
10 * cleared to prevent re-enabling the hardware by this driver.
12 static int ecc_enable_override
;
13 module_param(ecc_enable_override
, int, 0644);
15 static struct msr __percpu
*msrs
;
17 static inline u32
get_umc_reg(struct amd64_pvt
*pvt
, u32 reg
)
19 if (!pvt
->flags
.zn_regs_v2
)
23 case UMCCH_ADDR_CFG
: return UMCCH_ADDR_CFG_DDR5
;
24 case UMCCH_ADDR_MASK_SEC
: return UMCCH_ADDR_MASK_SEC_DDR5
;
25 case UMCCH_DIMM_CFG
: return UMCCH_DIMM_CFG_DDR5
;
28 WARN_ONCE(1, "%s: unknown register 0x%x", __func__
, reg
);
33 static struct ecc_settings
**ecc_stngs
;
35 /* Device for the PCI component */
36 static struct device
*pci_ctl_dev
;
39 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
40 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
43 *FIXME: Produce a better mapping/linearisation.
45 static const struct scrubrate
{
46 u32 scrubval
; /* bit pattern for scrub rate */
47 u32 bandwidth
; /* bandwidth consumed (bytes/sec) */
49 { 0x01, 1600000000UL},
71 { 0x00, 0UL}, /* scrubbing off */
74 int __amd64_read_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
75 u32
*val
, const char *func
)
79 err
= pci_read_config_dword(pdev
, offset
, val
);
81 amd64_warn("%s: error reading F%dx%03x.\n",
82 func
, PCI_FUNC(pdev
->devfn
), offset
);
84 return pcibios_err_to_errno(err
);
87 int __amd64_write_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
88 u32 val
, const char *func
)
92 err
= pci_write_config_dword(pdev
, offset
, val
);
94 amd64_warn("%s: error writing to F%dx%03x.\n",
95 func
, PCI_FUNC(pdev
->devfn
), offset
);
97 return pcibios_err_to_errno(err
);
101 * Select DCT to which PCI cfg accesses are routed
103 static void f15h_select_dct(struct amd64_pvt
*pvt
, u8 dct
)
107 amd64_read_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, ®
);
108 reg
&= (pvt
->model
== 0x30) ? ~3 : ~1;
110 amd64_write_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, reg
);
115 * Depending on the family, F2 DCT reads need special handling:
117 * K8: has a single DCT only and no address offsets >= 0x100
119 * F10h: each DCT has its own set of regs
123 * F16h: has only 1 DCT
125 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
127 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt
*pvt
, u8 dct
,
128 int offset
, u32
*val
)
132 if (dct
|| offset
>= 0x100)
139 * Note: If ganging is enabled, barring the regs
140 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
141 * return 0. (cf. Section 2.8.1 F10h BKDG)
143 if (dct_ganging_enabled(pvt
))
152 * F15h: F2x1xx addresses do not map explicitly to DCT1.
153 * We should select which DCT we access using F1x10C[DctCfgSel]
155 dct
= (dct
&& pvt
->model
== 0x30) ? 3 : dct
;
156 f15h_select_dct(pvt
, dct
);
167 return amd64_read_pci_cfg(pvt
->F2
, offset
, val
);
171 * Memory scrubber control interface. For K8, memory scrubbing is handled by
172 * hardware and can involve L2 cache, dcache as well as the main memory. With
173 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
176 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
177 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
178 * bytes/sec for the setting.
180 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
181 * other archs, we might not have access to the caches directly.
185 * Scan the scrub rate mapping table for a close or matching bandwidth value to
186 * issue. If requested is too big, then use last maximum value found.
188 static int __set_scrub_rate(struct amd64_pvt
*pvt
, u32 new_bw
, u32 min_rate
)
194 * map the configured rate (new_bw) to a value specific to the AMD64
195 * memory controller and apply to register. Search for the first
196 * bandwidth entry that is greater or equal than the setting requested
197 * and program that. If at last entry, turn off DRAM scrubbing.
199 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
200 * by falling back to the last element in scrubrates[].
202 for (i
= 0; i
< ARRAY_SIZE(scrubrates
) - 1; i
++) {
204 * skip scrub rates which aren't recommended
205 * (see F10 BKDG, F3x58)
207 if (scrubrates
[i
].scrubval
< min_rate
)
210 if (scrubrates
[i
].bandwidth
<= new_bw
)
214 scrubval
= scrubrates
[i
].scrubval
;
216 if (pvt
->fam
== 0x15 && pvt
->model
== 0x60) {
217 f15h_select_dct(pvt
, 0);
218 pci_write_bits32(pvt
->F2
, F15H_M60H_SCRCTRL
, scrubval
, 0x001F);
219 f15h_select_dct(pvt
, 1);
220 pci_write_bits32(pvt
->F2
, F15H_M60H_SCRCTRL
, scrubval
, 0x001F);
222 pci_write_bits32(pvt
->F3
, SCRCTRL
, scrubval
, 0x001F);
226 return scrubrates
[i
].bandwidth
;
231 static int set_scrub_rate(struct mem_ctl_info
*mci
, u32 bw
)
233 struct amd64_pvt
*pvt
= mci
->pvt_info
;
234 u32 min_scrubrate
= 0x5;
239 if (pvt
->fam
== 0x15) {
241 if (pvt
->model
< 0x10)
242 f15h_select_dct(pvt
, 0);
244 if (pvt
->model
== 0x60)
247 return __set_scrub_rate(pvt
, bw
, min_scrubrate
);
250 static int get_scrub_rate(struct mem_ctl_info
*mci
)
252 struct amd64_pvt
*pvt
= mci
->pvt_info
;
253 int i
, retval
= -EINVAL
;
256 if (pvt
->fam
== 0x15) {
258 if (pvt
->model
< 0x10)
259 f15h_select_dct(pvt
, 0);
261 if (pvt
->model
== 0x60)
262 amd64_read_pci_cfg(pvt
->F2
, F15H_M60H_SCRCTRL
, &scrubval
);
264 amd64_read_pci_cfg(pvt
->F3
, SCRCTRL
, &scrubval
);
266 amd64_read_pci_cfg(pvt
->F3
, SCRCTRL
, &scrubval
);
269 scrubval
= scrubval
& 0x001F;
271 for (i
= 0; i
< ARRAY_SIZE(scrubrates
); i
++) {
272 if (scrubrates
[i
].scrubval
== scrubval
) {
273 retval
= scrubrates
[i
].bandwidth
;
281 * returns true if the SysAddr given by sys_addr matches the
282 * DRAM base/limit associated with node_id
284 static bool base_limit_match(struct amd64_pvt
*pvt
, u64 sys_addr
, u8 nid
)
288 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
289 * all ones if the most significant implemented address bit is 1.
290 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
291 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
292 * Application Programming.
294 addr
= sys_addr
& 0x000000ffffffffffull
;
296 return ((addr
>= get_dram_base(pvt
, nid
)) &&
297 (addr
<= get_dram_limit(pvt
, nid
)));
301 * Attempt to map a SysAddr to a node. On success, return a pointer to the
302 * mem_ctl_info structure for the node that the SysAddr maps to.
304 * On failure, return NULL.
306 static struct mem_ctl_info
*find_mc_by_sys_addr(struct mem_ctl_info
*mci
,
309 struct amd64_pvt
*pvt
;
314 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
315 * 3.4.4.2) registers to map the SysAddr to a node ID.
320 * The value of this field should be the same for all DRAM Base
321 * registers. Therefore we arbitrarily choose to read it from the
322 * register for node 0.
324 intlv_en
= dram_intlv_en(pvt
, 0);
327 for (node_id
= 0; node_id
< DRAM_RANGES
; node_id
++) {
328 if (base_limit_match(pvt
, sys_addr
, node_id
))
334 if (unlikely((intlv_en
!= 0x01) &&
335 (intlv_en
!= 0x03) &&
336 (intlv_en
!= 0x07))) {
337 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en
);
341 bits
= (((u32
) sys_addr
) >> 12) & intlv_en
;
343 for (node_id
= 0; ; ) {
344 if ((dram_intlv_sel(pvt
, node_id
) & intlv_en
) == bits
)
345 break; /* intlv_sel field matches */
347 if (++node_id
>= DRAM_RANGES
)
351 /* sanity test for sys_addr */
352 if (unlikely(!base_limit_match(pvt
, sys_addr
, node_id
))) {
353 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
354 "range for node %d with node interleaving enabled.\n",
355 __func__
, sys_addr
, node_id
);
360 return edac_mc_find((int)node_id
);
363 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
364 (unsigned long)sys_addr
);
370 * compute the CS base address of the @csrow on the DRAM controller @dct.
371 * For details see F2x[5C:40] in the processor's BKDG
373 static void get_cs_base_and_mask(struct amd64_pvt
*pvt
, int csrow
, u8 dct
,
374 u64
*base
, u64
*mask
)
376 u64 csbase
, csmask
, base_bits
, mask_bits
;
379 if (pvt
->fam
== 0xf && pvt
->ext_model
< K8_REV_F
) {
380 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
381 csmask
= pvt
->csels
[dct
].csmasks
[csrow
];
382 base_bits
= GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
383 mask_bits
= GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
387 * F16h and F15h, models 30h and later need two addr_shift values:
388 * 8 for high and 6 for low (cf. F16h BKDG).
390 } else if (pvt
->fam
== 0x16 ||
391 (pvt
->fam
== 0x15 && pvt
->model
>= 0x30)) {
392 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
393 csmask
= pvt
->csels
[dct
].csmasks
[csrow
>> 1];
395 *base
= (csbase
& GENMASK_ULL(15, 5)) << 6;
396 *base
|= (csbase
& GENMASK_ULL(30, 19)) << 8;
399 /* poke holes for the csmask */
400 *mask
&= ~((GENMASK_ULL(15, 5) << 6) |
401 (GENMASK_ULL(30, 19) << 8));
403 *mask
|= (csmask
& GENMASK_ULL(15, 5)) << 6;
404 *mask
|= (csmask
& GENMASK_ULL(30, 19)) << 8;
408 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
409 csmask
= pvt
->csels
[dct
].csmasks
[csrow
>> 1];
412 if (pvt
->fam
== 0x15)
413 base_bits
= mask_bits
=
414 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
416 base_bits
= mask_bits
=
417 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
420 *base
= (csbase
& base_bits
) << addr_shift
;
423 /* poke holes for the csmask */
424 *mask
&= ~(mask_bits
<< addr_shift
);
426 *mask
|= (csmask
& mask_bits
) << addr_shift
;
429 #define for_each_chip_select(i, dct, pvt) \
430 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
432 #define chip_select_base(i, dct, pvt) \
433 pvt->csels[dct].csbases[i]
435 #define for_each_chip_select_mask(i, dct, pvt) \
436 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
438 #define for_each_umc(i) \
439 for (i = 0; i < pvt->max_mcs; i++)
442 * @input_addr is an InputAddr associated with the node given by mci. Return the
443 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
445 static int input_addr_to_csrow(struct mem_ctl_info
*mci
, u64 input_addr
)
447 struct amd64_pvt
*pvt
;
453 for_each_chip_select(csrow
, 0, pvt
) {
454 if (!csrow_enabled(csrow
, 0, pvt
))
457 get_cs_base_and_mask(pvt
, csrow
, 0, &base
, &mask
);
461 if ((input_addr
& mask
) == (base
& mask
)) {
462 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
463 (unsigned long)input_addr
, csrow
,
469 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
470 (unsigned long)input_addr
, pvt
->mc_node_id
);
476 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
477 * for the node represented by mci. Info is passed back in *hole_base,
478 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
479 * info is invalid. Info may be invalid for either of the following reasons:
481 * - The revision of the node is not E or greater. In this case, the DRAM Hole
482 * Address Register does not exist.
484 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
485 * indicating that its contents are not valid.
487 * The values passed back in *hole_base, *hole_offset, and *hole_size are
488 * complete 32-bit values despite the fact that the bitfields in the DHAR
489 * only represent bits 31-24 of the base and offset values.
491 static int get_dram_hole_info(struct mem_ctl_info
*mci
, u64
*hole_base
,
492 u64
*hole_offset
, u64
*hole_size
)
494 struct amd64_pvt
*pvt
= mci
->pvt_info
;
496 /* only revE and later have the DRAM Hole Address Register */
497 if (pvt
->fam
== 0xf && pvt
->ext_model
< K8_REV_E
) {
498 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
499 pvt
->ext_model
, pvt
->mc_node_id
);
503 /* valid for Fam10h and above */
504 if (pvt
->fam
>= 0x10 && !dhar_mem_hoist_valid(pvt
)) {
505 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
509 if (!dhar_valid(pvt
)) {
510 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
515 /* This node has Memory Hoisting */
517 /* +------------------+--------------------+--------------------+-----
518 * | memory | DRAM hole | relocated |
519 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
521 * | | | [0x100000000, |
522 * | | | (0x100000000+ |
523 * | | | (0xffffffff-x))] |
524 * +------------------+--------------------+--------------------+-----
526 * Above is a diagram of physical memory showing the DRAM hole and the
527 * relocated addresses from the DRAM hole. As shown, the DRAM hole
528 * starts at address x (the base address) and extends through address
529 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
530 * addresses in the hole so that they start at 0x100000000.
533 *hole_base
= dhar_base(pvt
);
534 *hole_size
= (1ULL << 32) - *hole_base
;
536 *hole_offset
= (pvt
->fam
> 0xf) ? f10_dhar_offset(pvt
)
537 : k8_dhar_offset(pvt
);
539 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
540 pvt
->mc_node_id
, (unsigned long)*hole_base
,
541 (unsigned long)*hole_offset
, (unsigned long)*hole_size
);
546 #ifdef CONFIG_EDAC_DEBUG
547 #define EDAC_DCT_ATTR_SHOW(reg) \
548 static ssize_t reg##_show(struct device *dev, \
549 struct device_attribute *mattr, char *data) \
551 struct mem_ctl_info *mci = to_mci(dev); \
552 struct amd64_pvt *pvt = mci->pvt_info; \
554 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
557 EDAC_DCT_ATTR_SHOW(dhar
);
558 EDAC_DCT_ATTR_SHOW(dbam0
);
559 EDAC_DCT_ATTR_SHOW(top_mem
);
560 EDAC_DCT_ATTR_SHOW(top_mem2
);
562 static ssize_t
dram_hole_show(struct device
*dev
, struct device_attribute
*mattr
,
565 struct mem_ctl_info
*mci
= to_mci(dev
);
571 get_dram_hole_info(mci
, &hole_base
, &hole_offset
, &hole_size
);
573 return sprintf(data
, "%llx %llx %llx\n", hole_base
, hole_offset
,
578 * update NUM_DBG_ATTRS in case you add new members
580 static DEVICE_ATTR(dhar
, S_IRUGO
, dhar_show
, NULL
);
581 static DEVICE_ATTR(dbam
, S_IRUGO
, dbam0_show
, NULL
);
582 static DEVICE_ATTR(topmem
, S_IRUGO
, top_mem_show
, NULL
);
583 static DEVICE_ATTR(topmem2
, S_IRUGO
, top_mem2_show
, NULL
);
584 static DEVICE_ATTR_RO(dram_hole
);
586 static struct attribute
*dbg_attrs
[] = {
589 &dev_attr_topmem
.attr
,
590 &dev_attr_topmem2
.attr
,
591 &dev_attr_dram_hole
.attr
,
595 static const struct attribute_group dbg_group
= {
599 static ssize_t
inject_section_show(struct device
*dev
,
600 struct device_attribute
*mattr
, char *buf
)
602 struct mem_ctl_info
*mci
= to_mci(dev
);
603 struct amd64_pvt
*pvt
= mci
->pvt_info
;
604 return sprintf(buf
, "0x%x\n", pvt
->injection
.section
);
608 * store error injection section value which refers to one of 4 16-byte sections
609 * within a 64-byte cacheline
613 static ssize_t
inject_section_store(struct device
*dev
,
614 struct device_attribute
*mattr
,
615 const char *data
, size_t count
)
617 struct mem_ctl_info
*mci
= to_mci(dev
);
618 struct amd64_pvt
*pvt
= mci
->pvt_info
;
622 ret
= kstrtoul(data
, 10, &value
);
627 amd64_warn("%s: invalid section 0x%lx\n", __func__
, value
);
631 pvt
->injection
.section
= (u32
) value
;
635 static ssize_t
inject_word_show(struct device
*dev
,
636 struct device_attribute
*mattr
, char *buf
)
638 struct mem_ctl_info
*mci
= to_mci(dev
);
639 struct amd64_pvt
*pvt
= mci
->pvt_info
;
640 return sprintf(buf
, "0x%x\n", pvt
->injection
.word
);
644 * store error injection word value which refers to one of 9 16-bit word of the
645 * 16-byte (128-bit + ECC bits) section
649 static ssize_t
inject_word_store(struct device
*dev
,
650 struct device_attribute
*mattr
,
651 const char *data
, size_t count
)
653 struct mem_ctl_info
*mci
= to_mci(dev
);
654 struct amd64_pvt
*pvt
= mci
->pvt_info
;
658 ret
= kstrtoul(data
, 10, &value
);
663 amd64_warn("%s: invalid word 0x%lx\n", __func__
, value
);
667 pvt
->injection
.word
= (u32
) value
;
671 static ssize_t
inject_ecc_vector_show(struct device
*dev
,
672 struct device_attribute
*mattr
,
675 struct mem_ctl_info
*mci
= to_mci(dev
);
676 struct amd64_pvt
*pvt
= mci
->pvt_info
;
677 return sprintf(buf
, "0x%x\n", pvt
->injection
.bit_map
);
681 * store 16 bit error injection vector which enables injecting errors to the
682 * corresponding bit within the error injection word above. When used during a
683 * DRAM ECC read, it holds the contents of the of the DRAM ECC bits.
685 static ssize_t
inject_ecc_vector_store(struct device
*dev
,
686 struct device_attribute
*mattr
,
687 const char *data
, size_t count
)
689 struct mem_ctl_info
*mci
= to_mci(dev
);
690 struct amd64_pvt
*pvt
= mci
->pvt_info
;
694 ret
= kstrtoul(data
, 16, &value
);
698 if (value
& 0xFFFF0000) {
699 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__
, value
);
703 pvt
->injection
.bit_map
= (u32
) value
;
708 * Do a DRAM ECC read. Assemble staged values in the pvt area, format into
709 * fields needed by the injection registers and read the NB Array Data Port.
711 static ssize_t
inject_read_store(struct device
*dev
,
712 struct device_attribute
*mattr
,
713 const char *data
, size_t count
)
715 struct mem_ctl_info
*mci
= to_mci(dev
);
716 struct amd64_pvt
*pvt
= mci
->pvt_info
;
718 u32 section
, word_bits
;
721 ret
= kstrtoul(data
, 10, &value
);
725 /* Form value to choose 16-byte section of cacheline */
726 section
= F10_NB_ARRAY_DRAM
| SET_NB_ARRAY_ADDR(pvt
->injection
.section
);
728 amd64_write_pci_cfg(pvt
->F3
, F10_NB_ARRAY_ADDR
, section
);
730 word_bits
= SET_NB_DRAM_INJECTION_READ(pvt
->injection
);
732 /* Issue 'word' and 'bit' along with the READ request */
733 amd64_write_pci_cfg(pvt
->F3
, F10_NB_ARRAY_DATA
, word_bits
);
735 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section
, word_bits
);
741 * Do a DRAM ECC write. Assemble staged values in the pvt area and format into
742 * fields needed by the injection registers.
744 static ssize_t
inject_write_store(struct device
*dev
,
745 struct device_attribute
*mattr
,
746 const char *data
, size_t count
)
748 struct mem_ctl_info
*mci
= to_mci(dev
);
749 struct amd64_pvt
*pvt
= mci
->pvt_info
;
750 u32 section
, word_bits
, tmp
;
754 ret
= kstrtoul(data
, 10, &value
);
758 /* Form value to choose 16-byte section of cacheline */
759 section
= F10_NB_ARRAY_DRAM
| SET_NB_ARRAY_ADDR(pvt
->injection
.section
);
761 amd64_write_pci_cfg(pvt
->F3
, F10_NB_ARRAY_ADDR
, section
);
763 word_bits
= SET_NB_DRAM_INJECTION_WRITE(pvt
->injection
);
765 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
766 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
767 "so that you can get the error report faster.\n");
769 on_each_cpu(disable_caches
, NULL
, 1);
771 /* Issue 'word' and 'bit' along with the READ request */
772 amd64_write_pci_cfg(pvt
->F3
, F10_NB_ARRAY_DATA
, word_bits
);
775 /* wait until injection happens */
776 amd64_read_pci_cfg(pvt
->F3
, F10_NB_ARRAY_DATA
, &tmp
);
777 if (tmp
& F10_NB_ARR_ECC_WR_REQ
) {
782 on_each_cpu(enable_caches
, NULL
, 1);
784 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section
, word_bits
);
790 * update NUM_INJ_ATTRS in case you add new members
793 static DEVICE_ATTR_RW(inject_section
);
794 static DEVICE_ATTR_RW(inject_word
);
795 static DEVICE_ATTR_RW(inject_ecc_vector
);
796 static DEVICE_ATTR_WO(inject_write
);
797 static DEVICE_ATTR_WO(inject_read
);
799 static struct attribute
*inj_attrs
[] = {
800 &dev_attr_inject_section
.attr
,
801 &dev_attr_inject_word
.attr
,
802 &dev_attr_inject_ecc_vector
.attr
,
803 &dev_attr_inject_write
.attr
,
804 &dev_attr_inject_read
.attr
,
808 static umode_t
inj_is_visible(struct kobject
*kobj
, struct attribute
*attr
, int idx
)
810 struct device
*dev
= kobj_to_dev(kobj
);
811 struct mem_ctl_info
*mci
= container_of(dev
, struct mem_ctl_info
, dev
);
812 struct amd64_pvt
*pvt
= mci
->pvt_info
;
814 /* Families which have that injection hw */
815 if (pvt
->fam
>= 0x10 && pvt
->fam
<= 0x16)
821 static const struct attribute_group inj_group
= {
823 .is_visible
= inj_is_visible
,
825 #endif /* CONFIG_EDAC_DEBUG */
828 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
829 * assumed that sys_addr maps to the node given by mci.
831 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
832 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
833 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
834 * then it is also involved in translating a SysAddr to a DramAddr. Sections
835 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
836 * These parts of the documentation are unclear. I interpret them as follows:
838 * When node n receives a SysAddr, it processes the SysAddr as follows:
840 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
841 * Limit registers for node n. If the SysAddr is not within the range
842 * specified by the base and limit values, then node n ignores the Sysaddr
843 * (since it does not map to node n). Otherwise continue to step 2 below.
845 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
846 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
847 * the range of relocated addresses (starting at 0x100000000) from the DRAM
848 * hole. If not, skip to step 3 below. Else get the value of the
849 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
850 * offset defined by this value from the SysAddr.
852 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
853 * Base register for node n. To obtain the DramAddr, subtract the base
854 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
856 static u64
sys_addr_to_dram_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
858 struct amd64_pvt
*pvt
= mci
->pvt_info
;
859 u64 dram_base
, hole_base
, hole_offset
, hole_size
, dram_addr
;
862 dram_base
= get_dram_base(pvt
, pvt
->mc_node_id
);
864 ret
= get_dram_hole_info(mci
, &hole_base
, &hole_offset
, &hole_size
);
866 if ((sys_addr
>= (1ULL << 32)) &&
867 (sys_addr
< ((1ULL << 32) + hole_size
))) {
868 /* use DHAR to translate SysAddr to DramAddr */
869 dram_addr
= sys_addr
- hole_offset
;
871 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
872 (unsigned long)sys_addr
,
873 (unsigned long)dram_addr
);
880 * Translate the SysAddr to a DramAddr as shown near the start of
881 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
882 * only deals with 40-bit values. Therefore we discard bits 63-40 of
883 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
884 * discard are all 1s. Otherwise the bits we discard are all 0s. See
885 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
886 * Programmer's Manual Volume 1 Application Programming.
888 dram_addr
= (sys_addr
& GENMASK_ULL(39, 0)) - dram_base
;
890 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
891 (unsigned long)sys_addr
, (unsigned long)dram_addr
);
896 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
897 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
898 * for node interleaving.
900 static int num_node_interleave_bits(unsigned intlv_en
)
902 static const int intlv_shift_table
[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
905 BUG_ON(intlv_en
> 7);
906 n
= intlv_shift_table
[intlv_en
];
910 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
911 static u64
dram_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 dram_addr
)
913 struct amd64_pvt
*pvt
;
920 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
921 * concerning translating a DramAddr to an InputAddr.
923 intlv_shift
= num_node_interleave_bits(dram_intlv_en(pvt
, 0));
924 input_addr
= ((dram_addr
>> intlv_shift
) & GENMASK_ULL(35, 12)) +
927 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
928 intlv_shift
, (unsigned long)dram_addr
,
929 (unsigned long)input_addr
);
935 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
936 * assumed that @sys_addr maps to the node given by mci.
938 static u64
sys_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
943 dram_addr_to_input_addr(mci
, sys_addr_to_dram_addr(mci
, sys_addr
));
945 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
946 (unsigned long)sys_addr
, (unsigned long)input_addr
);
951 /* Map the Error address to a PAGE and PAGE OFFSET. */
952 static inline void error_address_to_page_and_offset(u64 error_address
,
953 struct err_info
*err
)
955 err
->page
= (u32
) (error_address
>> PAGE_SHIFT
);
956 err
->offset
= ((u32
) error_address
) & ~PAGE_MASK
;
960 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
961 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
962 * of a node that detected an ECC memory error. mci represents the node that
963 * the error address maps to (possibly different from the node that detected
964 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
967 static int sys_addr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
)
971 csrow
= input_addr_to_csrow(mci
, sys_addr_to_input_addr(mci
, sys_addr
));
974 amd64_mc_err(mci
, "Failed to translate InputAddr to csrow for "
975 "address 0x%lx\n", (unsigned long)sys_addr
);
980 * See AMD PPR DF::LclNodeTypeMap
982 * This register gives information for nodes of the same type within a system.
984 * Reading this register from a GPU node will tell how many GPU nodes are in the
985 * system and what the lowest AMD Node ID value is for the GPU nodes. Use this
986 * info to fixup the Linux logical "Node ID" value set in the AMD NB code and EDAC.
988 static struct local_node_map
{
993 #define PCI_DEVICE_ID_AMD_MI200_DF_F1 0x14d1
994 #define REG_LOCAL_NODE_TYPE_MAP 0x144
996 /* Local Node Type Map (LNTM) fields */
997 #define LNTM_NODE_COUNT GENMASK(27, 16)
998 #define LNTM_BASE_NODE_ID GENMASK(11, 0)
1000 static int gpu_get_node_map(struct amd64_pvt
*pvt
)
1002 struct pci_dev
*pdev
;
1007 * Mapping of nodes from hardware-provided AMD Node ID to a
1008 * Linux logical one is applicable for MI200 models. Therefore,
1009 * return early for other heterogeneous systems.
1011 if (pvt
->F3
->device
!= PCI_DEVICE_ID_AMD_MI200_DF_F3
)
1015 * Node ID 0 is reserved for CPUs. Therefore, a non-zero Node ID
1016 * means the values have been already cached.
1018 if (gpu_node_map
.base_node_id
)
1021 pdev
= pci_get_device(PCI_VENDOR_ID_AMD
, PCI_DEVICE_ID_AMD_MI200_DF_F1
, NULL
);
1027 ret
= pci_read_config_dword(pdev
, REG_LOCAL_NODE_TYPE_MAP
, &tmp
);
1029 ret
= pcibios_err_to_errno(ret
);
1033 gpu_node_map
.node_count
= FIELD_GET(LNTM_NODE_COUNT
, tmp
);
1034 gpu_node_map
.base_node_id
= FIELD_GET(LNTM_BASE_NODE_ID
, tmp
);
1041 static int fixup_node_id(int node_id
, struct mce
*m
)
1043 /* MCA_IPID[InstanceIdHi] give the AMD Node ID for the bank. */
1044 u8 nid
= (m
->ipid
>> 44) & 0xF;
1046 if (smca_get_bank_type(m
->extcpu
, m
->bank
) != SMCA_UMC_V2
)
1049 /* Nodes below the GPU base node are CPU nodes and don't need a fixup. */
1050 if (nid
< gpu_node_map
.base_node_id
)
1053 /* Convert the hardware-provided AMD Node ID to a Linux logical one. */
1054 return nid
- gpu_node_map
.base_node_id
+ 1;
1057 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*, u16
);
1060 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
1063 static unsigned long dct_determine_edac_cap(struct amd64_pvt
*pvt
)
1065 unsigned long edac_cap
= EDAC_FLAG_NONE
;
1068 bit
= (pvt
->fam
> 0xf || pvt
->ext_model
>= K8_REV_F
)
1072 if (pvt
->dclr0
& BIT(bit
))
1073 edac_cap
= EDAC_FLAG_SECDED
;
1078 static unsigned long umc_determine_edac_cap(struct amd64_pvt
*pvt
)
1080 u8 i
, umc_en_mask
= 0, dimm_ecc_en_mask
= 0;
1081 unsigned long edac_cap
= EDAC_FLAG_NONE
;
1084 if (!(pvt
->umc
[i
].sdp_ctrl
& UMC_SDP_INIT
))
1087 umc_en_mask
|= BIT(i
);
1089 /* UMC Configuration bit 12 (DimmEccEn) */
1090 if (pvt
->umc
[i
].umc_cfg
& BIT(12))
1091 dimm_ecc_en_mask
|= BIT(i
);
1094 if (umc_en_mask
== dimm_ecc_en_mask
)
1095 edac_cap
= EDAC_FLAG_SECDED
;
1101 * debug routine to display the memory sizes of all logical DIMMs and its
1104 static void dct_debug_display_dimm_sizes(struct amd64_pvt
*pvt
, u8 ctrl
)
1106 u32
*dcsb
= ctrl
? pvt
->csels
[1].csbases
: pvt
->csels
[0].csbases
;
1107 u32 dbam
= ctrl
? pvt
->dbam1
: pvt
->dbam0
;
1108 int dimm
, size0
, size1
;
1110 if (pvt
->fam
== 0xf) {
1111 /* K8 families < revF not supported yet */
1112 if (pvt
->ext_model
< K8_REV_F
)
1118 if (pvt
->fam
== 0x10) {
1119 dbam
= (ctrl
&& !dct_ganging_enabled(pvt
)) ? pvt
->dbam1
1121 dcsb
= (ctrl
&& !dct_ganging_enabled(pvt
)) ?
1122 pvt
->csels
[1].csbases
:
1123 pvt
->csels
[0].csbases
;
1126 dcsb
= pvt
->csels
[1].csbases
;
1128 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1131 edac_printk(KERN_DEBUG
, EDAC_MC
, "DCT%d chip selects:\n", ctrl
);
1133 /* Dump memory sizes for DIMM and its CSROWs */
1134 for (dimm
= 0; dimm
< 4; dimm
++) {
1136 if (dcsb
[dimm
* 2] & DCSB_CS_ENABLE
)
1138 * For F15m60h, we need multiplier for LRDIMM cs_size
1139 * calculation. We pass dimm value to the dbam_to_cs
1140 * mapper so we can find the multiplier from the
1141 * corresponding DCSM.
1143 size0
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1144 DBAM_DIMM(dimm
, dbam
),
1148 if (dcsb
[dimm
* 2 + 1] & DCSB_CS_ENABLE
)
1149 size1
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1150 DBAM_DIMM(dimm
, dbam
),
1153 amd64_info(EDAC_MC
": %d: %5dMB %d: %5dMB\n",
1155 dimm
* 2 + 1, size1
);
1160 static void debug_dump_dramcfg_low(struct amd64_pvt
*pvt
, u32 dclr
, int chan
)
1162 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan
, dclr
);
1164 if (pvt
->dram_type
== MEM_LRDDR3
) {
1165 u32 dcsm
= pvt
->csels
[chan
].csmasks
[0];
1167 * It's assumed all LRDIMMs in a DCT are going to be of
1168 * same 'type' until proven otherwise. So, use a cs
1169 * value of '0' here to get dcsm value.
1171 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm
& 0x3));
1174 edac_dbg(1, "All DIMMs support ECC:%s\n",
1175 (dclr
& BIT(19)) ? "yes" : "no");
1178 edac_dbg(1, " PAR/ERR parity: %s\n",
1179 (dclr
& BIT(8)) ? "enabled" : "disabled");
1181 if (pvt
->fam
== 0x10)
1182 edac_dbg(1, " DCT 128bit mode width: %s\n",
1183 (dclr
& BIT(11)) ? "128b" : "64b");
1185 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1186 (dclr
& BIT(12)) ? "yes" : "no",
1187 (dclr
& BIT(13)) ? "yes" : "no",
1188 (dclr
& BIT(14)) ? "yes" : "no",
1189 (dclr
& BIT(15)) ? "yes" : "no");
1192 #define CS_EVEN_PRIMARY BIT(0)
1193 #define CS_ODD_PRIMARY BIT(1)
1194 #define CS_EVEN_SECONDARY BIT(2)
1195 #define CS_ODD_SECONDARY BIT(3)
1196 #define CS_3R_INTERLEAVE BIT(4)
1198 #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1199 #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1201 static int umc_get_cs_mode(int dimm
, u8 ctrl
, struct amd64_pvt
*pvt
)
1206 if (csrow_enabled(2 * dimm
, ctrl
, pvt
))
1207 cs_mode
|= CS_EVEN_PRIMARY
;
1209 if (csrow_enabled(2 * dimm
+ 1, ctrl
, pvt
))
1210 cs_mode
|= CS_ODD_PRIMARY
;
1212 /* Asymmetric dual-rank DIMM support. */
1213 if (csrow_sec_enabled(2 * dimm
+ 1, ctrl
, pvt
))
1214 cs_mode
|= CS_ODD_SECONDARY
;
1217 * 3 Rank inteleaving support.
1218 * There should be only three bases enabled and their two masks should
1221 for_each_chip_select(base
, ctrl
, pvt
)
1222 count
+= csrow_enabled(base
, ctrl
, pvt
);
1225 pvt
->csels
[ctrl
].csmasks
[0] == pvt
->csels
[ctrl
].csmasks
[1]) {
1226 edac_dbg(1, "3R interleaving in use.\n");
1227 cs_mode
|= CS_3R_INTERLEAVE
;
1233 static int __addr_mask_to_cs_size(u32 addr_mask_orig
, unsigned int cs_mode
,
1234 int csrow_nr
, int dimm
)
1236 u32 msb
, weight
, num_zero_bits
;
1237 u32 addr_mask_deinterleaved
;
1241 * The number of zero bits in the mask is equal to the number of bits
1242 * in a full mask minus the number of bits in the current mask.
1244 * The MSB is the number of bits in the full mask because BIT[0] is
1247 * In the special 3 Rank interleaving case, a single bit is flipped
1248 * without swapping with the most significant bit. This can be handled
1249 * by keeping the MSB where it is and ignoring the single zero bit.
1251 msb
= fls(addr_mask_orig
) - 1;
1252 weight
= hweight_long(addr_mask_orig
);
1253 num_zero_bits
= msb
- weight
- !!(cs_mode
& CS_3R_INTERLEAVE
);
1255 /* Take the number of zero bits off from the top of the mask. */
1256 addr_mask_deinterleaved
= GENMASK_ULL(msb
- num_zero_bits
, 1);
1258 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr
, dimm
);
1259 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig
);
1260 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved
);
1262 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1263 size
= (addr_mask_deinterleaved
>> 2) + 1;
1265 /* Return size in MBs. */
1269 static int umc_addr_mask_to_cs_size(struct amd64_pvt
*pvt
, u8 umc
,
1270 unsigned int cs_mode
, int csrow_nr
)
1272 int cs_mask_nr
= csrow_nr
;
1276 /* No Chip Selects are enabled. */
1280 /* Requested size of an even CS but none are enabled. */
1281 if (!(cs_mode
& CS_EVEN
) && !(csrow_nr
& 1))
1284 /* Requested size of an odd CS but none are enabled. */
1285 if (!(cs_mode
& CS_ODD
) && (csrow_nr
& 1))
1289 * Family 17h introduced systems with one mask per DIMM,
1290 * and two Chip Selects per DIMM.
1292 * CS0 and CS1 -> MASK0 / DIMM0
1293 * CS2 and CS3 -> MASK1 / DIMM1
1295 * Family 19h Model 10h introduced systems with one mask per Chip Select,
1296 * and two Chip Selects per DIMM.
1298 * CS0 -> MASK0 -> DIMM0
1299 * CS1 -> MASK1 -> DIMM0
1300 * CS2 -> MASK2 -> DIMM1
1301 * CS3 -> MASK3 -> DIMM1
1303 * Keep the mask number equal to the Chip Select number for newer systems,
1304 * and shift the mask number for older systems.
1306 dimm
= csrow_nr
>> 1;
1308 if (!pvt
->flags
.zn_regs_v2
)
1311 /* Asymmetric dual-rank DIMM support. */
1312 if ((csrow_nr
& 1) && (cs_mode
& CS_ODD_SECONDARY
))
1313 addr_mask_orig
= pvt
->csels
[umc
].csmasks_sec
[cs_mask_nr
];
1315 addr_mask_orig
= pvt
->csels
[umc
].csmasks
[cs_mask_nr
];
1317 return __addr_mask_to_cs_size(addr_mask_orig
, cs_mode
, csrow_nr
, dimm
);
1320 static void umc_debug_display_dimm_sizes(struct amd64_pvt
*pvt
, u8 ctrl
)
1322 int dimm
, size0
, size1
, cs0
, cs1
, cs_mode
;
1324 edac_printk(KERN_DEBUG
, EDAC_MC
, "UMC%d chip selects:\n", ctrl
);
1326 for (dimm
= 0; dimm
< 2; dimm
++) {
1330 cs_mode
= umc_get_cs_mode(dimm
, ctrl
, pvt
);
1332 size0
= umc_addr_mask_to_cs_size(pvt
, ctrl
, cs_mode
, cs0
);
1333 size1
= umc_addr_mask_to_cs_size(pvt
, ctrl
, cs_mode
, cs1
);
1335 amd64_info(EDAC_MC
": %d: %5dMB %d: %5dMB\n",
1341 static void umc_dump_misc_regs(struct amd64_pvt
*pvt
)
1343 struct amd64_umc
*umc
;
1344 u32 i
, tmp
, umc_base
;
1347 umc_base
= get_umc_base(i
);
1350 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i
, umc
->dimm_cfg
);
1351 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i
, umc
->umc_cfg
);
1352 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i
, umc
->sdp_ctrl
);
1353 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i
, umc
->ecc_ctrl
);
1355 amd_smn_read(pvt
->mc_node_id
, umc_base
+ UMCCH_ECC_BAD_SYMBOL
, &tmp
);
1356 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i
, tmp
);
1358 amd_smn_read(pvt
->mc_node_id
, umc_base
+ UMCCH_UMC_CAP
, &tmp
);
1359 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i
, tmp
);
1360 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i
, umc
->umc_cap_hi
);
1362 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1363 i
, (umc
->umc_cap_hi
& BIT(30)) ? "yes" : "no",
1364 (umc
->umc_cap_hi
& BIT(31)) ? "yes" : "no");
1365 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1366 i
, (umc
->umc_cfg
& BIT(12)) ? "yes" : "no");
1367 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1368 i
, (umc
->dimm_cfg
& BIT(6)) ? "yes" : "no");
1369 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1370 i
, (umc
->dimm_cfg
& BIT(7)) ? "yes" : "no");
1372 if (umc
->dram_type
== MEM_LRDDR4
|| umc
->dram_type
== MEM_LRDDR5
) {
1373 amd_smn_read(pvt
->mc_node_id
,
1374 umc_base
+ get_umc_reg(pvt
, UMCCH_ADDR_CFG
),
1376 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1377 i
, 1 << ((tmp
>> 4) & 0x3));
1380 umc_debug_display_dimm_sizes(pvt
, i
);
1384 static void dct_dump_misc_regs(struct amd64_pvt
*pvt
)
1386 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt
->nbcap
);
1388 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1389 (pvt
->nbcap
& NBCAP_DCT_DUAL
) ? "yes" : "no");
1391 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1392 (pvt
->nbcap
& NBCAP_SECDED
) ? "yes" : "no",
1393 (pvt
->nbcap
& NBCAP_CHIPKILL
) ? "yes" : "no");
1395 debug_dump_dramcfg_low(pvt
, pvt
->dclr0
, 0);
1397 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt
->online_spare
);
1399 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1400 pvt
->dhar
, dhar_base(pvt
),
1401 (pvt
->fam
== 0xf) ? k8_dhar_offset(pvt
)
1402 : f10_dhar_offset(pvt
));
1404 dct_debug_display_dimm_sizes(pvt
, 0);
1406 /* everything below this point is Fam10h and above */
1407 if (pvt
->fam
== 0xf)
1410 dct_debug_display_dimm_sizes(pvt
, 1);
1412 /* Only if NOT ganged does dclr1 have valid info */
1413 if (!dct_ganging_enabled(pvt
))
1414 debug_dump_dramcfg_low(pvt
, pvt
->dclr1
, 1);
1416 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt
) ? "yes" : "no");
1418 amd64_info("using x%u syndromes.\n", pvt
->ecc_sym_sz
);
1422 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
1424 static void dct_prep_chip_selects(struct amd64_pvt
*pvt
)
1426 if (pvt
->fam
== 0xf && pvt
->ext_model
< K8_REV_F
) {
1427 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
1428 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 8;
1429 } else if (pvt
->fam
== 0x15 && pvt
->model
== 0x30) {
1430 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 4;
1431 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 2;
1433 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
1434 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 4;
1438 static void umc_prep_chip_selects(struct amd64_pvt
*pvt
)
1443 pvt
->csels
[umc
].b_cnt
= 4;
1444 pvt
->csels
[umc
].m_cnt
= pvt
->flags
.zn_regs_v2
? 4 : 2;
1448 static void umc_read_base_mask(struct amd64_pvt
*pvt
)
1450 u32 umc_base_reg
, umc_base_reg_sec
;
1451 u32 umc_mask_reg
, umc_mask_reg_sec
;
1452 u32 base_reg
, base_reg_sec
;
1453 u32 mask_reg
, mask_reg_sec
;
1454 u32
*base
, *base_sec
;
1455 u32
*mask
, *mask_sec
;
1459 umc_base_reg
= get_umc_base(umc
) + UMCCH_BASE_ADDR
;
1460 umc_base_reg_sec
= get_umc_base(umc
) + UMCCH_BASE_ADDR_SEC
;
1462 for_each_chip_select(cs
, umc
, pvt
) {
1463 base
= &pvt
->csels
[umc
].csbases
[cs
];
1464 base_sec
= &pvt
->csels
[umc
].csbases_sec
[cs
];
1466 base_reg
= umc_base_reg
+ (cs
* 4);
1467 base_reg_sec
= umc_base_reg_sec
+ (cs
* 4);
1469 if (!amd_smn_read(pvt
->mc_node_id
, base_reg
, base
))
1470 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1471 umc
, cs
, *base
, base_reg
);
1473 if (!amd_smn_read(pvt
->mc_node_id
, base_reg_sec
, base_sec
))
1474 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1475 umc
, cs
, *base_sec
, base_reg_sec
);
1478 umc_mask_reg
= get_umc_base(umc
) + UMCCH_ADDR_MASK
;
1479 umc_mask_reg_sec
= get_umc_base(umc
) + get_umc_reg(pvt
, UMCCH_ADDR_MASK_SEC
);
1481 for_each_chip_select_mask(cs
, umc
, pvt
) {
1482 mask
= &pvt
->csels
[umc
].csmasks
[cs
];
1483 mask_sec
= &pvt
->csels
[umc
].csmasks_sec
[cs
];
1485 mask_reg
= umc_mask_reg
+ (cs
* 4);
1486 mask_reg_sec
= umc_mask_reg_sec
+ (cs
* 4);
1488 if (!amd_smn_read(pvt
->mc_node_id
, mask_reg
, mask
))
1489 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1490 umc
, cs
, *mask
, mask_reg
);
1492 if (!amd_smn_read(pvt
->mc_node_id
, mask_reg_sec
, mask_sec
))
1493 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1494 umc
, cs
, *mask_sec
, mask_reg_sec
);
1500 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
1502 static void dct_read_base_mask(struct amd64_pvt
*pvt
)
1506 for_each_chip_select(cs
, 0, pvt
) {
1507 int reg0
= DCSB0
+ (cs
* 4);
1508 int reg1
= DCSB1
+ (cs
* 4);
1509 u32
*base0
= &pvt
->csels
[0].csbases
[cs
];
1510 u32
*base1
= &pvt
->csels
[1].csbases
[cs
];
1512 if (!amd64_read_dct_pci_cfg(pvt
, 0, reg0
, base0
))
1513 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1516 if (pvt
->fam
== 0xf)
1519 if (!amd64_read_dct_pci_cfg(pvt
, 1, reg0
, base1
))
1520 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1521 cs
, *base1
, (pvt
->fam
== 0x10) ? reg1
1525 for_each_chip_select_mask(cs
, 0, pvt
) {
1526 int reg0
= DCSM0
+ (cs
* 4);
1527 int reg1
= DCSM1
+ (cs
* 4);
1528 u32
*mask0
= &pvt
->csels
[0].csmasks
[cs
];
1529 u32
*mask1
= &pvt
->csels
[1].csmasks
[cs
];
1531 if (!amd64_read_dct_pci_cfg(pvt
, 0, reg0
, mask0
))
1532 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1535 if (pvt
->fam
== 0xf)
1538 if (!amd64_read_dct_pci_cfg(pvt
, 1, reg0
, mask1
))
1539 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1540 cs
, *mask1
, (pvt
->fam
== 0x10) ? reg1
1545 static void umc_determine_memory_type(struct amd64_pvt
*pvt
)
1547 struct amd64_umc
*umc
;
1553 if (!(umc
->sdp_ctrl
& UMC_SDP_INIT
)) {
1554 umc
->dram_type
= MEM_EMPTY
;
1559 * Check if the system supports the "DDR Type" field in UMC Config
1560 * and has DDR5 DIMMs in use.
1562 if (pvt
->flags
.zn_regs_v2
&& ((umc
->umc_cfg
& GENMASK(2, 0)) == 0x1)) {
1563 if (umc
->dimm_cfg
& BIT(5))
1564 umc
->dram_type
= MEM_LRDDR5
;
1565 else if (umc
->dimm_cfg
& BIT(4))
1566 umc
->dram_type
= MEM_RDDR5
;
1568 umc
->dram_type
= MEM_DDR5
;
1570 if (umc
->dimm_cfg
& BIT(5))
1571 umc
->dram_type
= MEM_LRDDR4
;
1572 else if (umc
->dimm_cfg
& BIT(4))
1573 umc
->dram_type
= MEM_RDDR4
;
1575 umc
->dram_type
= MEM_DDR4
;
1578 edac_dbg(1, " UMC%d DIMM type: %s\n", i
, edac_mem_types
[umc
->dram_type
]);
1582 static void dct_determine_memory_type(struct amd64_pvt
*pvt
)
1584 u32 dram_ctrl
, dcsm
;
1588 if (pvt
->ext_model
>= K8_REV_F
)
1591 pvt
->dram_type
= (pvt
->dclr0
& BIT(18)) ? MEM_DDR
: MEM_RDDR
;
1595 if (pvt
->dchr0
& DDR3_MODE
)
1598 pvt
->dram_type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR2
: MEM_RDDR2
;
1602 if (pvt
->model
< 0x60)
1606 * Model 0x60h needs special handling:
1608 * We use a Chip Select value of '0' to obtain dcsm.
1609 * Theoretically, it is possible to populate LRDIMMs of different
1610 * 'Rank' value on a DCT. But this is not the common case. So,
1611 * it's reasonable to assume all DIMMs are going to be of same
1612 * 'type' until proven otherwise.
1614 amd64_read_dct_pci_cfg(pvt
, 0, DRAM_CONTROL
, &dram_ctrl
);
1615 dcsm
= pvt
->csels
[0].csmasks
[0];
1617 if (((dram_ctrl
>> 8) & 0x7) == 0x2)
1618 pvt
->dram_type
= MEM_DDR4
;
1619 else if (pvt
->dclr0
& BIT(16))
1620 pvt
->dram_type
= MEM_DDR3
;
1621 else if (dcsm
& 0x3)
1622 pvt
->dram_type
= MEM_LRDDR3
;
1624 pvt
->dram_type
= MEM_RDDR3
;
1632 WARN(1, KERN_ERR
"%s: Family??? 0x%x\n", __func__
, pvt
->fam
);
1633 pvt
->dram_type
= MEM_EMPTY
;
1636 edac_dbg(1, " DIMM type: %s\n", edac_mem_types
[pvt
->dram_type
]);
1640 pvt
->dram_type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR3
: MEM_RDDR3
;
1643 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1644 static u64
get_error_address(struct amd64_pvt
*pvt
, struct mce
*m
)
1646 u16 mce_nid
= topology_amd_node_id(m
->extcpu
);
1647 struct mem_ctl_info
*mci
;
1652 mci
= edac_mc_find(mce_nid
);
1656 pvt
= mci
->pvt_info
;
1658 if (pvt
->fam
== 0xf) {
1663 addr
= m
->addr
& GENMASK_ULL(end_bit
, start_bit
);
1666 * Erratum 637 workaround
1668 if (pvt
->fam
== 0x15) {
1669 u64 cc6_base
, tmp_addr
;
1673 if ((addr
& GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1677 amd64_read_pci_cfg(pvt
->F1
, DRAM_LOCAL_NODE_LIM
, &tmp
);
1678 intlv_en
= tmp
>> 21 & 0x7;
1680 /* add [47:27] + 3 trailing bits */
1681 cc6_base
= (tmp
& GENMASK_ULL(20, 0)) << 3;
1683 /* reverse and add DramIntlvEn */
1684 cc6_base
|= intlv_en
^ 0x7;
1686 /* pin at [47:24] */
1690 return cc6_base
| (addr
& GENMASK_ULL(23, 0));
1692 amd64_read_pci_cfg(pvt
->F1
, DRAM_LOCAL_NODE_BASE
, &tmp
);
1695 tmp_addr
= (addr
& GENMASK_ULL(23, 12)) << __fls(intlv_en
+ 1);
1697 /* OR DramIntlvSel into bits [14:12] */
1698 tmp_addr
|= (tmp
& GENMASK_ULL(23, 21)) >> 9;
1700 /* add remaining [11:0] bits from original MC4_ADDR */
1701 tmp_addr
|= addr
& GENMASK_ULL(11, 0);
1703 return cc6_base
| tmp_addr
;
1709 static struct pci_dev
*pci_get_related_function(unsigned int vendor
,
1710 unsigned int device
,
1711 struct pci_dev
*related
)
1713 struct pci_dev
*dev
= NULL
;
1715 while ((dev
= pci_get_device(vendor
, device
, dev
))) {
1716 if (pci_domain_nr(dev
->bus
) == pci_domain_nr(related
->bus
) &&
1717 (dev
->bus
->number
== related
->bus
->number
) &&
1718 (PCI_SLOT(dev
->devfn
) == PCI_SLOT(related
->devfn
)))
1725 static void read_dram_base_limit_regs(struct amd64_pvt
*pvt
, unsigned range
)
1727 struct amd_northbridge
*nb
;
1728 struct pci_dev
*f1
= NULL
;
1729 unsigned int pci_func
;
1730 int off
= range
<< 3;
1733 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_LO
+ off
, &pvt
->ranges
[range
].base
.lo
);
1734 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_LO
+ off
, &pvt
->ranges
[range
].lim
.lo
);
1736 if (pvt
->fam
== 0xf)
1739 if (!dram_rw(pvt
, range
))
1742 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_HI
+ off
, &pvt
->ranges
[range
].base
.hi
);
1743 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_HI
+ off
, &pvt
->ranges
[range
].lim
.hi
);
1745 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1746 if (pvt
->fam
!= 0x15)
1749 nb
= node_to_amd_nb(dram_dst_node(pvt
, range
));
1753 if (pvt
->model
== 0x60)
1754 pci_func
= PCI_DEVICE_ID_AMD_15H_M60H_NB_F1
;
1755 else if (pvt
->model
== 0x30)
1756 pci_func
= PCI_DEVICE_ID_AMD_15H_M30H_NB_F1
;
1758 pci_func
= PCI_DEVICE_ID_AMD_15H_NB_F1
;
1760 f1
= pci_get_related_function(nb
->misc
->vendor
, pci_func
, nb
->misc
);
1764 amd64_read_pci_cfg(f1
, DRAM_LOCAL_NODE_LIM
, &llim
);
1766 pvt
->ranges
[range
].lim
.lo
&= GENMASK_ULL(15, 0);
1768 /* {[39:27],111b} */
1769 pvt
->ranges
[range
].lim
.lo
|= ((llim
& 0x1fff) << 3 | 0x7) << 16;
1771 pvt
->ranges
[range
].lim
.hi
&= GENMASK_ULL(7, 0);
1774 pvt
->ranges
[range
].lim
.hi
|= llim
>> 13;
1779 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
1780 struct err_info
*err
)
1782 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1784 error_address_to_page_and_offset(sys_addr
, err
);
1787 * Find out which node the error address belongs to. This may be
1788 * different from the node that detected the error.
1790 err
->src_mci
= find_mc_by_sys_addr(mci
, sys_addr
);
1791 if (!err
->src_mci
) {
1792 amd64_mc_err(mci
, "failed to map error addr 0x%lx to a node\n",
1793 (unsigned long)sys_addr
);
1794 err
->err_code
= ERR_NODE
;
1798 /* Now map the sys_addr to a CSROW */
1799 err
->csrow
= sys_addr_to_csrow(err
->src_mci
, sys_addr
);
1800 if (err
->csrow
< 0) {
1801 err
->err_code
= ERR_CSROW
;
1805 /* CHIPKILL enabled */
1806 if (pvt
->nbcfg
& NBCFG_CHIPKILL
) {
1807 err
->channel
= get_channel_from_ecc_syndrome(mci
, err
->syndrome
);
1808 if (err
->channel
< 0) {
1810 * Syndrome didn't map, so we don't know which of the
1811 * 2 DIMMs is in error. So we need to ID 'both' of them
1814 amd64_mc_warn(err
->src_mci
, "unknown syndrome 0x%04x - "
1815 "possible error reporting race\n",
1817 err
->err_code
= ERR_CHANNEL
;
1822 * non-chipkill ecc mode
1824 * The k8 documentation is unclear about how to determine the
1825 * channel number when using non-chipkill memory. This method
1826 * was obtained from email communication with someone at AMD.
1827 * (Wish the email was placed in this comment - norsk)
1829 err
->channel
= ((sys_addr
& BIT(3)) != 0);
1833 static int ddr2_cs_size(unsigned i
, bool dct_width
)
1839 else if (!(i
& 0x1))
1842 shift
= (i
+ 1) >> 1;
1844 return 128 << (shift
+ !!dct_width
);
1847 static int k8_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1848 unsigned cs_mode
, int cs_mask_nr
)
1850 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1852 if (pvt
->ext_model
>= K8_REV_F
) {
1853 WARN_ON(cs_mode
> 11);
1854 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1856 else if (pvt
->ext_model
>= K8_REV_D
) {
1858 WARN_ON(cs_mode
> 10);
1861 * the below calculation, besides trying to win an obfuscated C
1862 * contest, maps cs_mode values to DIMM chip select sizes. The
1865 * cs_mode CS size (mb)
1866 * ======= ============
1879 * Basically, it calculates a value with which to shift the
1880 * smallest CS size of 32MB.
1882 * ddr[23]_cs_size have a similar purpose.
1884 diff
= cs_mode
/3 + (unsigned)(cs_mode
> 5);
1886 return 32 << (cs_mode
- diff
);
1889 WARN_ON(cs_mode
> 6);
1890 return 32 << cs_mode
;
1894 static int ddr3_cs_size(unsigned i
, bool dct_width
)
1899 if (i
== 0 || i
== 3 || i
== 4)
1905 else if (!(i
& 0x1))
1908 shift
= (i
+ 1) >> 1;
1911 cs_size
= (128 * (1 << !!dct_width
)) << shift
;
1916 static int ddr3_lrdimm_cs_size(unsigned i
, unsigned rank_multiply
)
1921 if (i
< 4 || i
== 6)
1925 else if (!(i
& 0x1))
1928 shift
= (i
+ 1) >> 1;
1931 cs_size
= rank_multiply
* (128 << shift
);
1936 static int ddr4_cs_size(unsigned i
)
1945 /* Min cs_size = 1G */
1946 cs_size
= 1024 * (1 << (i
>> 1));
1951 static int f10_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1952 unsigned cs_mode
, int cs_mask_nr
)
1954 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1956 WARN_ON(cs_mode
> 11);
1958 if (pvt
->dchr0
& DDR3_MODE
|| pvt
->dchr1
& DDR3_MODE
)
1959 return ddr3_cs_size(cs_mode
, dclr
& WIDTH_128
);
1961 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1965 * F15h supports only 64bit DCT interfaces
1967 static int f15_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1968 unsigned cs_mode
, int cs_mask_nr
)
1970 WARN_ON(cs_mode
> 12);
1972 return ddr3_cs_size(cs_mode
, false);
1975 /* F15h M60h supports DDR4 mapping as well.. */
1976 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1977 unsigned cs_mode
, int cs_mask_nr
)
1980 u32 dcsm
= pvt
->csels
[dct
].csmasks
[cs_mask_nr
];
1982 WARN_ON(cs_mode
> 12);
1984 if (pvt
->dram_type
== MEM_DDR4
) {
1988 cs_size
= ddr4_cs_size(cs_mode
);
1989 } else if (pvt
->dram_type
== MEM_LRDDR3
) {
1990 unsigned rank_multiply
= dcsm
& 0xf;
1992 if (rank_multiply
== 3)
1994 cs_size
= ddr3_lrdimm_cs_size(cs_mode
, rank_multiply
);
1996 /* Minimum cs size is 512mb for F15hM60h*/
2000 cs_size
= ddr3_cs_size(cs_mode
, false);
2007 * F16h and F15h model 30h have only limited cs_modes.
2009 static int f16_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
2010 unsigned cs_mode
, int cs_mask_nr
)
2012 WARN_ON(cs_mode
> 12);
2014 if (cs_mode
== 6 || cs_mode
== 8 ||
2015 cs_mode
== 9 || cs_mode
== 12)
2018 return ddr3_cs_size(cs_mode
, false);
2021 static void read_dram_ctl_register(struct amd64_pvt
*pvt
)
2024 if (pvt
->fam
== 0xf)
2027 if (!amd64_read_pci_cfg(pvt
->F2
, DCT_SEL_LO
, &pvt
->dct_sel_lo
)) {
2028 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
2029 pvt
->dct_sel_lo
, dct_sel_baseaddr(pvt
));
2031 edac_dbg(0, " DCTs operate in %s mode\n",
2032 (dct_ganging_enabled(pvt
) ? "ganged" : "unganged"));
2034 if (!dct_ganging_enabled(pvt
))
2035 edac_dbg(0, " Address range split per DCT: %s\n",
2036 (dct_high_range_enabled(pvt
) ? "yes" : "no"));
2038 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
2039 (dct_data_intlv_enabled(pvt
) ? "enabled" : "disabled"),
2040 (dct_memory_cleared(pvt
) ? "yes" : "no"));
2042 edac_dbg(0, " channel interleave: %s, "
2043 "interleave bits selector: 0x%x\n",
2044 (dct_interleave_enabled(pvt
) ? "enabled" : "disabled"),
2045 dct_sel_interleave_addr(pvt
));
2048 amd64_read_pci_cfg(pvt
->F2
, DCT_SEL_HI
, &pvt
->dct_sel_hi
);
2052 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
2053 * 2.10.12 Memory Interleaving Modes).
2055 static u8
f15_m30h_determine_channel(struct amd64_pvt
*pvt
, u64 sys_addr
,
2056 u8 intlv_en
, int num_dcts_intlv
,
2063 return (u8
)(dct_sel
);
2065 if (num_dcts_intlv
== 2) {
2066 select
= (sys_addr
>> 8) & 0x3;
2067 channel
= select
? 0x3 : 0;
2068 } else if (num_dcts_intlv
== 4) {
2069 u8 intlv_addr
= dct_sel_interleave_addr(pvt
);
2070 switch (intlv_addr
) {
2072 channel
= (sys_addr
>> 8) & 0x3;
2075 channel
= (sys_addr
>> 9) & 0x3;
2083 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
2084 * Interleaving Modes.
2086 static u8
f1x_determine_channel(struct amd64_pvt
*pvt
, u64 sys_addr
,
2087 bool hi_range_sel
, u8 intlv_en
)
2089 u8 dct_sel_high
= (pvt
->dct_sel_lo
>> 1) & 1;
2091 if (dct_ganging_enabled(pvt
))
2095 return dct_sel_high
;
2098 * see F2x110[DctSelIntLvAddr] - channel interleave mode
2100 if (dct_interleave_enabled(pvt
)) {
2101 u8 intlv_addr
= dct_sel_interleave_addr(pvt
);
2103 /* return DCT select function: 0=DCT0, 1=DCT1 */
2105 return sys_addr
>> 6 & 1;
2107 if (intlv_addr
& 0x2) {
2108 u8 shift
= intlv_addr
& 0x1 ? 9 : 6;
2109 u32 temp
= hweight_long((u32
) ((sys_addr
>> 16) & 0x1F)) & 1;
2111 return ((sys_addr
>> shift
) & 1) ^ temp
;
2114 if (intlv_addr
& 0x4) {
2115 u8 shift
= intlv_addr
& 0x1 ? 9 : 8;
2117 return (sys_addr
>> shift
) & 1;
2120 return (sys_addr
>> (12 + hweight8(intlv_en
))) & 1;
2123 if (dct_high_range_enabled(pvt
))
2124 return ~dct_sel_high
& 1;
2129 /* Convert the sys_addr to the normalized DCT address */
2130 static u64
f1x_get_norm_dct_addr(struct amd64_pvt
*pvt
, u8 range
,
2131 u64 sys_addr
, bool hi_rng
,
2132 u32 dct_sel_base_addr
)
2135 u64 dram_base
= get_dram_base(pvt
, range
);
2136 u64 hole_off
= f10_dhar_offset(pvt
);
2137 u64 dct_sel_base_off
= (u64
)(pvt
->dct_sel_hi
& 0xFFFFFC00) << 16;
2142 * base address of high range is below 4Gb
2143 * (bits [47:27] at [31:11])
2144 * DRAM address space on this DCT is hoisted above 4Gb &&
2147 * remove hole offset from sys_addr
2149 * remove high range offset from sys_addr
2151 if ((!(dct_sel_base_addr
>> 16) ||
2152 dct_sel_base_addr
< dhar_base(pvt
)) &&
2154 (sys_addr
>= BIT_64(32)))
2155 chan_off
= hole_off
;
2157 chan_off
= dct_sel_base_off
;
2161 * we have a valid hole &&
2166 * remove dram base to normalize to DCT address
2168 if (dhar_valid(pvt
) && (sys_addr
>= BIT_64(32)))
2169 chan_off
= hole_off
;
2171 chan_off
= dram_base
;
2174 return (sys_addr
& GENMASK_ULL(47,6)) - (chan_off
& GENMASK_ULL(47,23));
2178 * checks if the csrow passed in is marked as SPARED, if so returns the new
2181 static int f10_process_possible_spare(struct amd64_pvt
*pvt
, u8 dct
, int csrow
)
2185 if (online_spare_swap_done(pvt
, dct
) &&
2186 csrow
== online_spare_bad_dramcs(pvt
, dct
)) {
2188 for_each_chip_select(tmp_cs
, dct
, pvt
) {
2189 if (chip_select_base(tmp_cs
, dct
, pvt
) & 0x2) {
2199 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
2200 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
2203 * -EINVAL: NOT FOUND
2204 * 0..csrow = Chip-Select Row
2206 static int f1x_lookup_addr_in_dct(u64 in_addr
, u8 nid
, u8 dct
)
2208 struct mem_ctl_info
*mci
;
2209 struct amd64_pvt
*pvt
;
2210 u64 cs_base
, cs_mask
;
2211 int cs_found
= -EINVAL
;
2214 mci
= edac_mc_find(nid
);
2218 pvt
= mci
->pvt_info
;
2220 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr
, dct
);
2222 for_each_chip_select(csrow
, dct
, pvt
) {
2223 if (!csrow_enabled(csrow
, dct
, pvt
))
2226 get_cs_base_and_mask(pvt
, csrow
, dct
, &cs_base
, &cs_mask
);
2228 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2229 csrow
, cs_base
, cs_mask
);
2233 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2234 (in_addr
& cs_mask
), (cs_base
& cs_mask
));
2236 if ((in_addr
& cs_mask
) == (cs_base
& cs_mask
)) {
2237 if (pvt
->fam
== 0x15 && pvt
->model
>= 0x30) {
2241 cs_found
= f10_process_possible_spare(pvt
, dct
, csrow
);
2243 edac_dbg(1, " MATCH csrow=%d\n", cs_found
);
2251 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
2252 * swapped with a region located at the bottom of memory so that the GPU can use
2253 * the interleaved region and thus two channels.
2255 static u64
f1x_swap_interleaved_region(struct amd64_pvt
*pvt
, u64 sys_addr
)
2257 u32 swap_reg
, swap_base
, swap_limit
, rgn_size
, tmp_addr
;
2259 if (pvt
->fam
== 0x10) {
2260 /* only revC3 and revE have that feature */
2261 if (pvt
->model
< 4 || (pvt
->model
< 0xa && pvt
->stepping
< 3))
2265 amd64_read_pci_cfg(pvt
->F2
, SWAP_INTLV_REG
, &swap_reg
);
2267 if (!(swap_reg
& 0x1))
2270 swap_base
= (swap_reg
>> 3) & 0x7f;
2271 swap_limit
= (swap_reg
>> 11) & 0x7f;
2272 rgn_size
= (swap_reg
>> 20) & 0x7f;
2273 tmp_addr
= sys_addr
>> 27;
2275 if (!(sys_addr
>> 34) &&
2276 (((tmp_addr
>= swap_base
) &&
2277 (tmp_addr
<= swap_limit
)) ||
2278 (tmp_addr
< rgn_size
)))
2279 return sys_addr
^ (u64
)swap_base
<< 27;
2284 /* For a given @dram_range, check if @sys_addr falls within it. */
2285 static int f1x_match_to_this_node(struct amd64_pvt
*pvt
, unsigned range
,
2286 u64 sys_addr
, int *chan_sel
)
2288 int cs_found
= -EINVAL
;
2292 bool high_range
= false;
2294 u8 node_id
= dram_dst_node(pvt
, range
);
2295 u8 intlv_en
= dram_intlv_en(pvt
, range
);
2296 u32 intlv_sel
= dram_intlv_sel(pvt
, range
);
2298 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2299 range
, sys_addr
, get_dram_limit(pvt
, range
));
2301 if (dhar_valid(pvt
) &&
2302 dhar_base(pvt
) <= sys_addr
&&
2303 sys_addr
< BIT_64(32)) {
2304 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2309 if (intlv_en
&& (intlv_sel
!= ((sys_addr
>> 12) & intlv_en
)))
2312 sys_addr
= f1x_swap_interleaved_region(pvt
, sys_addr
);
2314 dct_sel_base
= dct_sel_baseaddr(pvt
);
2317 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
2318 * select between DCT0 and DCT1.
2320 if (dct_high_range_enabled(pvt
) &&
2321 !dct_ganging_enabled(pvt
) &&
2322 ((sys_addr
>> 27) >= (dct_sel_base
>> 11)))
2325 channel
= f1x_determine_channel(pvt
, sys_addr
, high_range
, intlv_en
);
2327 chan_addr
= f1x_get_norm_dct_addr(pvt
, range
, sys_addr
,
2328 high_range
, dct_sel_base
);
2330 /* Remove node interleaving, see F1x120 */
2332 chan_addr
= ((chan_addr
>> (12 + hweight8(intlv_en
))) << 12) |
2333 (chan_addr
& 0xfff);
2335 /* remove channel interleave */
2336 if (dct_interleave_enabled(pvt
) &&
2337 !dct_high_range_enabled(pvt
) &&
2338 !dct_ganging_enabled(pvt
)) {
2340 if (dct_sel_interleave_addr(pvt
) != 1) {
2341 if (dct_sel_interleave_addr(pvt
) == 0x3)
2343 chan_addr
= ((chan_addr
>> 10) << 9) |
2344 (chan_addr
& 0x1ff);
2346 /* A[6] or hash 6 */
2347 chan_addr
= ((chan_addr
>> 7) << 6) |
2351 chan_addr
= ((chan_addr
>> 13) << 12) |
2352 (chan_addr
& 0xfff);
2355 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr
);
2357 cs_found
= f1x_lookup_addr_in_dct(chan_addr
, node_id
, channel
);
2360 *chan_sel
= channel
;
2365 static int f15_m30h_match_to_this_node(struct amd64_pvt
*pvt
, unsigned range
,
2366 u64 sys_addr
, int *chan_sel
)
2368 int cs_found
= -EINVAL
;
2369 int num_dcts_intlv
= 0;
2370 u64 chan_addr
, chan_offset
;
2371 u64 dct_base
, dct_limit
;
2372 u32 dct_cont_base_reg
, dct_cont_limit_reg
, tmp
;
2373 u8 channel
, alias_channel
, leg_mmio_hole
, dct_sel
, dct_offset_en
;
2375 u64 dhar_offset
= f10_dhar_offset(pvt
);
2376 u8 intlv_addr
= dct_sel_interleave_addr(pvt
);
2377 u8 node_id
= dram_dst_node(pvt
, range
);
2378 u8 intlv_en
= dram_intlv_en(pvt
, range
);
2380 amd64_read_pci_cfg(pvt
->F1
, DRAM_CONT_BASE
, &dct_cont_base_reg
);
2381 amd64_read_pci_cfg(pvt
->F1
, DRAM_CONT_LIMIT
, &dct_cont_limit_reg
);
2383 dct_offset_en
= (u8
) ((dct_cont_base_reg
>> 3) & BIT(0));
2384 dct_sel
= (u8
) ((dct_cont_base_reg
>> 4) & 0x7);
2386 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2387 range
, sys_addr
, get_dram_limit(pvt
, range
));
2389 if (!(get_dram_base(pvt
, range
) <= sys_addr
) &&
2390 !(get_dram_limit(pvt
, range
) >= sys_addr
))
2393 if (dhar_valid(pvt
) &&
2394 dhar_base(pvt
) <= sys_addr
&&
2395 sys_addr
< BIT_64(32)) {
2396 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2401 /* Verify sys_addr is within DCT Range. */
2402 dct_base
= (u64
) dct_sel_baseaddr(pvt
);
2403 dct_limit
= (dct_cont_limit_reg
>> 11) & 0x1FFF;
2405 if (!(dct_cont_base_reg
& BIT(0)) &&
2406 !(dct_base
<= (sys_addr
>> 27) &&
2407 dct_limit
>= (sys_addr
>> 27)))
2410 /* Verify number of dct's that participate in channel interleaving. */
2411 num_dcts_intlv
= (int) hweight8(intlv_en
);
2413 if (!(num_dcts_intlv
% 2 == 0) || (num_dcts_intlv
> 4))
2416 if (pvt
->model
>= 0x60)
2417 channel
= f1x_determine_channel(pvt
, sys_addr
, false, intlv_en
);
2419 channel
= f15_m30h_determine_channel(pvt
, sys_addr
, intlv_en
,
2420 num_dcts_intlv
, dct_sel
);
2422 /* Verify we stay within the MAX number of channels allowed */
2426 leg_mmio_hole
= (u8
) (dct_cont_base_reg
>> 1 & BIT(0));
2428 /* Get normalized DCT addr */
2429 if (leg_mmio_hole
&& (sys_addr
>= BIT_64(32)))
2430 chan_offset
= dhar_offset
;
2432 chan_offset
= dct_base
<< 27;
2434 chan_addr
= sys_addr
- chan_offset
;
2436 /* remove channel interleave */
2437 if (num_dcts_intlv
== 2) {
2438 if (intlv_addr
== 0x4)
2439 chan_addr
= ((chan_addr
>> 9) << 8) |
2441 else if (intlv_addr
== 0x5)
2442 chan_addr
= ((chan_addr
>> 10) << 9) |
2443 (chan_addr
& 0x1ff);
2447 } else if (num_dcts_intlv
== 4) {
2448 if (intlv_addr
== 0x4)
2449 chan_addr
= ((chan_addr
>> 10) << 8) |
2451 else if (intlv_addr
== 0x5)
2452 chan_addr
= ((chan_addr
>> 11) << 9) |
2453 (chan_addr
& 0x1ff);
2458 if (dct_offset_en
) {
2459 amd64_read_pci_cfg(pvt
->F1
,
2460 DRAM_CONT_HIGH_OFF
+ (int) channel
* 4,
2462 chan_addr
+= (u64
) ((tmp
>> 11) & 0xfff) << 27;
2465 f15h_select_dct(pvt
, channel
);
2467 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr
);
2471 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2472 * there is support for 4 DCT's, but only 2 are currently functional.
2473 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2474 * pvt->csels[1]. So we need to use '1' here to get correct info.
2475 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2477 alias_channel
= (channel
== 3) ? 1 : channel
;
2479 cs_found
= f1x_lookup_addr_in_dct(chan_addr
, node_id
, alias_channel
);
2482 *chan_sel
= alias_channel
;
2487 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt
*pvt
,
2491 int cs_found
= -EINVAL
;
2494 for (range
= 0; range
< DRAM_RANGES
; range
++) {
2495 if (!dram_rw(pvt
, range
))
2498 if (pvt
->fam
== 0x15 && pvt
->model
>= 0x30)
2499 cs_found
= f15_m30h_match_to_this_node(pvt
, range
,
2503 else if ((get_dram_base(pvt
, range
) <= sys_addr
) &&
2504 (get_dram_limit(pvt
, range
) >= sys_addr
)) {
2505 cs_found
= f1x_match_to_this_node(pvt
, range
,
2506 sys_addr
, chan_sel
);
2515 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2516 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2518 * The @sys_addr is usually an error address received from the hardware
2521 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
2522 struct err_info
*err
)
2524 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2526 error_address_to_page_and_offset(sys_addr
, err
);
2528 err
->csrow
= f1x_translate_sysaddr_to_cs(pvt
, sys_addr
, &err
->channel
);
2529 if (err
->csrow
< 0) {
2530 err
->err_code
= ERR_CSROW
;
2535 * We need the syndromes for channel detection only when we're
2536 * ganged. Otherwise @chan should already contain the channel at
2539 if (dct_ganging_enabled(pvt
))
2540 err
->channel
= get_channel_from_ecc_syndrome(mci
, err
->syndrome
);
2544 * These are tables of eigenvectors (one per line) which can be used for the
2545 * construction of the syndrome tables. The modified syndrome search algorithm
2546 * uses those to find the symbol in error and thus the DIMM.
2548 * Algorithm courtesy of Ross LaFetra from AMD.
2550 static const u16 x4_vectors
[] = {
2551 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2552 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2553 0x0001, 0x0002, 0x0004, 0x0008,
2554 0x1013, 0x3032, 0x4044, 0x8088,
2555 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2556 0x4857, 0xc4fe, 0x13cc, 0x3288,
2557 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2558 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2559 0x15c1, 0x2a42, 0x89ac, 0x4758,
2560 0x2b03, 0x1602, 0x4f0c, 0xca08,
2561 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2562 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2563 0x2b87, 0x164e, 0x642c, 0xdc18,
2564 0x40b9, 0x80de, 0x1094, 0x20e8,
2565 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2566 0x11c1, 0x2242, 0x84ac, 0x4c58,
2567 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2568 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2569 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2570 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2571 0x16b3, 0x3d62, 0x4f34, 0x8518,
2572 0x1e2f, 0x391a, 0x5cac, 0xf858,
2573 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2574 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2575 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2576 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2577 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2578 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2579 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2580 0x185d, 0x2ca6, 0x7914, 0x9e28,
2581 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2582 0x4199, 0x82ee, 0x19f4, 0x2e58,
2583 0x4807, 0xc40e, 0x130c, 0x3208,
2584 0x1905, 0x2e0a, 0x5804, 0xac08,
2585 0x213f, 0x132a, 0xadfc, 0x5ba8,
2586 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2589 static const u16 x8_vectors
[] = {
2590 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2591 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2592 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2593 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2594 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2595 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2596 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2597 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2598 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2599 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2600 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2601 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2602 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2603 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2604 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2605 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2606 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2607 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2608 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2611 static int decode_syndrome(u16 syndrome
, const u16
*vectors
, unsigned num_vecs
,
2614 unsigned int i
, err_sym
;
2616 for (err_sym
= 0; err_sym
< num_vecs
/ v_dim
; err_sym
++) {
2618 unsigned v_idx
= err_sym
* v_dim
;
2619 unsigned v_end
= (err_sym
+ 1) * v_dim
;
2621 /* walk over all 16 bits of the syndrome */
2622 for (i
= 1; i
< (1U << 16); i
<<= 1) {
2624 /* if bit is set in that eigenvector... */
2625 if (v_idx
< v_end
&& vectors
[v_idx
] & i
) {
2626 u16 ev_comp
= vectors
[v_idx
++];
2628 /* ... and bit set in the modified syndrome, */
2638 /* can't get to zero, move to next symbol */
2643 edac_dbg(0, "syndrome(%x) not found\n", syndrome
);
2647 static int map_err_sym_to_channel(int err_sym
, int sym_size
)
2658 return err_sym
>> 4;
2663 /* imaginary bits not in a DIMM */
2665 WARN(1, KERN_ERR
"Invalid error symbol: 0x%x\n",
2673 return err_sym
>> 3;
2678 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*mci
, u16 syndrome
)
2680 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2683 if (pvt
->ecc_sym_sz
== 8)
2684 err_sym
= decode_syndrome(syndrome
, x8_vectors
,
2685 ARRAY_SIZE(x8_vectors
),
2687 else if (pvt
->ecc_sym_sz
== 4)
2688 err_sym
= decode_syndrome(syndrome
, x4_vectors
,
2689 ARRAY_SIZE(x4_vectors
),
2692 amd64_warn("Illegal syndrome type: %u\n", pvt
->ecc_sym_sz
);
2696 return map_err_sym_to_channel(err_sym
, pvt
->ecc_sym_sz
);
2699 static void __log_ecc_error(struct mem_ctl_info
*mci
, struct err_info
*err
,
2702 enum hw_event_mc_err_type err_type
;
2706 err_type
= HW_EVENT_ERR_CORRECTED
;
2707 else if (ecc_type
== 1)
2708 err_type
= HW_EVENT_ERR_UNCORRECTED
;
2709 else if (ecc_type
== 3)
2710 err_type
= HW_EVENT_ERR_DEFERRED
;
2712 WARN(1, "Something is rotten in the state of Denmark.\n");
2716 switch (err
->err_code
) {
2721 string
= "Failed to map error addr to a node";
2724 string
= "Failed to map error addr to a csrow";
2727 string
= "Unknown syndrome - possible error reporting race";
2730 string
= "MCA_SYND not valid - unknown syndrome and csrow";
2733 string
= "Cannot decode normalized address";
2736 string
= "WTF error";
2740 edac_mc_handle_error(err_type
, mci
, 1,
2741 err
->page
, err
->offset
, err
->syndrome
,
2742 err
->csrow
, err
->channel
, -1,
2746 static inline void decode_bus_error(int node_id
, struct mce
*m
)
2748 struct mem_ctl_info
*mci
;
2749 struct amd64_pvt
*pvt
;
2750 u8 ecc_type
= (m
->status
>> 45) & 0x3;
2751 u8 xec
= XEC(m
->status
, 0x1f);
2752 u16 ec
= EC(m
->status
);
2754 struct err_info err
;
2756 mci
= edac_mc_find(node_id
);
2760 pvt
= mci
->pvt_info
;
2762 /* Bail out early if this was an 'observed' error */
2763 if (PP(ec
) == NBSL_PP_OBS
)
2766 /* Do only ECC errors */
2767 if (xec
&& xec
!= F10_NBSL_EXT_ERR_ECC
)
2770 memset(&err
, 0, sizeof(err
));
2772 sys_addr
= get_error_address(pvt
, m
);
2775 err
.syndrome
= extract_syndrome(m
->status
);
2777 pvt
->ops
->map_sysaddr_to_csrow(mci
, sys_addr
, &err
);
2779 __log_ecc_error(mci
, &err
, ecc_type
);
2783 * To find the UMC channel represented by this bank we need to match on its
2784 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2787 * Currently, we can derive the channel number by looking at the 6th nibble in
2788 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
2791 * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of
2792 * the MCA_SYND[ErrorInformation] field.
2794 static void umc_get_err_info(struct mce
*m
, struct err_info
*err
)
2796 err
->channel
= (m
->ipid
& GENMASK(31, 0)) >> 20;
2797 err
->csrow
= m
->synd
& 0x7;
2800 static void decode_umc_error(int node_id
, struct mce
*m
)
2802 u8 ecc_type
= (m
->status
>> 45) & 0x3;
2803 struct mem_ctl_info
*mci
;
2804 unsigned long sys_addr
;
2805 struct amd64_pvt
*pvt
;
2806 struct atl_err a_err
;
2807 struct err_info err
;
2809 node_id
= fixup_node_id(node_id
, m
);
2811 mci
= edac_mc_find(node_id
);
2815 pvt
= mci
->pvt_info
;
2817 memset(&err
, 0, sizeof(err
));
2819 if (m
->status
& MCI_STATUS_DEFERRED
)
2822 if (!(m
->status
& MCI_STATUS_SYNDV
)) {
2823 err
.err_code
= ERR_SYND
;
2827 if (ecc_type
== 2) {
2828 u8 length
= (m
->synd
>> 18) & 0x3f;
2831 err
.syndrome
= (m
->synd
>> 32) & GENMASK(length
- 1, 0);
2833 err
.err_code
= ERR_CHANNEL
;
2836 pvt
->ops
->get_err_info(m
, &err
);
2838 a_err
.addr
= m
->addr
;
2839 a_err
.ipid
= m
->ipid
;
2840 a_err
.cpu
= m
->extcpu
;
2842 sys_addr
= amd_convert_umc_mca_addr_to_sys_addr(&a_err
);
2843 if (IS_ERR_VALUE(sys_addr
)) {
2844 err
.err_code
= ERR_NORM_ADDR
;
2848 error_address_to_page_and_offset(sys_addr
, &err
);
2851 __log_ecc_error(mci
, &err
, ecc_type
);
2855 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2856 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2859 reserve_mc_sibling_devs(struct amd64_pvt
*pvt
, u16 pci_id1
, u16 pci_id2
)
2861 /* Reserve the ADDRESS MAP Device */
2862 pvt
->F1
= pci_get_related_function(pvt
->F3
->vendor
, pci_id1
, pvt
->F3
);
2864 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1
);
2868 /* Reserve the DCT Device */
2869 pvt
->F2
= pci_get_related_function(pvt
->F3
->vendor
, pci_id2
, pvt
->F3
);
2871 pci_dev_put(pvt
->F1
);
2874 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2
);
2879 pci_ctl_dev
= &pvt
->F2
->dev
;
2881 edac_dbg(1, "F1: %s\n", pci_name(pvt
->F1
));
2882 edac_dbg(1, "F2: %s\n", pci_name(pvt
->F2
));
2883 edac_dbg(1, "F3: %s\n", pci_name(pvt
->F3
));
2888 static void determine_ecc_sym_sz(struct amd64_pvt
*pvt
)
2890 pvt
->ecc_sym_sz
= 4;
2892 if (pvt
->fam
>= 0x10) {
2895 amd64_read_pci_cfg(pvt
->F3
, EXT_NB_MCA_CFG
, &tmp
);
2896 /* F16h has only DCT0, so no need to read dbam1. */
2897 if (pvt
->fam
!= 0x16)
2898 amd64_read_dct_pci_cfg(pvt
, 1, DBAM0
, &pvt
->dbam1
);
2900 /* F10h, revD and later can do x8 ECC too. */
2901 if ((pvt
->fam
> 0x10 || pvt
->model
> 7) && tmp
& BIT(25))
2902 pvt
->ecc_sym_sz
= 8;
2907 * Retrieve the hardware registers of the memory controller.
2909 static void umc_read_mc_regs(struct amd64_pvt
*pvt
)
2911 u8 nid
= pvt
->mc_node_id
;
2912 struct amd64_umc
*umc
;
2915 /* Read registers from each UMC */
2918 umc_base
= get_umc_base(i
);
2921 amd_smn_read(nid
, umc_base
+ get_umc_reg(pvt
, UMCCH_DIMM_CFG
), &umc
->dimm_cfg
);
2922 amd_smn_read(nid
, umc_base
+ UMCCH_UMC_CFG
, &umc
->umc_cfg
);
2923 amd_smn_read(nid
, umc_base
+ UMCCH_SDP_CTRL
, &umc
->sdp_ctrl
);
2924 amd_smn_read(nid
, umc_base
+ UMCCH_ECC_CTRL
, &umc
->ecc_ctrl
);
2925 amd_smn_read(nid
, umc_base
+ UMCCH_UMC_CAP_HI
, &umc
->umc_cap_hi
);
2930 * Retrieve the hardware registers of the memory controller (this includes the
2931 * 'Address Map' and 'Misc' device regs)
2933 static void dct_read_mc_regs(struct amd64_pvt
*pvt
)
2939 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2940 * those are Read-As-Zero.
2942 rdmsrl(MSR_K8_TOP_MEM1
, pvt
->top_mem
);
2943 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt
->top_mem
);
2945 /* Check first whether TOP_MEM2 is enabled: */
2946 rdmsrl(MSR_AMD64_SYSCFG
, msr_val
);
2947 if (msr_val
& BIT(21)) {
2948 rdmsrl(MSR_K8_TOP_MEM2
, pvt
->top_mem2
);
2949 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt
->top_mem2
);
2951 edac_dbg(0, " TOP_MEM2 disabled\n");
2954 amd64_read_pci_cfg(pvt
->F3
, NBCAP
, &pvt
->nbcap
);
2956 read_dram_ctl_register(pvt
);
2958 for (range
= 0; range
< DRAM_RANGES
; range
++) {
2961 /* read settings for this DRAM range */
2962 read_dram_base_limit_regs(pvt
, range
);
2964 rw
= dram_rw(pvt
, range
);
2968 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2970 get_dram_base(pvt
, range
),
2971 get_dram_limit(pvt
, range
));
2973 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2974 dram_intlv_en(pvt
, range
) ? "Enabled" : "Disabled",
2975 (rw
& 0x1) ? "R" : "-",
2976 (rw
& 0x2) ? "W" : "-",
2977 dram_intlv_sel(pvt
, range
),
2978 dram_dst_node(pvt
, range
));
2981 amd64_read_pci_cfg(pvt
->F1
, DHAR
, &pvt
->dhar
);
2982 amd64_read_dct_pci_cfg(pvt
, 0, DBAM0
, &pvt
->dbam0
);
2984 amd64_read_pci_cfg(pvt
->F3
, F10_ONLINE_SPARE
, &pvt
->online_spare
);
2986 amd64_read_dct_pci_cfg(pvt
, 0, DCLR0
, &pvt
->dclr0
);
2987 amd64_read_dct_pci_cfg(pvt
, 0, DCHR0
, &pvt
->dchr0
);
2989 if (!dct_ganging_enabled(pvt
)) {
2990 amd64_read_dct_pci_cfg(pvt
, 1, DCLR0
, &pvt
->dclr1
);
2991 amd64_read_dct_pci_cfg(pvt
, 1, DCHR0
, &pvt
->dchr1
);
2994 determine_ecc_sym_sz(pvt
);
2998 * NOTE: CPU Revision Dependent code
3001 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
3002 * k8 private pointer to -->
3003 * DRAM Bank Address mapping register
3005 * DCL register where dual_channel_active is
3007 * The DBAM register consists of 4 sets of 4 bits each definitions:
3010 * 0-3 CSROWs 0 and 1
3011 * 4-7 CSROWs 2 and 3
3012 * 8-11 CSROWs 4 and 5
3013 * 12-15 CSROWs 6 and 7
3015 * Values range from: 0 to 15
3016 * The meaning of the values depends on CPU revision and dual-channel state,
3017 * see relevant BKDG more info.
3019 * The memory controller provides for total of only 8 CSROWs in its current
3020 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
3021 * single channel or two (2) DIMMs in dual channel mode.
3023 * The following code logic collapses the various tables for CSROW based on CPU
3027 * The number of PAGE_SIZE pages on the specified CSROW number it
3031 static u32
dct_get_csrow_nr_pages(struct amd64_pvt
*pvt
, u8 dct
, int csrow_nr
)
3033 u32 dbam
= dct
? pvt
->dbam1
: pvt
->dbam0
;
3034 u32 cs_mode
, nr_pages
;
3037 cs_mode
= DBAM_DIMM(csrow_nr
, dbam
);
3039 nr_pages
= pvt
->ops
->dbam_to_cs(pvt
, dct
, cs_mode
, csrow_nr
);
3040 nr_pages
<<= 20 - PAGE_SHIFT
;
3042 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3043 csrow_nr
, dct
, cs_mode
);
3044 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages
);
3049 static u32
umc_get_csrow_nr_pages(struct amd64_pvt
*pvt
, u8 dct
, int csrow_nr_orig
)
3051 int csrow_nr
= csrow_nr_orig
;
3052 u32 cs_mode
, nr_pages
;
3054 cs_mode
= umc_get_cs_mode(csrow_nr
>> 1, dct
, pvt
);
3056 nr_pages
= umc_addr_mask_to_cs_size(pvt
, dct
, cs_mode
, csrow_nr
);
3057 nr_pages
<<= 20 - PAGE_SHIFT
;
3059 edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n",
3060 csrow_nr_orig
, dct
, cs_mode
);
3061 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages
);
3066 static void umc_init_csrows(struct mem_ctl_info
*mci
)
3068 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3069 enum edac_type edac_mode
= EDAC_NONE
;
3070 enum dev_type dev_type
= DEV_UNKNOWN
;
3071 struct dimm_info
*dimm
;
3074 if (mci
->edac_ctl_cap
& EDAC_FLAG_S16ECD16ED
) {
3075 edac_mode
= EDAC_S16ECD16ED
;
3077 } else if (mci
->edac_ctl_cap
& EDAC_FLAG_S8ECD8ED
) {
3078 edac_mode
= EDAC_S8ECD8ED
;
3080 } else if (mci
->edac_ctl_cap
& EDAC_FLAG_S4ECD4ED
) {
3081 edac_mode
= EDAC_S4ECD4ED
;
3083 } else if (mci
->edac_ctl_cap
& EDAC_FLAG_SECDED
) {
3084 edac_mode
= EDAC_SECDED
;
3088 for_each_chip_select(cs
, umc
, pvt
) {
3089 if (!csrow_enabled(cs
, umc
, pvt
))
3092 dimm
= mci
->csrows
[cs
]->channels
[umc
]->dimm
;
3094 edac_dbg(1, "MC node: %d, csrow: %d\n",
3095 pvt
->mc_node_id
, cs
);
3097 dimm
->nr_pages
= umc_get_csrow_nr_pages(pvt
, umc
, cs
);
3098 dimm
->mtype
= pvt
->umc
[umc
].dram_type
;
3099 dimm
->edac_mode
= edac_mode
;
3100 dimm
->dtype
= dev_type
;
3107 * Initialize the array of csrow attribute instances, based on the values
3108 * from pci config hardware registers.
3110 static void dct_init_csrows(struct mem_ctl_info
*mci
)
3112 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3113 enum edac_type edac_mode
= EDAC_NONE
;
3114 struct csrow_info
*csrow
;
3115 struct dimm_info
*dimm
;
3120 amd64_read_pci_cfg(pvt
->F3
, NBCFG
, &val
);
3124 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3125 pvt
->mc_node_id
, val
,
3126 !!(val
& NBCFG_CHIPKILL
), !!(val
& NBCFG_ECC_ENABLE
));
3129 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
3131 for_each_chip_select(i
, 0, pvt
) {
3132 bool row_dct0
= !!csrow_enabled(i
, 0, pvt
);
3133 bool row_dct1
= false;
3135 if (pvt
->fam
!= 0xf)
3136 row_dct1
= !!csrow_enabled(i
, 1, pvt
);
3138 if (!row_dct0
&& !row_dct1
)
3141 csrow
= mci
->csrows
[i
];
3143 edac_dbg(1, "MC node: %d, csrow: %d\n",
3144 pvt
->mc_node_id
, i
);
3147 nr_pages
= dct_get_csrow_nr_pages(pvt
, 0, i
);
3148 csrow
->channels
[0]->dimm
->nr_pages
= nr_pages
;
3151 /* K8 has only one DCT */
3152 if (pvt
->fam
!= 0xf && row_dct1
) {
3153 int row_dct1_pages
= dct_get_csrow_nr_pages(pvt
, 1, i
);
3155 csrow
->channels
[1]->dimm
->nr_pages
= row_dct1_pages
;
3156 nr_pages
+= row_dct1_pages
;
3159 edac_dbg(1, "Total csrow%d pages: %u\n", i
, nr_pages
);
3161 /* Determine DIMM ECC mode: */
3162 if (pvt
->nbcfg
& NBCFG_ECC_ENABLE
) {
3163 edac_mode
= (pvt
->nbcfg
& NBCFG_CHIPKILL
)
3168 for (j
= 0; j
< pvt
->max_mcs
; j
++) {
3169 dimm
= csrow
->channels
[j
]->dimm
;
3170 dimm
->mtype
= pvt
->dram_type
;
3171 dimm
->edac_mode
= edac_mode
;
3177 /* get all cores on this DCT */
3178 static void get_cpus_on_this_dct_cpumask(struct cpumask
*mask
, u16 nid
)
3182 for_each_online_cpu(cpu
)
3183 if (topology_amd_node_id(cpu
) == nid
)
3184 cpumask_set_cpu(cpu
, mask
);
3187 /* check MCG_CTL on all the cpus on this node */
3188 static bool nb_mce_bank_enabled_on_node(u16 nid
)
3194 if (!zalloc_cpumask_var(&mask
, GFP_KERNEL
)) {
3195 amd64_warn("%s: Error allocating mask\n", __func__
);
3199 get_cpus_on_this_dct_cpumask(mask
, nid
);
3201 rdmsr_on_cpus(mask
, MSR_IA32_MCG_CTL
, msrs
);
3203 for_each_cpu(cpu
, mask
) {
3204 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
3205 nbe
= reg
->l
& MSR_MCGCTL_NBE
;
3207 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3209 (nbe
? "enabled" : "disabled"));
3217 free_cpumask_var(mask
);
3221 static int toggle_ecc_err_reporting(struct ecc_settings
*s
, u16 nid
, bool on
)
3223 cpumask_var_t cmask
;
3226 if (!zalloc_cpumask_var(&cmask
, GFP_KERNEL
)) {
3227 amd64_warn("%s: error allocating mask\n", __func__
);
3231 get_cpus_on_this_dct_cpumask(cmask
, nid
);
3233 rdmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
3235 for_each_cpu(cpu
, cmask
) {
3237 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
3240 if (reg
->l
& MSR_MCGCTL_NBE
)
3241 s
->flags
.nb_mce_enable
= 1;
3243 reg
->l
|= MSR_MCGCTL_NBE
;
3246 * Turn off NB MCE reporting only when it was off before
3248 if (!s
->flags
.nb_mce_enable
)
3249 reg
->l
&= ~MSR_MCGCTL_NBE
;
3252 wrmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
3254 free_cpumask_var(cmask
);
3259 static bool enable_ecc_error_reporting(struct ecc_settings
*s
, u16 nid
,
3263 u32 value
, mask
= 0x3; /* UECC/CECC enable */
3265 if (toggle_ecc_err_reporting(s
, nid
, ON
)) {
3266 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3270 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
3272 s
->old_nbctl
= value
& mask
;
3273 s
->nbctl_valid
= true;
3276 amd64_write_pci_cfg(F3
, NBCTL
, value
);
3278 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
3280 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3281 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
3283 if (!(value
& NBCFG_ECC_ENABLE
)) {
3284 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3286 s
->flags
.nb_ecc_prev
= 0;
3288 /* Attempt to turn on DRAM ECC Enable */
3289 value
|= NBCFG_ECC_ENABLE
;
3290 amd64_write_pci_cfg(F3
, NBCFG
, value
);
3292 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
3294 if (!(value
& NBCFG_ECC_ENABLE
)) {
3295 amd64_warn("Hardware rejected DRAM ECC enable,"
3296 "check memory DIMM configuration.\n");
3299 amd64_info("Hardware accepted DRAM ECC Enable\n");
3302 s
->flags
.nb_ecc_prev
= 1;
3305 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3306 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
3311 static void restore_ecc_error_reporting(struct ecc_settings
*s
, u16 nid
,
3314 u32 value
, mask
= 0x3; /* UECC/CECC enable */
3316 if (!s
->nbctl_valid
)
3319 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
3321 value
|= s
->old_nbctl
;
3323 amd64_write_pci_cfg(F3
, NBCTL
, value
);
3325 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3326 if (!s
->flags
.nb_ecc_prev
) {
3327 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
3328 value
&= ~NBCFG_ECC_ENABLE
;
3329 amd64_write_pci_cfg(F3
, NBCFG
, value
);
3332 /* restore the NB Enable MCGCTL bit */
3333 if (toggle_ecc_err_reporting(s
, nid
, OFF
))
3334 amd64_warn("Error restoring NB MCGCTL settings!\n");
3337 static bool dct_ecc_enabled(struct amd64_pvt
*pvt
)
3339 u16 nid
= pvt
->mc_node_id
;
3340 bool nb_mce_en
= false;
3344 amd64_read_pci_cfg(pvt
->F3
, NBCFG
, &value
);
3346 ecc_en
= !!(value
& NBCFG_ECC_ENABLE
);
3348 nb_mce_en
= nb_mce_bank_enabled_on_node(nid
);
3350 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3351 MSR_IA32_MCG_CTL
, nid
);
3353 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid
, (ecc_en
? "enabled" : "disabled"));
3355 if (!ecc_en
|| !nb_mce_en
)
3361 static bool umc_ecc_enabled(struct amd64_pvt
*pvt
)
3363 u8 umc_en_mask
= 0, ecc_en_mask
= 0;
3364 u16 nid
= pvt
->mc_node_id
;
3365 struct amd64_umc
*umc
;
3371 /* Only check enabled UMCs. */
3372 if (!(umc
->sdp_ctrl
& UMC_SDP_INIT
))
3375 umc_en_mask
|= BIT(i
);
3377 if (umc
->umc_cap_hi
& UMC_ECC_ENABLED
)
3378 ecc_en_mask
|= BIT(i
);
3381 /* Check whether at least one UMC is enabled: */
3383 ecc_en
= umc_en_mask
== ecc_en_mask
;
3385 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid
);
3387 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid
, (ecc_en
? "enabled" : "disabled"));
3396 umc_determine_edac_ctl_cap(struct mem_ctl_info
*mci
, struct amd64_pvt
*pvt
)
3398 u8 i
, ecc_en
= 1, cpk_en
= 1, dev_x4
= 1, dev_x16
= 1;
3401 if (pvt
->umc
[i
].sdp_ctrl
& UMC_SDP_INIT
) {
3402 ecc_en
&= !!(pvt
->umc
[i
].umc_cap_hi
& UMC_ECC_ENABLED
);
3403 cpk_en
&= !!(pvt
->umc
[i
].umc_cap_hi
& UMC_ECC_CHIPKILL_CAP
);
3405 dev_x4
&= !!(pvt
->umc
[i
].dimm_cfg
& BIT(6));
3406 dev_x16
&= !!(pvt
->umc
[i
].dimm_cfg
& BIT(7));
3410 /* Set chipkill only if ECC is enabled: */
3412 mci
->edac_ctl_cap
|= EDAC_FLAG_SECDED
;
3418 mci
->edac_ctl_cap
|= EDAC_FLAG_S4ECD4ED
;
3420 mci
->edac_ctl_cap
|= EDAC_FLAG_S16ECD16ED
;
3422 mci
->edac_ctl_cap
|= EDAC_FLAG_S8ECD8ED
;
3426 static void dct_setup_mci_misc_attrs(struct mem_ctl_info
*mci
)
3428 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3430 mci
->mtype_cap
= MEM_FLAG_DDR2
| MEM_FLAG_RDDR2
;
3431 mci
->edac_ctl_cap
= EDAC_FLAG_NONE
;
3433 if (pvt
->nbcap
& NBCAP_SECDED
)
3434 mci
->edac_ctl_cap
|= EDAC_FLAG_SECDED
;
3436 if (pvt
->nbcap
& NBCAP_CHIPKILL
)
3437 mci
->edac_ctl_cap
|= EDAC_FLAG_S4ECD4ED
;
3439 mci
->edac_cap
= dct_determine_edac_cap(pvt
);
3440 mci
->mod_name
= EDAC_MOD_STR
;
3441 mci
->ctl_name
= pvt
->ctl_name
;
3442 mci
->dev_name
= pci_name(pvt
->F3
);
3443 mci
->ctl_page_to_phys
= NULL
;
3445 /* memory scrubber interface */
3446 mci
->set_sdram_scrub_rate
= set_scrub_rate
;
3447 mci
->get_sdram_scrub_rate
= get_scrub_rate
;
3449 dct_init_csrows(mci
);
3452 static void umc_setup_mci_misc_attrs(struct mem_ctl_info
*mci
)
3454 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3456 mci
->mtype_cap
= MEM_FLAG_DDR4
| MEM_FLAG_RDDR4
;
3457 mci
->edac_ctl_cap
= EDAC_FLAG_NONE
;
3459 umc_determine_edac_ctl_cap(mci
, pvt
);
3461 mci
->edac_cap
= umc_determine_edac_cap(pvt
);
3462 mci
->mod_name
= EDAC_MOD_STR
;
3463 mci
->ctl_name
= pvt
->ctl_name
;
3464 mci
->dev_name
= pci_name(pvt
->F3
);
3465 mci
->ctl_page_to_phys
= NULL
;
3467 umc_init_csrows(mci
);
3470 static int dct_hw_info_get(struct amd64_pvt
*pvt
)
3472 int ret
= reserve_mc_sibling_devs(pvt
, pvt
->f1_id
, pvt
->f2_id
);
3477 dct_prep_chip_selects(pvt
);
3478 dct_read_base_mask(pvt
);
3479 dct_read_mc_regs(pvt
);
3480 dct_determine_memory_type(pvt
);
3485 static int umc_hw_info_get(struct amd64_pvt
*pvt
)
3487 pvt
->umc
= kcalloc(pvt
->max_mcs
, sizeof(struct amd64_umc
), GFP_KERNEL
);
3491 umc_prep_chip_selects(pvt
);
3492 umc_read_base_mask(pvt
);
3493 umc_read_mc_regs(pvt
);
3494 umc_determine_memory_type(pvt
);
3500 * The CPUs have one channel per UMC, so UMC number is equivalent to a
3501 * channel number. The GPUs have 8 channels per UMC, so the UMC number no
3502 * longer works as a channel number.
3504 * The channel number within a GPU UMC is given in MCA_IPID[15:12].
3505 * However, the IDs are split such that two UMC values go to one UMC, and
3506 * the channel numbers are split in two groups of four.
3508 * Refer to comment on gpu_get_umc_base().
3511 * UMC0 CH[3:0] = 0x0005[3:0]000
3512 * UMC0 CH[7:4] = 0x0015[3:0]000
3513 * UMC1 CH[3:0] = 0x0025[3:0]000
3514 * UMC1 CH[7:4] = 0x0035[3:0]000
3516 static void gpu_get_err_info(struct mce
*m
, struct err_info
*err
)
3518 u8 ch
= (m
->ipid
& GENMASK(31, 0)) >> 20;
3519 u8 phy
= ((m
->ipid
>> 12) & 0xf);
3521 err
->channel
= ch
% 2 ? phy
+ 4 : phy
;
3525 static int gpu_addr_mask_to_cs_size(struct amd64_pvt
*pvt
, u8 umc
,
3526 unsigned int cs_mode
, int csrow_nr
)
3528 u32 addr_mask_orig
= pvt
->csels
[umc
].csmasks
[csrow_nr
];
3530 return __addr_mask_to_cs_size(addr_mask_orig
, cs_mode
, csrow_nr
, csrow_nr
>> 1);
3533 static void gpu_debug_display_dimm_sizes(struct amd64_pvt
*pvt
, u8 ctrl
)
3535 int size
, cs_mode
, cs
= 0;
3537 edac_printk(KERN_DEBUG
, EDAC_MC
, "UMC%d chip selects:\n", ctrl
);
3539 cs_mode
= CS_EVEN_PRIMARY
| CS_ODD_PRIMARY
;
3541 for_each_chip_select(cs
, ctrl
, pvt
) {
3542 size
= gpu_addr_mask_to_cs_size(pvt
, ctrl
, cs_mode
, cs
);
3543 amd64_info(EDAC_MC
": %d: %5dMB\n", cs
, size
);
3547 static void gpu_dump_misc_regs(struct amd64_pvt
*pvt
)
3549 struct amd64_umc
*umc
;
3555 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i
, umc
->umc_cfg
);
3556 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i
, umc
->sdp_ctrl
);
3557 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i
, umc
->ecc_ctrl
);
3558 edac_dbg(1, "UMC%d All HBMs support ECC: yes\n", i
);
3560 gpu_debug_display_dimm_sizes(pvt
, i
);
3564 static u32
gpu_get_csrow_nr_pages(struct amd64_pvt
*pvt
, u8 dct
, int csrow_nr
)
3567 int cs_mode
= CS_EVEN_PRIMARY
| CS_ODD_PRIMARY
;
3569 nr_pages
= gpu_addr_mask_to_cs_size(pvt
, dct
, cs_mode
, csrow_nr
);
3570 nr_pages
<<= 20 - PAGE_SHIFT
;
3572 edac_dbg(0, "csrow: %d, channel: %d\n", csrow_nr
, dct
);
3573 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages
);
3578 static void gpu_init_csrows(struct mem_ctl_info
*mci
)
3580 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3581 struct dimm_info
*dimm
;
3585 for_each_chip_select(cs
, umc
, pvt
) {
3586 if (!csrow_enabled(cs
, umc
, pvt
))
3589 dimm
= mci
->csrows
[umc
]->channels
[cs
]->dimm
;
3591 edac_dbg(1, "MC node: %d, csrow: %d\n",
3592 pvt
->mc_node_id
, cs
);
3594 dimm
->nr_pages
= gpu_get_csrow_nr_pages(pvt
, umc
, cs
);
3595 dimm
->edac_mode
= EDAC_SECDED
;
3596 dimm
->mtype
= pvt
->dram_type
;
3597 dimm
->dtype
= DEV_X16
;
3603 static void gpu_setup_mci_misc_attrs(struct mem_ctl_info
*mci
)
3605 struct amd64_pvt
*pvt
= mci
->pvt_info
;
3607 mci
->mtype_cap
= MEM_FLAG_HBM2
;
3608 mci
->edac_ctl_cap
= EDAC_FLAG_SECDED
;
3610 mci
->edac_cap
= EDAC_FLAG_EC
;
3611 mci
->mod_name
= EDAC_MOD_STR
;
3612 mci
->ctl_name
= pvt
->ctl_name
;
3613 mci
->dev_name
= pci_name(pvt
->F3
);
3614 mci
->ctl_page_to_phys
= NULL
;
3616 gpu_init_csrows(mci
);
3619 /* ECC is enabled by default on GPU nodes */
3620 static bool gpu_ecc_enabled(struct amd64_pvt
*pvt
)
3625 static inline u32
gpu_get_umc_base(struct amd64_pvt
*pvt
, u8 umc
, u8 channel
)
3628 * On CPUs, there is one channel per UMC, so UMC numbering equals
3629 * channel numbering. On GPUs, there are eight channels per UMC,
3630 * so the channel numbering is different from UMC numbering.
3632 * On CPU nodes channels are selected in 6th nibble
3633 * UMC chY[3:0]= [(chY*2 + 1) : (chY*2)]50000;
3635 * On GPU nodes channels are selected in 3rd nibble
3636 * HBM chX[3:0]= [Y ]5X[3:0]000;
3637 * HBM chX[7:4]= [Y+1]5X[3:0]000
3639 * On MI300 APU nodes, same as GPU nodes but channels are selected
3640 * in the base address of 0x90000
3647 return pvt
->gpu_umc_base
+ (umc
<< 20) + ((channel
% 4) << 12);
3650 static void gpu_read_mc_regs(struct amd64_pvt
*pvt
)
3652 u8 nid
= pvt
->mc_node_id
;
3653 struct amd64_umc
*umc
;
3656 /* Read registers from each UMC */
3658 umc_base
= gpu_get_umc_base(pvt
, i
, 0);
3661 amd_smn_read(nid
, umc_base
+ UMCCH_UMC_CFG
, &umc
->umc_cfg
);
3662 amd_smn_read(nid
, umc_base
+ UMCCH_SDP_CTRL
, &umc
->sdp_ctrl
);
3663 amd_smn_read(nid
, umc_base
+ UMCCH_ECC_CTRL
, &umc
->ecc_ctrl
);
3667 static void gpu_read_base_mask(struct amd64_pvt
*pvt
)
3669 u32 base_reg
, mask_reg
;
3674 for_each_chip_select(cs
, umc
, pvt
) {
3675 base_reg
= gpu_get_umc_base(pvt
, umc
, cs
) + UMCCH_BASE_ADDR
;
3676 base
= &pvt
->csels
[umc
].csbases
[cs
];
3678 if (!amd_smn_read(pvt
->mc_node_id
, base_reg
, base
)) {
3679 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
3680 umc
, cs
, *base
, base_reg
);
3683 mask_reg
= gpu_get_umc_base(pvt
, umc
, cs
) + UMCCH_ADDR_MASK
;
3684 mask
= &pvt
->csels
[umc
].csmasks
[cs
];
3686 if (!amd_smn_read(pvt
->mc_node_id
, mask_reg
, mask
)) {
3687 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
3688 umc
, cs
, *mask
, mask_reg
);
3694 static void gpu_prep_chip_selects(struct amd64_pvt
*pvt
)
3699 pvt
->csels
[umc
].b_cnt
= 8;
3700 pvt
->csels
[umc
].m_cnt
= 8;
3704 static int gpu_hw_info_get(struct amd64_pvt
*pvt
)
3708 ret
= gpu_get_node_map(pvt
);
3712 pvt
->umc
= kcalloc(pvt
->max_mcs
, sizeof(struct amd64_umc
), GFP_KERNEL
);
3716 gpu_prep_chip_selects(pvt
);
3717 gpu_read_base_mask(pvt
);
3718 gpu_read_mc_regs(pvt
);
3723 static void hw_info_put(struct amd64_pvt
*pvt
)
3725 pci_dev_put(pvt
->F1
);
3726 pci_dev_put(pvt
->F2
);
3730 static struct low_ops umc_ops
= {
3731 .hw_info_get
= umc_hw_info_get
,
3732 .ecc_enabled
= umc_ecc_enabled
,
3733 .setup_mci_misc_attrs
= umc_setup_mci_misc_attrs
,
3734 .dump_misc_regs
= umc_dump_misc_regs
,
3735 .get_err_info
= umc_get_err_info
,
3738 static struct low_ops gpu_ops
= {
3739 .hw_info_get
= gpu_hw_info_get
,
3740 .ecc_enabled
= gpu_ecc_enabled
,
3741 .setup_mci_misc_attrs
= gpu_setup_mci_misc_attrs
,
3742 .dump_misc_regs
= gpu_dump_misc_regs
,
3743 .get_err_info
= gpu_get_err_info
,
3746 /* Use Family 16h versions for defaults and adjust as needed below. */
3747 static struct low_ops dct_ops
= {
3748 .map_sysaddr_to_csrow
= f1x_map_sysaddr_to_csrow
,
3749 .dbam_to_cs
= f16_dbam_to_chip_select
,
3750 .hw_info_get
= dct_hw_info_get
,
3751 .ecc_enabled
= dct_ecc_enabled
,
3752 .setup_mci_misc_attrs
= dct_setup_mci_misc_attrs
,
3753 .dump_misc_regs
= dct_dump_misc_regs
,
3756 static int per_family_init(struct amd64_pvt
*pvt
)
3758 pvt
->ext_model
= boot_cpu_data
.x86_model
>> 4;
3759 pvt
->stepping
= boot_cpu_data
.x86_stepping
;
3760 pvt
->model
= boot_cpu_data
.x86_model
;
3761 pvt
->fam
= boot_cpu_data
.x86
;
3765 * Decide on which ops group to use here and do any family/model
3768 if (pvt
->fam
>= 0x17)
3769 pvt
->ops
= &umc_ops
;
3771 pvt
->ops
= &dct_ops
;
3775 pvt
->ctl_name
= (pvt
->ext_model
>= K8_REV_F
) ?
3776 "K8 revF or later" : "K8 revE or earlier";
3777 pvt
->f1_id
= PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP
;
3778 pvt
->f2_id
= PCI_DEVICE_ID_AMD_K8_NB_MEMCTL
;
3779 pvt
->ops
->map_sysaddr_to_csrow
= k8_map_sysaddr_to_csrow
;
3780 pvt
->ops
->dbam_to_cs
= k8_dbam_to_chip_select
;
3784 pvt
->ctl_name
= "F10h";
3785 pvt
->f1_id
= PCI_DEVICE_ID_AMD_10H_NB_MAP
;
3786 pvt
->f2_id
= PCI_DEVICE_ID_AMD_10H_NB_DRAM
;
3787 pvt
->ops
->dbam_to_cs
= f10_dbam_to_chip_select
;
3791 switch (pvt
->model
) {
3793 pvt
->ctl_name
= "F15h_M30h";
3794 pvt
->f1_id
= PCI_DEVICE_ID_AMD_15H_M30H_NB_F1
;
3795 pvt
->f2_id
= PCI_DEVICE_ID_AMD_15H_M30H_NB_F2
;
3798 pvt
->ctl_name
= "F15h_M60h";
3799 pvt
->f1_id
= PCI_DEVICE_ID_AMD_15H_M60H_NB_F1
;
3800 pvt
->f2_id
= PCI_DEVICE_ID_AMD_15H_M60H_NB_F2
;
3801 pvt
->ops
->dbam_to_cs
= f15_m60h_dbam_to_chip_select
;
3804 /* Richland is only client */
3807 pvt
->ctl_name
= "F15h";
3808 pvt
->f1_id
= PCI_DEVICE_ID_AMD_15H_NB_F1
;
3809 pvt
->f2_id
= PCI_DEVICE_ID_AMD_15H_NB_F2
;
3810 pvt
->ops
->dbam_to_cs
= f15_dbam_to_chip_select
;
3816 switch (pvt
->model
) {
3818 pvt
->ctl_name
= "F16h_M30h";
3819 pvt
->f1_id
= PCI_DEVICE_ID_AMD_16H_M30H_NB_F1
;
3820 pvt
->f2_id
= PCI_DEVICE_ID_AMD_16H_M30H_NB_F2
;
3823 pvt
->ctl_name
= "F16h";
3824 pvt
->f1_id
= PCI_DEVICE_ID_AMD_16H_NB_F1
;
3825 pvt
->f2_id
= PCI_DEVICE_ID_AMD_16H_NB_F2
;
3831 switch (pvt
->model
) {
3833 pvt
->ctl_name
= "F17h_M10h";
3836 pvt
->ctl_name
= "F17h_M30h";
3840 pvt
->ctl_name
= "F17h_M60h";
3843 pvt
->ctl_name
= "F17h_M70h";
3846 pvt
->ctl_name
= "F17h";
3852 pvt
->ctl_name
= "F18h";
3856 switch (pvt
->model
) {
3858 pvt
->ctl_name
= "F19h";
3862 pvt
->ctl_name
= "F19h_M10h";
3864 pvt
->flags
.zn_regs_v2
= 1;
3867 pvt
->ctl_name
= "F19h_M20h";
3870 if (pvt
->F3
->device
== PCI_DEVICE_ID_AMD_MI200_DF_F3
) {
3871 pvt
->ctl_name
= "MI200";
3873 pvt
->dram_type
= MEM_HBM2
;
3874 pvt
->gpu_umc_base
= 0x50000;
3875 pvt
->ops
= &gpu_ops
;
3877 pvt
->ctl_name
= "F19h_M30h";
3882 pvt
->ctl_name
= "F19h_M50h";
3885 pvt
->ctl_name
= "F19h_M60h";
3886 pvt
->flags
.zn_regs_v2
= 1;
3889 pvt
->ctl_name
= "F19h_M70h";
3890 pvt
->flags
.zn_regs_v2
= 1;
3893 pvt
->ctl_name
= "F19h_M90h";
3895 pvt
->dram_type
= MEM_HBM3
;
3896 pvt
->gpu_umc_base
= 0x90000;
3897 pvt
->ops
= &gpu_ops
;
3900 pvt
->ctl_name
= "F19h_MA0h";
3902 pvt
->flags
.zn_regs_v2
= 1;
3908 switch (pvt
->model
) {
3910 pvt
->ctl_name
= "F1Ah";
3912 pvt
->flags
.zn_regs_v2
= 1;
3915 pvt
->ctl_name
= "F1Ah_M40h";
3916 pvt
->flags
.zn_regs_v2
= 1;
3922 amd64_err("Unsupported family!\n");
3929 static const struct attribute_group
*amd64_edac_attr_groups
[] = {
3930 #ifdef CONFIG_EDAC_DEBUG
3938 * For heterogeneous and APU models EDAC CHIP_SELECT and CHANNEL layers
3939 * should be swapped to fit into the layers.
3941 static unsigned int get_layer_size(struct amd64_pvt
*pvt
, u8 layer
)
3943 bool is_gpu
= (pvt
->ops
== &gpu_ops
);
3946 return is_gpu
? pvt
->max_mcs
3947 : pvt
->csels
[0].b_cnt
;
3949 return is_gpu
? pvt
->csels
[0].b_cnt
3953 static int init_one_instance(struct amd64_pvt
*pvt
)
3955 struct mem_ctl_info
*mci
= NULL
;
3956 struct edac_mc_layer layers
[2];
3959 layers
[0].type
= EDAC_MC_LAYER_CHIP_SELECT
;
3960 layers
[0].size
= get_layer_size(pvt
, 0);
3961 layers
[0].is_virt_csrow
= true;
3962 layers
[1].type
= EDAC_MC_LAYER_CHANNEL
;
3963 layers
[1].size
= get_layer_size(pvt
, 1);
3964 layers
[1].is_virt_csrow
= false;
3966 mci
= edac_mc_alloc(pvt
->mc_node_id
, ARRAY_SIZE(layers
), layers
, 0);
3970 mci
->pvt_info
= pvt
;
3971 mci
->pdev
= &pvt
->F3
->dev
;
3973 pvt
->ops
->setup_mci_misc_attrs(mci
);
3976 if (edac_mc_add_mc_with_groups(mci
, amd64_edac_attr_groups
)) {
3977 edac_dbg(1, "failed edac_mc_add_mc()\n");
3985 static bool instance_has_memory(struct amd64_pvt
*pvt
)
3987 bool cs_enabled
= false;
3988 int cs
= 0, dct
= 0;
3990 for (dct
= 0; dct
< pvt
->max_mcs
; dct
++) {
3991 for_each_chip_select(cs
, dct
, pvt
)
3992 cs_enabled
|= csrow_enabled(cs
, dct
, pvt
);
3998 static int probe_one_instance(unsigned int nid
)
4000 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
4001 struct amd64_pvt
*pvt
= NULL
;
4002 struct ecc_settings
*s
;
4006 s
= kzalloc(sizeof(struct ecc_settings
), GFP_KERNEL
);
4012 pvt
= kzalloc(sizeof(struct amd64_pvt
), GFP_KERNEL
);
4016 pvt
->mc_node_id
= nid
;
4019 ret
= per_family_init(pvt
);
4023 ret
= pvt
->ops
->hw_info_get(pvt
);
4028 if (!instance_has_memory(pvt
)) {
4029 amd64_info("Node %d: No DIMMs detected.\n", nid
);
4033 if (!pvt
->ops
->ecc_enabled(pvt
)) {
4036 if (!ecc_enable_override
)
4039 if (boot_cpu_data
.x86
>= 0x17) {
4040 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
4043 amd64_warn("Forcing ECC on!\n");
4045 if (!enable_ecc_error_reporting(s
, nid
, F3
))
4049 ret
= init_one_instance(pvt
);
4051 amd64_err("Error probing instance: %d\n", nid
);
4053 if (boot_cpu_data
.x86
< 0x17)
4054 restore_ecc_error_reporting(s
, nid
, F3
);
4059 amd64_info("%s detected (node %d).\n", pvt
->ctl_name
, pvt
->mc_node_id
);
4061 /* Display and decode various registers for debug purposes. */
4062 pvt
->ops
->dump_misc_regs(pvt
);
4072 ecc_stngs
[nid
] = NULL
;
4078 static void remove_one_instance(unsigned int nid
)
4080 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
4081 struct ecc_settings
*s
= ecc_stngs
[nid
];
4082 struct mem_ctl_info
*mci
;
4083 struct amd64_pvt
*pvt
;
4085 /* Remove from EDAC CORE tracking list */
4086 mci
= edac_mc_del_mc(&F3
->dev
);
4090 pvt
= mci
->pvt_info
;
4092 restore_ecc_error_reporting(s
, nid
, F3
);
4094 kfree(ecc_stngs
[nid
]);
4095 ecc_stngs
[nid
] = NULL
;
4097 /* Free the EDAC CORE resources */
4098 mci
->pvt_info
= NULL
;
4105 static void setup_pci_device(void)
4110 pci_ctl
= edac_pci_create_generic_ctl(pci_ctl_dev
, EDAC_MOD_STR
);
4112 pr_warn("%s(): Unable to create PCI control\n", __func__
);
4113 pr_warn("%s(): PCI error report via EDAC not set\n", __func__
);
4117 static const struct x86_cpu_id amd64_cpuids
[] = {
4118 X86_MATCH_VENDOR_FAM(AMD
, 0x0F, NULL
),
4119 X86_MATCH_VENDOR_FAM(AMD
, 0x10, NULL
),
4120 X86_MATCH_VENDOR_FAM(AMD
, 0x15, NULL
),
4121 X86_MATCH_VENDOR_FAM(AMD
, 0x16, NULL
),
4122 X86_MATCH_VENDOR_FAM(AMD
, 0x17, NULL
),
4123 X86_MATCH_VENDOR_FAM(HYGON
, 0x18, NULL
),
4124 X86_MATCH_VENDOR_FAM(AMD
, 0x19, NULL
),
4125 X86_MATCH_VENDOR_FAM(AMD
, 0x1A, NULL
),
4128 MODULE_DEVICE_TABLE(x86cpu
, amd64_cpuids
);
4130 static int __init
amd64_edac_init(void)
4136 if (ghes_get_devices())
4139 owner
= edac_get_owner();
4140 if (owner
&& strncmp(owner
, EDAC_MOD_STR
, sizeof(EDAC_MOD_STR
)))
4143 if (!x86_match_cpu(amd64_cpuids
))
4152 ecc_stngs
= kcalloc(amd_nb_num(), sizeof(ecc_stngs
[0]), GFP_KERNEL
);
4156 msrs
= msrs_alloc();
4160 for (i
= 0; i
< amd_nb_num(); i
++) {
4161 err
= probe_one_instance(i
);
4163 /* unwind properly */
4165 remove_one_instance(i
);
4171 if (!edac_has_mcs()) {
4176 /* register stuff with EDAC MCE */
4177 if (boot_cpu_data
.x86
>= 0x17) {
4178 amd_register_ecc_decoder(decode_umc_error
);
4180 amd_register_ecc_decoder(decode_bus_error
);
4184 #ifdef CONFIG_X86_32
4185 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR
);
4203 static void __exit
amd64_edac_exit(void)
4208 edac_pci_release_generic_ctl(pci_ctl
);
4210 /* unregister from EDAC MCE */
4211 if (boot_cpu_data
.x86
>= 0x17)
4212 amd_unregister_ecc_decoder(decode_umc_error
);
4214 amd_unregister_ecc_decoder(decode_bus_error
);
4216 for (i
= 0; i
< amd_nb_num(); i
++)
4217 remove_one_instance(i
);
4228 module_init(amd64_edac_init
);
4229 module_exit(amd64_edac_exit
);
4231 MODULE_LICENSE("GPL");
4232 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, Dave Peterson, Thayne Harbaugh; AMD");
4233 MODULE_DESCRIPTION("MC support for AMD64 memory controllers");
4235 module_param(edac_op_state
, int, 0444);
4236 MODULE_PARM_DESC(edac_op_state
, "EDAC Error Reporting state: 0=Poll,1=NMI");