1 // SPDX-License-Identifier: GPL-2.0-only
2 #include "amd64_edac.h"
3 #include <asm/amd_nb.h>
5 static struct edac_pci_ctl_info *pci_ctl;
7 static int report_gart_errors;
8 module_param(report_gart_errors, int, 0644);
11 * Set by command line parameter. If BIOS has enabled the ECC, this override is
12 * cleared to prevent re-enabling the hardware by this driver.
14 static int ecc_enable_override;
15 module_param(ecc_enable_override, int, 0644);
17 static struct msr __percpu *msrs;
20 static struct ecc_settings **ecc_stngs;
22 /* Number of Unified Memory Controllers */
26 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
27 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 *FIXME: Produce a better mapping/linearisation.
32 static const struct scrubrate {
33 u32 scrubval; /* bit pattern for scrub rate */
34 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
36 { 0x01, 1600000000UL},
58 { 0x00, 0UL}, /* scrubbing off */
61 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
62 u32 *val, const char *func)
66 err = pci_read_config_dword(pdev, offset, val);
68 amd64_warn("%s: error reading F%dx%03x.\n",
69 func, PCI_FUNC(pdev->devfn), offset);
74 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
75 u32 val, const char *func)
79 err = pci_write_config_dword(pdev, offset, val);
81 amd64_warn("%s: error writing to F%dx%03x.\n",
82 func, PCI_FUNC(pdev->devfn), offset);
88 * Select DCT to which PCI cfg accesses are routed
90 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
94 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
95 reg &= (pvt->model == 0x30) ? ~3 : ~1;
97 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
102 * Depending on the family, F2 DCT reads need special handling:
104 * K8: has a single DCT only and no address offsets >= 0x100
106 * F10h: each DCT has its own set of regs
110 * F16h: has only 1 DCT
112 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
114 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
115 int offset, u32 *val)
119 if (dct || offset >= 0x100)
126 * Note: If ganging is enabled, barring the regs
127 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
128 * return 0. (cf. Section 2.8.1 F10h BKDG)
130 if (dct_ganging_enabled(pvt))
139 * F15h: F2x1xx addresses do not map explicitly to DCT1.
140 * We should select which DCT we access using F1x10C[DctCfgSel]
142 dct = (dct && pvt->model == 0x30) ? 3 : dct;
143 f15h_select_dct(pvt, dct);
154 return amd64_read_pci_cfg(pvt->F2, offset, val);
158 * Memory scrubber control interface. For K8, memory scrubbing is handled by
159 * hardware and can involve L2 cache, dcache as well as the main memory. With
160 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
163 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
164 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
165 * bytes/sec for the setting.
167 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
168 * other archs, we might not have access to the caches directly.
171 static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
174 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
175 * are shifted down by 0x5, so scrubval 0x5 is written to the register
176 * as 0x0, scrubval 0x6 as 0x1, etc.
178 if (scrubval >= 0x5 && scrubval <= 0x14) {
180 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
181 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
183 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
187 * Scan the scrub rate mapping table for a close or matching bandwidth value to
188 * issue. If requested is too big, then use last maximum value found.
190 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
196 * map the configured rate (new_bw) to a value specific to the AMD64
197 * memory controller and apply to register. Search for the first
198 * bandwidth entry that is greater or equal than the setting requested
199 * and program that. If at last entry, turn off DRAM scrubbing.
201 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
202 * by falling back to the last element in scrubrates[].
204 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
206 * skip scrub rates which aren't recommended
207 * (see F10 BKDG, F3x58)
209 if (scrubrates[i].scrubval < min_rate)
212 if (scrubrates[i].bandwidth <= new_bw)
216 scrubval = scrubrates[i].scrubval;
218 if (pvt->fam == 0x17 || pvt->fam == 0x18) {
219 __f17h_set_scrubval(pvt, scrubval);
220 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
221 f15h_select_dct(pvt, 0);
222 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
223 f15h_select_dct(pvt, 1);
224 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
226 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
230 return scrubrates[i].bandwidth;
235 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
237 struct amd64_pvt *pvt = mci->pvt_info;
238 u32 min_scrubrate = 0x5;
243 if (pvt->fam == 0x15) {
245 if (pvt->model < 0x10)
246 f15h_select_dct(pvt, 0);
248 if (pvt->model == 0x60)
251 return __set_scrub_rate(pvt, bw, min_scrubrate);
254 static int get_scrub_rate(struct mem_ctl_info *mci)
256 struct amd64_pvt *pvt = mci->pvt_info;
257 int i, retval = -EINVAL;
263 if (pvt->model < 0x10)
264 f15h_select_dct(pvt, 0);
266 if (pvt->model == 0x60)
267 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
272 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
273 if (scrubval & BIT(0)) {
274 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
283 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
287 scrubval = scrubval & 0x001F;
289 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
290 if (scrubrates[i].scrubval == scrubval) {
291 retval = scrubrates[i].bandwidth;
299 * returns true if the SysAddr given by sys_addr matches the
300 * DRAM base/limit associated with node_id
302 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
306 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
307 * all ones if the most significant implemented address bit is 1.
308 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
309 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
310 * Application Programming.
312 addr = sys_addr & 0x000000ffffffffffull;
314 return ((addr >= get_dram_base(pvt, nid)) &&
315 (addr <= get_dram_limit(pvt, nid)));
319 * Attempt to map a SysAddr to a node. On success, return a pointer to the
320 * mem_ctl_info structure for the node that the SysAddr maps to.
322 * On failure, return NULL.
324 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
327 struct amd64_pvt *pvt;
332 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
333 * 3.4.4.2) registers to map the SysAddr to a node ID.
338 * The value of this field should be the same for all DRAM Base
339 * registers. Therefore we arbitrarily choose to read it from the
340 * register for node 0.
342 intlv_en = dram_intlv_en(pvt, 0);
345 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
346 if (base_limit_match(pvt, sys_addr, node_id))
352 if (unlikely((intlv_en != 0x01) &&
353 (intlv_en != 0x03) &&
354 (intlv_en != 0x07))) {
355 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
359 bits = (((u32) sys_addr) >> 12) & intlv_en;
361 for (node_id = 0; ; ) {
362 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
363 break; /* intlv_sel field matches */
365 if (++node_id >= DRAM_RANGES)
369 /* sanity test for sys_addr */
370 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
371 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
372 "range for node %d with node interleaving enabled.\n",
373 __func__, sys_addr, node_id);
378 return edac_mc_find((int)node_id);
381 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
382 (unsigned long)sys_addr);
388 * compute the CS base address of the @csrow on the DRAM controller @dct.
389 * For details see F2x[5C:40] in the processor's BKDG
391 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
392 u64 *base, u64 *mask)
394 u64 csbase, csmask, base_bits, mask_bits;
397 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
398 csbase = pvt->csels[dct].csbases[csrow];
399 csmask = pvt->csels[dct].csmasks[csrow];
400 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
401 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
405 * F16h and F15h, models 30h and later need two addr_shift values:
406 * 8 for high and 6 for low (cf. F16h BKDG).
408 } else if (pvt->fam == 0x16 ||
409 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
410 csbase = pvt->csels[dct].csbases[csrow];
411 csmask = pvt->csels[dct].csmasks[csrow >> 1];
413 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
414 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
417 /* poke holes for the csmask */
418 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
419 (GENMASK_ULL(30, 19) << 8));
421 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
422 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
426 csbase = pvt->csels[dct].csbases[csrow];
427 csmask = pvt->csels[dct].csmasks[csrow >> 1];
430 if (pvt->fam == 0x15)
431 base_bits = mask_bits =
432 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
434 base_bits = mask_bits =
435 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
438 *base = (csbase & base_bits) << addr_shift;
441 /* poke holes for the csmask */
442 *mask &= ~(mask_bits << addr_shift);
444 *mask |= (csmask & mask_bits) << addr_shift;
447 #define for_each_chip_select(i, dct, pvt) \
448 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
450 #define chip_select_base(i, dct, pvt) \
451 pvt->csels[dct].csbases[i]
453 #define for_each_chip_select_mask(i, dct, pvt) \
454 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
456 #define for_each_umc(i) \
457 for (i = 0; i < num_umcs; i++)
460 * @input_addr is an InputAddr associated with the node given by mci. Return the
461 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
463 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
465 struct amd64_pvt *pvt;
471 for_each_chip_select(csrow, 0, pvt) {
472 if (!csrow_enabled(csrow, 0, pvt))
475 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
479 if ((input_addr & mask) == (base & mask)) {
480 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
481 (unsigned long)input_addr, csrow,
487 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
488 (unsigned long)input_addr, pvt->mc_node_id);
494 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
495 * for the node represented by mci. Info is passed back in *hole_base,
496 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
497 * info is invalid. Info may be invalid for either of the following reasons:
499 * - The revision of the node is not E or greater. In this case, the DRAM Hole
500 * Address Register does not exist.
502 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
503 * indicating that its contents are not valid.
505 * The values passed back in *hole_base, *hole_offset, and *hole_size are
506 * complete 32-bit values despite the fact that the bitfields in the DHAR
507 * only represent bits 31-24 of the base and offset values.
509 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
510 u64 *hole_offset, u64 *hole_size)
512 struct amd64_pvt *pvt = mci->pvt_info;
514 /* only revE and later have the DRAM Hole Address Register */
515 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
516 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
517 pvt->ext_model, pvt->mc_node_id);
521 /* valid for Fam10h and above */
522 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
523 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
527 if (!dhar_valid(pvt)) {
528 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
533 /* This node has Memory Hoisting */
535 /* +------------------+--------------------+--------------------+-----
536 * | memory | DRAM hole | relocated |
537 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
539 * | | | [0x100000000, |
540 * | | | (0x100000000+ |
541 * | | | (0xffffffff-x))] |
542 * +------------------+--------------------+--------------------+-----
544 * Above is a diagram of physical memory showing the DRAM hole and the
545 * relocated addresses from the DRAM hole. As shown, the DRAM hole
546 * starts at address x (the base address) and extends through address
547 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
548 * addresses in the hole so that they start at 0x100000000.
551 *hole_base = dhar_base(pvt);
552 *hole_size = (1ULL << 32) - *hole_base;
554 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
555 : k8_dhar_offset(pvt);
557 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
558 pvt->mc_node_id, (unsigned long)*hole_base,
559 (unsigned long)*hole_offset, (unsigned long)*hole_size);
563 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
566 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
567 * assumed that sys_addr maps to the node given by mci.
569 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
570 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
571 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
572 * then it is also involved in translating a SysAddr to a DramAddr. Sections
573 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
574 * These parts of the documentation are unclear. I interpret them as follows:
576 * When node n receives a SysAddr, it processes the SysAddr as follows:
578 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
579 * Limit registers for node n. If the SysAddr is not within the range
580 * specified by the base and limit values, then node n ignores the Sysaddr
581 * (since it does not map to node n). Otherwise continue to step 2 below.
583 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
584 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
585 * the range of relocated addresses (starting at 0x100000000) from the DRAM
586 * hole. If not, skip to step 3 below. Else get the value of the
587 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
588 * offset defined by this value from the SysAddr.
590 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
591 * Base register for node n. To obtain the DramAddr, subtract the base
592 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
594 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
596 struct amd64_pvt *pvt = mci->pvt_info;
597 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
600 dram_base = get_dram_base(pvt, pvt->mc_node_id);
602 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
605 if ((sys_addr >= (1ULL << 32)) &&
606 (sys_addr < ((1ULL << 32) + hole_size))) {
607 /* use DHAR to translate SysAddr to DramAddr */
608 dram_addr = sys_addr - hole_offset;
610 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
611 (unsigned long)sys_addr,
612 (unsigned long)dram_addr);
619 * Translate the SysAddr to a DramAddr as shown near the start of
620 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
621 * only deals with 40-bit values. Therefore we discard bits 63-40 of
622 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
623 * discard are all 1s. Otherwise the bits we discard are all 0s. See
624 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
625 * Programmer's Manual Volume 1 Application Programming.
627 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
629 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
630 (unsigned long)sys_addr, (unsigned long)dram_addr);
635 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
636 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
637 * for node interleaving.
639 static int num_node_interleave_bits(unsigned intlv_en)
641 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
644 BUG_ON(intlv_en > 7);
645 n = intlv_shift_table[intlv_en];
649 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
650 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
652 struct amd64_pvt *pvt;
659 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
660 * concerning translating a DramAddr to an InputAddr.
662 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
663 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
666 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
667 intlv_shift, (unsigned long)dram_addr,
668 (unsigned long)input_addr);
674 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
675 * assumed that @sys_addr maps to the node given by mci.
677 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
682 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
684 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
685 (unsigned long)sys_addr, (unsigned long)input_addr);
690 /* Map the Error address to a PAGE and PAGE OFFSET. */
691 static inline void error_address_to_page_and_offset(u64 error_address,
692 struct err_info *err)
694 err->page = (u32) (error_address >> PAGE_SHIFT);
695 err->offset = ((u32) error_address) & ~PAGE_MASK;
699 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
700 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
701 * of a node that detected an ECC memory error. mci represents the node that
702 * the error address maps to (possibly different from the node that detected
703 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
706 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
710 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
713 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
714 "address 0x%lx\n", (unsigned long)sys_addr);
718 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
721 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
724 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
726 unsigned long edac_cap = EDAC_FLAG_NONE;
730 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
733 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
736 umc_en_mask |= BIT(i);
738 /* UMC Configuration bit 12 (DimmEccEn) */
739 if (pvt->umc[i].umc_cfg & BIT(12))
740 dimm_ecc_en_mask |= BIT(i);
743 if (umc_en_mask == dimm_ecc_en_mask)
744 edac_cap = EDAC_FLAG_SECDED;
746 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
750 if (pvt->dclr0 & BIT(bit))
751 edac_cap = EDAC_FLAG_SECDED;
757 static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
759 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
761 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
763 if (pvt->dram_type == MEM_LRDDR3) {
764 u32 dcsm = pvt->csels[chan].csmasks[0];
766 * It's assumed all LRDIMMs in a DCT are going to be of
767 * same 'type' until proven otherwise. So, use a cs
768 * value of '0' here to get dcsm value.
770 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
773 edac_dbg(1, "All DIMMs support ECC:%s\n",
774 (dclr & BIT(19)) ? "yes" : "no");
777 edac_dbg(1, " PAR/ERR parity: %s\n",
778 (dclr & BIT(8)) ? "enabled" : "disabled");
780 if (pvt->fam == 0x10)
781 edac_dbg(1, " DCT 128bit mode width: %s\n",
782 (dclr & BIT(11)) ? "128b" : "64b");
784 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
785 (dclr & BIT(12)) ? "yes" : "no",
786 (dclr & BIT(13)) ? "yes" : "no",
787 (dclr & BIT(14)) ? "yes" : "no",
788 (dclr & BIT(15)) ? "yes" : "no");
792 * The Address Mask should be a contiguous set of bits in the non-interleaved
793 * case. So to check for CS interleaving, find the most- and least-significant
794 * bits of the mask, generate a contiguous bitmask, and compare the two.
796 static bool f17_cs_interleaved(struct amd64_pvt *pvt, u8 ctrl, int cs)
798 u32 mask = pvt->csels[ctrl].csmasks[cs >> 1];
799 u32 msb = fls(mask) - 1, lsb = ffs(mask) - 1;
800 u32 test_mask = GENMASK(msb, lsb);
802 edac_dbg(1, "mask=0x%08x test_mask=0x%08x\n", mask, test_mask);
804 return mask ^ test_mask;
807 static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
809 int dimm, size0, size1, cs0, cs1;
811 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
813 for (dimm = 0; dimm < 4; dimm++) {
817 if (csrow_enabled(cs0, ctrl, pvt))
818 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs0);
823 if (csrow_enabled(cs1, ctrl, pvt)) {
825 * CS interleaving is only supported if both CSes have
826 * the same amount of memory. Because they are
827 * interleaved, it will look like both CSes have the
828 * full amount of memory. Save the size for both as
829 * half the amount we found on CS0, if interleaved.
831 if (f17_cs_interleaved(pvt, ctrl, cs1))
832 size1 = size0 = (size0 >> 1);
834 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs1);
837 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
843 static void __dump_misc_regs_df(struct amd64_pvt *pvt)
845 struct amd64_umc *umc;
846 u32 i, tmp, umc_base;
849 umc_base = get_umc_base(i);
852 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
853 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
854 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
855 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
857 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
858 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
860 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
861 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
862 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
864 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
865 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
866 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
867 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
868 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
869 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
870 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
871 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
872 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
874 if (pvt->dram_type == MEM_LRDDR4) {
875 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
876 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
877 i, 1 << ((tmp >> 4) & 0x3));
880 debug_display_dimm_sizes_df(pvt, i);
883 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
884 pvt->dhar, dhar_base(pvt));
887 /* Display and decode various NB registers for debug purposes. */
888 static void __dump_misc_regs(struct amd64_pvt *pvt)
890 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
892 edac_dbg(1, " NB two channel DRAM capable: %s\n",
893 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
895 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
896 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
897 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
899 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
901 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
903 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
904 pvt->dhar, dhar_base(pvt),
905 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
906 : f10_dhar_offset(pvt));
908 debug_display_dimm_sizes(pvt, 0);
910 /* everything below this point is Fam10h and above */
914 debug_display_dimm_sizes(pvt, 1);
916 /* Only if NOT ganged does dclr1 have valid info */
917 if (!dct_ganging_enabled(pvt))
918 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
921 /* Display and decode various NB registers for debug purposes. */
922 static void dump_misc_regs(struct amd64_pvt *pvt)
925 __dump_misc_regs_df(pvt);
927 __dump_misc_regs(pvt);
929 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
931 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
935 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
937 static void prep_chip_selects(struct amd64_pvt *pvt)
939 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
940 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
941 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
942 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
943 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
944 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
946 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
947 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
952 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
954 static void read_dct_base_mask(struct amd64_pvt *pvt)
956 int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
958 prep_chip_selects(pvt);
961 base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
962 base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
963 mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
964 mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
972 for_each_chip_select(cs, 0, pvt) {
973 int reg0 = base_reg0 + (cs * 4);
974 int reg1 = base_reg1 + (cs * 4);
975 u32 *base0 = &pvt->csels[0].csbases[cs];
976 u32 *base1 = &pvt->csels[1].csbases[cs];
979 if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
980 edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
983 if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
984 edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
987 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
988 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
994 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
995 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
996 cs, *base1, (pvt->fam == 0x10) ? reg1
1001 for_each_chip_select_mask(cs, 0, pvt) {
1002 int reg0 = mask_reg0 + (cs * 4);
1003 int reg1 = mask_reg1 + (cs * 4);
1004 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1005 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1008 if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
1009 edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
1012 if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
1013 edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
1016 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1017 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1020 if (pvt->fam == 0xf)
1023 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1024 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1025 cs, *mask1, (pvt->fam == 0x10) ? reg1
1031 static void determine_memory_type(struct amd64_pvt *pvt)
1033 u32 dram_ctrl, dcsm;
1037 if (pvt->ext_model >= K8_REV_F)
1040 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1044 if (pvt->dchr0 & DDR3_MODE)
1047 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1051 if (pvt->model < 0x60)
1055 * Model 0x60h needs special handling:
1057 * We use a Chip Select value of '0' to obtain dcsm.
1058 * Theoretically, it is possible to populate LRDIMMs of different
1059 * 'Rank' value on a DCT. But this is not the common case. So,
1060 * it's reasonable to assume all DIMMs are going to be of same
1061 * 'type' until proven otherwise.
1063 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1064 dcsm = pvt->csels[0].csmasks[0];
1066 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1067 pvt->dram_type = MEM_DDR4;
1068 else if (pvt->dclr0 & BIT(16))
1069 pvt->dram_type = MEM_DDR3;
1070 else if (dcsm & 0x3)
1071 pvt->dram_type = MEM_LRDDR3;
1073 pvt->dram_type = MEM_RDDR3;
1082 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1083 pvt->dram_type = MEM_LRDDR4;
1084 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1085 pvt->dram_type = MEM_RDDR4;
1087 pvt->dram_type = MEM_DDR4;
1091 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1092 pvt->dram_type = MEM_EMPTY;
1097 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1100 /* Get the number of DCT channels the memory controller is using. */
1101 static int k8_early_channel_count(struct amd64_pvt *pvt)
1105 if (pvt->ext_model >= K8_REV_F)
1106 /* RevF (NPT) and later */
1107 flag = pvt->dclr0 & WIDTH_128;
1109 /* RevE and earlier */
1110 flag = pvt->dclr0 & REVE_WIDTH_128;
1115 return (flag) ? 2 : 1;
1118 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1119 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1121 u16 mce_nid = amd_get_nb_id(m->extcpu);
1122 struct mem_ctl_info *mci;
1127 mci = edac_mc_find(mce_nid);
1131 pvt = mci->pvt_info;
1133 if (pvt->fam == 0xf) {
1138 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1141 * Erratum 637 workaround
1143 if (pvt->fam == 0x15) {
1144 u64 cc6_base, tmp_addr;
1148 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1152 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1153 intlv_en = tmp >> 21 & 0x7;
1155 /* add [47:27] + 3 trailing bits */
1156 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1158 /* reverse and add DramIntlvEn */
1159 cc6_base |= intlv_en ^ 0x7;
1161 /* pin at [47:24] */
1165 return cc6_base | (addr & GENMASK_ULL(23, 0));
1167 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1170 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1172 /* OR DramIntlvSel into bits [14:12] */
1173 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1175 /* add remaining [11:0] bits from original MC4_ADDR */
1176 tmp_addr |= addr & GENMASK_ULL(11, 0);
1178 return cc6_base | tmp_addr;
1184 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1185 unsigned int device,
1186 struct pci_dev *related)
1188 struct pci_dev *dev = NULL;
1190 while ((dev = pci_get_device(vendor, device, dev))) {
1191 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1192 (dev->bus->number == related->bus->number) &&
1193 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1200 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1202 struct amd_northbridge *nb;
1203 struct pci_dev *f1 = NULL;
1204 unsigned int pci_func;
1205 int off = range << 3;
1208 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1209 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1211 if (pvt->fam == 0xf)
1214 if (!dram_rw(pvt, range))
1217 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1218 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1220 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1221 if (pvt->fam != 0x15)
1224 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1228 if (pvt->model == 0x60)
1229 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1230 else if (pvt->model == 0x30)
1231 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1233 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1235 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1239 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1241 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1243 /* {[39:27],111b} */
1244 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1246 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1249 pvt->ranges[range].lim.hi |= llim >> 13;
1254 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1255 struct err_info *err)
1257 struct amd64_pvt *pvt = mci->pvt_info;
1259 error_address_to_page_and_offset(sys_addr, err);
1262 * Find out which node the error address belongs to. This may be
1263 * different from the node that detected the error.
1265 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1266 if (!err->src_mci) {
1267 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1268 (unsigned long)sys_addr);
1269 err->err_code = ERR_NODE;
1273 /* Now map the sys_addr to a CSROW */
1274 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1275 if (err->csrow < 0) {
1276 err->err_code = ERR_CSROW;
1280 /* CHIPKILL enabled */
1281 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1282 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1283 if (err->channel < 0) {
1285 * Syndrome didn't map, so we don't know which of the
1286 * 2 DIMMs is in error. So we need to ID 'both' of them
1289 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1290 "possible error reporting race\n",
1292 err->err_code = ERR_CHANNEL;
1297 * non-chipkill ecc mode
1299 * The k8 documentation is unclear about how to determine the
1300 * channel number when using non-chipkill memory. This method
1301 * was obtained from email communication with someone at AMD.
1302 * (Wish the email was placed in this comment - norsk)
1304 err->channel = ((sys_addr & BIT(3)) != 0);
1308 static int ddr2_cs_size(unsigned i, bool dct_width)
1314 else if (!(i & 0x1))
1317 shift = (i + 1) >> 1;
1319 return 128 << (shift + !!dct_width);
1322 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1323 unsigned cs_mode, int cs_mask_nr)
1325 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1327 if (pvt->ext_model >= K8_REV_F) {
1328 WARN_ON(cs_mode > 11);
1329 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1331 else if (pvt->ext_model >= K8_REV_D) {
1333 WARN_ON(cs_mode > 10);
1336 * the below calculation, besides trying to win an obfuscated C
1337 * contest, maps cs_mode values to DIMM chip select sizes. The
1340 * cs_mode CS size (mb)
1341 * ======= ============
1354 * Basically, it calculates a value with which to shift the
1355 * smallest CS size of 32MB.
1357 * ddr[23]_cs_size have a similar purpose.
1359 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1361 return 32 << (cs_mode - diff);
1364 WARN_ON(cs_mode > 6);
1365 return 32 << cs_mode;
1370 * Get the number of DCT channels in use.
1373 * number of Memory Channels in operation
1375 * contents of the DCL0_LOW register
1377 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1379 int i, j, channels = 0;
1381 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1382 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1386 * Need to check if in unganged mode: In such, there are 2 channels,
1387 * but they are not in 128 bit mode and thus the above 'dclr0' status
1390 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1391 * their CSEnable bit on. If so, then SINGLE DIMM case.
1393 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1396 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1397 * is more than just one DIMM present in unganged mode. Need to check
1398 * both controllers since DIMMs can be placed in either one.
1400 for (i = 0; i < 2; i++) {
1401 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1403 for (j = 0; j < 4; j++) {
1404 if (DBAM_DIMM(j, dbam) > 0) {
1414 amd64_info("MCT channel count: %d\n", channels);
1419 static int f17_early_channel_count(struct amd64_pvt *pvt)
1421 int i, channels = 0;
1423 /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
1425 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1427 amd64_info("MCT channel count: %d\n", channels);
1432 static int ddr3_cs_size(unsigned i, bool dct_width)
1437 if (i == 0 || i == 3 || i == 4)
1443 else if (!(i & 0x1))
1446 shift = (i + 1) >> 1;
1449 cs_size = (128 * (1 << !!dct_width)) << shift;
1454 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1459 if (i < 4 || i == 6)
1463 else if (!(i & 0x1))
1466 shift = (i + 1) >> 1;
1469 cs_size = rank_multiply * (128 << shift);
1474 static int ddr4_cs_size(unsigned i)
1483 /* Min cs_size = 1G */
1484 cs_size = 1024 * (1 << (i >> 1));
1489 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1490 unsigned cs_mode, int cs_mask_nr)
1492 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1494 WARN_ON(cs_mode > 11);
1496 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1497 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1499 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1503 * F15h supports only 64bit DCT interfaces
1505 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1506 unsigned cs_mode, int cs_mask_nr)
1508 WARN_ON(cs_mode > 12);
1510 return ddr3_cs_size(cs_mode, false);
1513 /* F15h M60h supports DDR4 mapping as well.. */
1514 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1515 unsigned cs_mode, int cs_mask_nr)
1518 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1520 WARN_ON(cs_mode > 12);
1522 if (pvt->dram_type == MEM_DDR4) {
1526 cs_size = ddr4_cs_size(cs_mode);
1527 } else if (pvt->dram_type == MEM_LRDDR3) {
1528 unsigned rank_multiply = dcsm & 0xf;
1530 if (rank_multiply == 3)
1532 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1534 /* Minimum cs size is 512mb for F15hM60h*/
1538 cs_size = ddr3_cs_size(cs_mode, false);
1545 * F16h and F15h model 30h have only limited cs_modes.
1547 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1548 unsigned cs_mode, int cs_mask_nr)
1550 WARN_ON(cs_mode > 12);
1552 if (cs_mode == 6 || cs_mode == 8 ||
1553 cs_mode == 9 || cs_mode == 12)
1556 return ddr3_cs_size(cs_mode, false);
1559 static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1560 unsigned int cs_mode, int csrow_nr)
1562 u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
1564 /* Each mask is used for every two base addresses. */
1565 u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
1567 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1568 u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
1570 edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
1572 /* Return size in MBs. */
1576 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1579 if (pvt->fam == 0xf)
1582 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1583 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1584 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1586 edac_dbg(0, " DCTs operate in %s mode\n",
1587 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1589 if (!dct_ganging_enabled(pvt))
1590 edac_dbg(0, " Address range split per DCT: %s\n",
1591 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1593 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1594 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1595 (dct_memory_cleared(pvt) ? "yes" : "no"));
1597 edac_dbg(0, " channel interleave: %s, "
1598 "interleave bits selector: 0x%x\n",
1599 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1600 dct_sel_interleave_addr(pvt));
1603 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1607 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1608 * 2.10.12 Memory Interleaving Modes).
1610 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1611 u8 intlv_en, int num_dcts_intlv,
1618 return (u8)(dct_sel);
1620 if (num_dcts_intlv == 2) {
1621 select = (sys_addr >> 8) & 0x3;
1622 channel = select ? 0x3 : 0;
1623 } else if (num_dcts_intlv == 4) {
1624 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1625 switch (intlv_addr) {
1627 channel = (sys_addr >> 8) & 0x3;
1630 channel = (sys_addr >> 9) & 0x3;
1638 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1639 * Interleaving Modes.
1641 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1642 bool hi_range_sel, u8 intlv_en)
1644 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1646 if (dct_ganging_enabled(pvt))
1650 return dct_sel_high;
1653 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1655 if (dct_interleave_enabled(pvt)) {
1656 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1658 /* return DCT select function: 0=DCT0, 1=DCT1 */
1660 return sys_addr >> 6 & 1;
1662 if (intlv_addr & 0x2) {
1663 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1664 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
1666 return ((sys_addr >> shift) & 1) ^ temp;
1669 if (intlv_addr & 0x4) {
1670 u8 shift = intlv_addr & 0x1 ? 9 : 8;
1672 return (sys_addr >> shift) & 1;
1675 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1678 if (dct_high_range_enabled(pvt))
1679 return ~dct_sel_high & 1;
1684 /* Convert the sys_addr to the normalized DCT address */
1685 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1686 u64 sys_addr, bool hi_rng,
1687 u32 dct_sel_base_addr)
1690 u64 dram_base = get_dram_base(pvt, range);
1691 u64 hole_off = f10_dhar_offset(pvt);
1692 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1697 * base address of high range is below 4Gb
1698 * (bits [47:27] at [31:11])
1699 * DRAM address space on this DCT is hoisted above 4Gb &&
1702 * remove hole offset from sys_addr
1704 * remove high range offset from sys_addr
1706 if ((!(dct_sel_base_addr >> 16) ||
1707 dct_sel_base_addr < dhar_base(pvt)) &&
1709 (sys_addr >= BIT_64(32)))
1710 chan_off = hole_off;
1712 chan_off = dct_sel_base_off;
1716 * we have a valid hole &&
1721 * remove dram base to normalize to DCT address
1723 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1724 chan_off = hole_off;
1726 chan_off = dram_base;
1729 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1733 * checks if the csrow passed in is marked as SPARED, if so returns the new
1736 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1740 if (online_spare_swap_done(pvt, dct) &&
1741 csrow == online_spare_bad_dramcs(pvt, dct)) {
1743 for_each_chip_select(tmp_cs, dct, pvt) {
1744 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1754 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1755 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1758 * -EINVAL: NOT FOUND
1759 * 0..csrow = Chip-Select Row
1761 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1763 struct mem_ctl_info *mci;
1764 struct amd64_pvt *pvt;
1765 u64 cs_base, cs_mask;
1766 int cs_found = -EINVAL;
1769 mci = edac_mc_find(nid);
1773 pvt = mci->pvt_info;
1775 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1777 for_each_chip_select(csrow, dct, pvt) {
1778 if (!csrow_enabled(csrow, dct, pvt))
1781 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1783 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1784 csrow, cs_base, cs_mask);
1788 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1789 (in_addr & cs_mask), (cs_base & cs_mask));
1791 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1792 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1796 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1798 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1806 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1807 * swapped with a region located at the bottom of memory so that the GPU can use
1808 * the interleaved region and thus two channels.
1810 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1812 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1814 if (pvt->fam == 0x10) {
1815 /* only revC3 and revE have that feature */
1816 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1820 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1822 if (!(swap_reg & 0x1))
1825 swap_base = (swap_reg >> 3) & 0x7f;
1826 swap_limit = (swap_reg >> 11) & 0x7f;
1827 rgn_size = (swap_reg >> 20) & 0x7f;
1828 tmp_addr = sys_addr >> 27;
1830 if (!(sys_addr >> 34) &&
1831 (((tmp_addr >= swap_base) &&
1832 (tmp_addr <= swap_limit)) ||
1833 (tmp_addr < rgn_size)))
1834 return sys_addr ^ (u64)swap_base << 27;
1839 /* For a given @dram_range, check if @sys_addr falls within it. */
1840 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1841 u64 sys_addr, int *chan_sel)
1843 int cs_found = -EINVAL;
1847 bool high_range = false;
1849 u8 node_id = dram_dst_node(pvt, range);
1850 u8 intlv_en = dram_intlv_en(pvt, range);
1851 u32 intlv_sel = dram_intlv_sel(pvt, range);
1853 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1854 range, sys_addr, get_dram_limit(pvt, range));
1856 if (dhar_valid(pvt) &&
1857 dhar_base(pvt) <= sys_addr &&
1858 sys_addr < BIT_64(32)) {
1859 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1864 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1867 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1869 dct_sel_base = dct_sel_baseaddr(pvt);
1872 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1873 * select between DCT0 and DCT1.
1875 if (dct_high_range_enabled(pvt) &&
1876 !dct_ganging_enabled(pvt) &&
1877 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1880 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1882 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1883 high_range, dct_sel_base);
1885 /* Remove node interleaving, see F1x120 */
1887 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1888 (chan_addr & 0xfff);
1890 /* remove channel interleave */
1891 if (dct_interleave_enabled(pvt) &&
1892 !dct_high_range_enabled(pvt) &&
1893 !dct_ganging_enabled(pvt)) {
1895 if (dct_sel_interleave_addr(pvt) != 1) {
1896 if (dct_sel_interleave_addr(pvt) == 0x3)
1898 chan_addr = ((chan_addr >> 10) << 9) |
1899 (chan_addr & 0x1ff);
1901 /* A[6] or hash 6 */
1902 chan_addr = ((chan_addr >> 7) << 6) |
1906 chan_addr = ((chan_addr >> 13) << 12) |
1907 (chan_addr & 0xfff);
1910 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1912 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1915 *chan_sel = channel;
1920 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1921 u64 sys_addr, int *chan_sel)
1923 int cs_found = -EINVAL;
1924 int num_dcts_intlv = 0;
1925 u64 chan_addr, chan_offset;
1926 u64 dct_base, dct_limit;
1927 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1928 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1930 u64 dhar_offset = f10_dhar_offset(pvt);
1931 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1932 u8 node_id = dram_dst_node(pvt, range);
1933 u8 intlv_en = dram_intlv_en(pvt, range);
1935 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1936 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1938 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1939 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1941 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1942 range, sys_addr, get_dram_limit(pvt, range));
1944 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1945 !(get_dram_limit(pvt, range) >= sys_addr))
1948 if (dhar_valid(pvt) &&
1949 dhar_base(pvt) <= sys_addr &&
1950 sys_addr < BIT_64(32)) {
1951 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1956 /* Verify sys_addr is within DCT Range. */
1957 dct_base = (u64) dct_sel_baseaddr(pvt);
1958 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1960 if (!(dct_cont_base_reg & BIT(0)) &&
1961 !(dct_base <= (sys_addr >> 27) &&
1962 dct_limit >= (sys_addr >> 27)))
1965 /* Verify number of dct's that participate in channel interleaving. */
1966 num_dcts_intlv = (int) hweight8(intlv_en);
1968 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1971 if (pvt->model >= 0x60)
1972 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
1974 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1975 num_dcts_intlv, dct_sel);
1977 /* Verify we stay within the MAX number of channels allowed */
1981 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1983 /* Get normalized DCT addr */
1984 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1985 chan_offset = dhar_offset;
1987 chan_offset = dct_base << 27;
1989 chan_addr = sys_addr - chan_offset;
1991 /* remove channel interleave */
1992 if (num_dcts_intlv == 2) {
1993 if (intlv_addr == 0x4)
1994 chan_addr = ((chan_addr >> 9) << 8) |
1996 else if (intlv_addr == 0x5)
1997 chan_addr = ((chan_addr >> 10) << 9) |
1998 (chan_addr & 0x1ff);
2002 } else if (num_dcts_intlv == 4) {
2003 if (intlv_addr == 0x4)
2004 chan_addr = ((chan_addr >> 10) << 8) |
2006 else if (intlv_addr == 0x5)
2007 chan_addr = ((chan_addr >> 11) << 9) |
2008 (chan_addr & 0x1ff);
2013 if (dct_offset_en) {
2014 amd64_read_pci_cfg(pvt->F1,
2015 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2017 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2020 f15h_select_dct(pvt, channel);
2022 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2026 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2027 * there is support for 4 DCT's, but only 2 are currently functional.
2028 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2029 * pvt->csels[1]. So we need to use '1' here to get correct info.
2030 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2032 alias_channel = (channel == 3) ? 1 : channel;
2034 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2037 *chan_sel = alias_channel;
2042 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2046 int cs_found = -EINVAL;
2049 for (range = 0; range < DRAM_RANGES; range++) {
2050 if (!dram_rw(pvt, range))
2053 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2054 cs_found = f15_m30h_match_to_this_node(pvt, range,
2058 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2059 (get_dram_limit(pvt, range) >= sys_addr)) {
2060 cs_found = f1x_match_to_this_node(pvt, range,
2061 sys_addr, chan_sel);
2070 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2071 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2073 * The @sys_addr is usually an error address received from the hardware
2076 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2077 struct err_info *err)
2079 struct amd64_pvt *pvt = mci->pvt_info;
2081 error_address_to_page_and_offset(sys_addr, err);
2083 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2084 if (err->csrow < 0) {
2085 err->err_code = ERR_CSROW;
2090 * We need the syndromes for channel detection only when we're
2091 * ganged. Otherwise @chan should already contain the channel at
2094 if (dct_ganging_enabled(pvt))
2095 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2099 * debug routine to display the memory sizes of all logical DIMMs and its
2102 static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2104 int dimm, size0, size1;
2105 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2106 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2108 if (pvt->fam == 0xf) {
2109 /* K8 families < revF not supported yet */
2110 if (pvt->ext_model < K8_REV_F)
2116 if (pvt->fam == 0x10) {
2117 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2119 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2120 pvt->csels[1].csbases :
2121 pvt->csels[0].csbases;
2124 dcsb = pvt->csels[1].csbases;
2126 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2129 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2131 /* Dump memory sizes for DIMM and its CSROWs */
2132 for (dimm = 0; dimm < 4; dimm++) {
2135 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2137 * For F15m60h, we need multiplier for LRDIMM cs_size
2138 * calculation. We pass dimm value to the dbam_to_cs
2139 * mapper so we can find the multiplier from the
2140 * corresponding DCSM.
2142 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2143 DBAM_DIMM(dimm, dbam),
2147 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2148 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2149 DBAM_DIMM(dimm, dbam),
2152 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2154 dimm * 2 + 1, size1);
2158 static struct amd64_family_type family_types[] = {
2161 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2162 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2164 .early_channel_count = k8_early_channel_count,
2165 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2166 .dbam_to_cs = k8_dbam_to_chip_select,
2171 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2172 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2174 .early_channel_count = f1x_early_channel_count,
2175 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2176 .dbam_to_cs = f10_dbam_to_chip_select,
2181 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2182 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2184 .early_channel_count = f1x_early_channel_count,
2185 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2186 .dbam_to_cs = f15_dbam_to_chip_select,
2190 .ctl_name = "F15h_M30h",
2191 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2192 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2194 .early_channel_count = f1x_early_channel_count,
2195 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2196 .dbam_to_cs = f16_dbam_to_chip_select,
2200 .ctl_name = "F15h_M60h",
2201 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2202 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2204 .early_channel_count = f1x_early_channel_count,
2205 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2206 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2211 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2212 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2214 .early_channel_count = f1x_early_channel_count,
2215 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2216 .dbam_to_cs = f16_dbam_to_chip_select,
2220 .ctl_name = "F16h_M30h",
2221 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2222 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2224 .early_channel_count = f1x_early_channel_count,
2225 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2226 .dbam_to_cs = f16_dbam_to_chip_select,
2231 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2232 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2234 .early_channel_count = f17_early_channel_count,
2235 .dbam_to_cs = f17_base_addr_to_cs_size,
2239 .ctl_name = "F17h_M10h",
2240 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2241 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2243 .early_channel_count = f17_early_channel_count,
2244 .dbam_to_cs = f17_base_addr_to_cs_size,
2248 .ctl_name = "F17h_M30h",
2249 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2250 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2252 .early_channel_count = f17_early_channel_count,
2253 .dbam_to_cs = f17_base_addr_to_cs_size,
2259 * These are tables of eigenvectors (one per line) which can be used for the
2260 * construction of the syndrome tables. The modified syndrome search algorithm
2261 * uses those to find the symbol in error and thus the DIMM.
2263 * Algorithm courtesy of Ross LaFetra from AMD.
2265 static const u16 x4_vectors[] = {
2266 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2267 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2268 0x0001, 0x0002, 0x0004, 0x0008,
2269 0x1013, 0x3032, 0x4044, 0x8088,
2270 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2271 0x4857, 0xc4fe, 0x13cc, 0x3288,
2272 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2273 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2274 0x15c1, 0x2a42, 0x89ac, 0x4758,
2275 0x2b03, 0x1602, 0x4f0c, 0xca08,
2276 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2277 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2278 0x2b87, 0x164e, 0x642c, 0xdc18,
2279 0x40b9, 0x80de, 0x1094, 0x20e8,
2280 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2281 0x11c1, 0x2242, 0x84ac, 0x4c58,
2282 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2283 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2284 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2285 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2286 0x16b3, 0x3d62, 0x4f34, 0x8518,
2287 0x1e2f, 0x391a, 0x5cac, 0xf858,
2288 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2289 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2290 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2291 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2292 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2293 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2294 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2295 0x185d, 0x2ca6, 0x7914, 0x9e28,
2296 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2297 0x4199, 0x82ee, 0x19f4, 0x2e58,
2298 0x4807, 0xc40e, 0x130c, 0x3208,
2299 0x1905, 0x2e0a, 0x5804, 0xac08,
2300 0x213f, 0x132a, 0xadfc, 0x5ba8,
2301 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2304 static const u16 x8_vectors[] = {
2305 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2306 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2307 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2308 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2309 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2310 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2311 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2312 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2313 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2314 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2315 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2316 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2317 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2318 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2319 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2320 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2321 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2322 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2323 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2326 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2329 unsigned int i, err_sym;
2331 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2333 unsigned v_idx = err_sym * v_dim;
2334 unsigned v_end = (err_sym + 1) * v_dim;
2336 /* walk over all 16 bits of the syndrome */
2337 for (i = 1; i < (1U << 16); i <<= 1) {
2339 /* if bit is set in that eigenvector... */
2340 if (v_idx < v_end && vectors[v_idx] & i) {
2341 u16 ev_comp = vectors[v_idx++];
2343 /* ... and bit set in the modified syndrome, */
2353 /* can't get to zero, move to next symbol */
2358 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2362 static int map_err_sym_to_channel(int err_sym, int sym_size)
2375 return err_sym >> 4;
2381 /* imaginary bits not in a DIMM */
2383 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2395 return err_sym >> 3;
2401 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2403 struct amd64_pvt *pvt = mci->pvt_info;
2406 if (pvt->ecc_sym_sz == 8)
2407 err_sym = decode_syndrome(syndrome, x8_vectors,
2408 ARRAY_SIZE(x8_vectors),
2410 else if (pvt->ecc_sym_sz == 4)
2411 err_sym = decode_syndrome(syndrome, x4_vectors,
2412 ARRAY_SIZE(x4_vectors),
2415 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2419 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2422 static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2425 enum hw_event_mc_err_type err_type;
2429 err_type = HW_EVENT_ERR_CORRECTED;
2430 else if (ecc_type == 1)
2431 err_type = HW_EVENT_ERR_UNCORRECTED;
2432 else if (ecc_type == 3)
2433 err_type = HW_EVENT_ERR_DEFERRED;
2435 WARN(1, "Something is rotten in the state of Denmark.\n");
2439 switch (err->err_code) {
2444 string = "Failed to map error addr to a node";
2447 string = "Failed to map error addr to a csrow";
2450 string = "Unknown syndrome - possible error reporting race";
2453 string = "MCA_SYND not valid - unknown syndrome and csrow";
2456 string = "Cannot decode normalized address";
2459 string = "WTF error";
2463 edac_mc_handle_error(err_type, mci, 1,
2464 err->page, err->offset, err->syndrome,
2465 err->csrow, err->channel, -1,
2469 static inline void decode_bus_error(int node_id, struct mce *m)
2471 struct mem_ctl_info *mci;
2472 struct amd64_pvt *pvt;
2473 u8 ecc_type = (m->status >> 45) & 0x3;
2474 u8 xec = XEC(m->status, 0x1f);
2475 u16 ec = EC(m->status);
2477 struct err_info err;
2479 mci = edac_mc_find(node_id);
2483 pvt = mci->pvt_info;
2485 /* Bail out early if this was an 'observed' error */
2486 if (PP(ec) == NBSL_PP_OBS)
2489 /* Do only ECC errors */
2490 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2493 memset(&err, 0, sizeof(err));
2495 sys_addr = get_error_address(pvt, m);
2498 err.syndrome = extract_syndrome(m->status);
2500 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2502 __log_ecc_error(mci, &err, ecc_type);
2506 * To find the UMC channel represented by this bank we need to match on its
2507 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2510 * Currently, we can derive the channel number by looking at the 6th nibble in
2511 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
2514 static int find_umc_channel(struct mce *m)
2516 return (m->ipid & GENMASK(31, 0)) >> 20;
2519 static void decode_umc_error(int node_id, struct mce *m)
2521 u8 ecc_type = (m->status >> 45) & 0x3;
2522 struct mem_ctl_info *mci;
2523 struct amd64_pvt *pvt;
2524 struct err_info err;
2527 mci = edac_mc_find(node_id);
2531 pvt = mci->pvt_info;
2533 memset(&err, 0, sizeof(err));
2535 if (m->status & MCI_STATUS_DEFERRED)
2538 err.channel = find_umc_channel(m);
2540 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2541 err.err_code = ERR_NORM_ADDR;
2545 error_address_to_page_and_offset(sys_addr, &err);
2547 if (!(m->status & MCI_STATUS_SYNDV)) {
2548 err.err_code = ERR_SYND;
2552 if (ecc_type == 2) {
2553 u8 length = (m->synd >> 18) & 0x3f;
2556 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2558 err.err_code = ERR_CHANNEL;
2561 err.csrow = m->synd & 0x7;
2564 __log_ecc_error(mci, &err, ecc_type);
2568 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2569 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2570 * Reserve F0 and F6 on systems with a UMC.
2573 reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2576 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2578 amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1);
2582 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2584 pci_dev_put(pvt->F0);
2587 amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2591 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2592 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2593 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2598 /* Reserve the ADDRESS MAP Device */
2599 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2601 amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1);
2605 /* Reserve the DCT Device */
2606 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2608 pci_dev_put(pvt->F1);
2611 amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2615 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2616 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2617 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2622 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2625 pci_dev_put(pvt->F0);
2626 pci_dev_put(pvt->F6);
2628 pci_dev_put(pvt->F1);
2629 pci_dev_put(pvt->F2);
2633 static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
2635 pvt->ecc_sym_sz = 4;
2641 /* Check enabled channels only: */
2642 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
2643 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
2644 pvt->ecc_sym_sz = 16;
2646 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
2647 pvt->ecc_sym_sz = 8;
2652 } else if (pvt->fam >= 0x10) {
2655 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2656 /* F16h has only DCT0, so no need to read dbam1. */
2657 if (pvt->fam != 0x16)
2658 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2660 /* F10h, revD and later can do x8 ECC too. */
2661 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2662 pvt->ecc_sym_sz = 8;
2667 * Retrieve the hardware registers of the memory controller.
2669 static void __read_mc_regs_df(struct amd64_pvt *pvt)
2671 u8 nid = pvt->mc_node_id;
2672 struct amd64_umc *umc;
2675 /* Read registers from each UMC */
2678 umc_base = get_umc_base(i);
2681 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
2682 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
2683 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
2684 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
2685 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
2690 * Retrieve the hardware registers of the memory controller (this includes the
2691 * 'Address Map' and 'Misc' device regs)
2693 static void read_mc_regs(struct amd64_pvt *pvt)
2699 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2700 * those are Read-As-Zero.
2702 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2703 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2705 /* Check first whether TOP_MEM2 is enabled: */
2706 rdmsrl(MSR_K8_SYSCFG, msr_val);
2707 if (msr_val & BIT(21)) {
2708 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2709 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2711 edac_dbg(0, " TOP_MEM2 disabled\n");
2715 __read_mc_regs_df(pvt);
2716 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
2721 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2723 read_dram_ctl_register(pvt);
2725 for (range = 0; range < DRAM_RANGES; range++) {
2728 /* read settings for this DRAM range */
2729 read_dram_base_limit_regs(pvt, range);
2731 rw = dram_rw(pvt, range);
2735 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2737 get_dram_base(pvt, range),
2738 get_dram_limit(pvt, range));
2740 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2741 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2742 (rw & 0x1) ? "R" : "-",
2743 (rw & 0x2) ? "W" : "-",
2744 dram_intlv_sel(pvt, range),
2745 dram_dst_node(pvt, range));
2748 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2749 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2751 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2753 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2754 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2756 if (!dct_ganging_enabled(pvt)) {
2757 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2758 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2762 read_dct_base_mask(pvt);
2764 determine_memory_type(pvt);
2765 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2767 determine_ecc_sym_sz(pvt);
2769 dump_misc_regs(pvt);
2773 * NOTE: CPU Revision Dependent code
2776 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2777 * k8 private pointer to -->
2778 * DRAM Bank Address mapping register
2780 * DCL register where dual_channel_active is
2782 * The DBAM register consists of 4 sets of 4 bits each definitions:
2785 * 0-3 CSROWs 0 and 1
2786 * 4-7 CSROWs 2 and 3
2787 * 8-11 CSROWs 4 and 5
2788 * 12-15 CSROWs 6 and 7
2790 * Values range from: 0 to 15
2791 * The meaning of the values depends on CPU revision and dual-channel state,
2792 * see relevant BKDG more info.
2794 * The memory controller provides for total of only 8 CSROWs in its current
2795 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2796 * single channel or two (2) DIMMs in dual channel mode.
2798 * The following code logic collapses the various tables for CSROW based on CPU
2802 * The number of PAGE_SIZE pages on the specified CSROW number it
2806 static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
2808 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2809 int csrow_nr = csrow_nr_orig;
2810 u32 cs_mode, nr_pages;
2815 cs_mode = DBAM_DIMM(csrow_nr, dbam);
2817 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
2818 nr_pages <<= 20 - PAGE_SHIFT;
2820 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2821 csrow_nr_orig, dct, cs_mode);
2822 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2828 * Initialize the array of csrow attribute instances, based on the values
2829 * from pci config hardware registers.
2831 static int init_csrows(struct mem_ctl_info *mci)
2833 struct amd64_pvt *pvt = mci->pvt_info;
2834 enum edac_type edac_mode = EDAC_NONE;
2835 struct csrow_info *csrow;
2836 struct dimm_info *dimm;
2837 int i, j, empty = 1;
2842 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2846 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2847 pvt->mc_node_id, val,
2848 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2852 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2854 for_each_chip_select(i, 0, pvt) {
2855 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2856 bool row_dct1 = false;
2858 if (pvt->fam != 0xf)
2859 row_dct1 = !!csrow_enabled(i, 1, pvt);
2861 if (!row_dct0 && !row_dct1)
2864 csrow = mci->csrows[i];
2867 edac_dbg(1, "MC node: %d, csrow: %d\n",
2868 pvt->mc_node_id, i);
2871 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2872 csrow->channels[0]->dimm->nr_pages = nr_pages;
2875 /* K8 has only one DCT */
2876 if (pvt->fam != 0xf && row_dct1) {
2877 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2879 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2880 nr_pages += row_dct1_pages;
2883 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2885 /* Determine DIMM ECC mode: */
2887 if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
2888 edac_mode = EDAC_S4ECD4ED;
2889 else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
2890 edac_mode = EDAC_SECDED;
2892 } else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
2893 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
2898 for (j = 0; j < pvt->channel_count; j++) {
2899 dimm = csrow->channels[j]->dimm;
2900 dimm->mtype = pvt->dram_type;
2901 dimm->edac_mode = edac_mode;
2908 /* get all cores on this DCT */
2909 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2913 for_each_online_cpu(cpu)
2914 if (amd_get_nb_id(cpu) == nid)
2915 cpumask_set_cpu(cpu, mask);
2918 /* check MCG_CTL on all the cpus on this node */
2919 static bool nb_mce_bank_enabled_on_node(u16 nid)
2925 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2926 amd64_warn("%s: Error allocating mask\n", __func__);
2930 get_cpus_on_this_dct_cpumask(mask, nid);
2932 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2934 for_each_cpu(cpu, mask) {
2935 struct msr *reg = per_cpu_ptr(msrs, cpu);
2936 nbe = reg->l & MSR_MCGCTL_NBE;
2938 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2940 (nbe ? "enabled" : "disabled"));
2948 free_cpumask_var(mask);
2952 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2954 cpumask_var_t cmask;
2957 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2958 amd64_warn("%s: error allocating mask\n", __func__);
2962 get_cpus_on_this_dct_cpumask(cmask, nid);
2964 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2966 for_each_cpu(cpu, cmask) {
2968 struct msr *reg = per_cpu_ptr(msrs, cpu);
2971 if (reg->l & MSR_MCGCTL_NBE)
2972 s->flags.nb_mce_enable = 1;
2974 reg->l |= MSR_MCGCTL_NBE;
2977 * Turn off NB MCE reporting only when it was off before
2979 if (!s->flags.nb_mce_enable)
2980 reg->l &= ~MSR_MCGCTL_NBE;
2983 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2985 free_cpumask_var(cmask);
2990 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2994 u32 value, mask = 0x3; /* UECC/CECC enable */
2996 if (toggle_ecc_err_reporting(s, nid, ON)) {
2997 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3001 amd64_read_pci_cfg(F3, NBCTL, &value);
3003 s->old_nbctl = value & mask;
3004 s->nbctl_valid = true;
3007 amd64_write_pci_cfg(F3, NBCTL, value);
3009 amd64_read_pci_cfg(F3, NBCFG, &value);
3011 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3012 nid, value, !!(value & NBCFG_ECC_ENABLE));
3014 if (!(value & NBCFG_ECC_ENABLE)) {
3015 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3017 s->flags.nb_ecc_prev = 0;
3019 /* Attempt to turn on DRAM ECC Enable */
3020 value |= NBCFG_ECC_ENABLE;
3021 amd64_write_pci_cfg(F3, NBCFG, value);
3023 amd64_read_pci_cfg(F3, NBCFG, &value);
3025 if (!(value & NBCFG_ECC_ENABLE)) {
3026 amd64_warn("Hardware rejected DRAM ECC enable,"
3027 "check memory DIMM configuration.\n");
3030 amd64_info("Hardware accepted DRAM ECC Enable\n");
3033 s->flags.nb_ecc_prev = 1;
3036 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3037 nid, value, !!(value & NBCFG_ECC_ENABLE));
3042 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3045 u32 value, mask = 0x3; /* UECC/CECC enable */
3047 if (!s->nbctl_valid)
3050 amd64_read_pci_cfg(F3, NBCTL, &value);
3052 value |= s->old_nbctl;
3054 amd64_write_pci_cfg(F3, NBCTL, value);
3056 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3057 if (!s->flags.nb_ecc_prev) {
3058 amd64_read_pci_cfg(F3, NBCFG, &value);
3059 value &= ~NBCFG_ECC_ENABLE;
3060 amd64_write_pci_cfg(F3, NBCFG, value);
3063 /* restore the NB Enable MCGCTL bit */
3064 if (toggle_ecc_err_reporting(s, nid, OFF))
3065 amd64_warn("Error restoring NB MCGCTL settings!\n");
3069 * EDAC requires that the BIOS have ECC enabled before
3070 * taking over the processing of ECC errors. A command line
3071 * option allows to force-enable hardware ECC later in
3072 * enable_ecc_error_reporting().
3074 static const char *ecc_msg =
3075 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
3076 " Either enable ECC checking or force module loading by setting "
3077 "'ecc_enable_override'.\n"
3078 " (Note that use of the override may cause unknown side effects.)\n";
3080 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
3082 bool nb_mce_en = false;
3086 if (boot_cpu_data.x86 >= 0x17) {
3087 u8 umc_en_mask = 0, ecc_en_mask = 0;
3090 u32 base = get_umc_base(i);
3092 /* Only check enabled UMCs. */
3093 if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
3096 if (!(value & UMC_SDP_INIT))
3099 umc_en_mask |= BIT(i);
3101 if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
3104 if (value & UMC_ECC_ENABLED)
3105 ecc_en_mask |= BIT(i);
3108 /* Check whether at least one UMC is enabled: */
3110 ecc_en = umc_en_mask == ecc_en_mask;
3112 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3114 /* Assume UMC MCA banks are enabled. */
3117 amd64_read_pci_cfg(F3, NBCFG, &value);
3119 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3121 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3123 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3124 MSR_IA32_MCG_CTL, nid);
3127 amd64_info("Node %d: DRAM ECC %s.\n",
3128 nid, (ecc_en ? "enabled" : "disabled"));
3130 if (!ecc_en || !nb_mce_en) {
3131 amd64_info("%s", ecc_msg);
3138 f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3140 u8 i, ecc_en = 1, cpk_en = 1;
3143 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3144 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3145 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3149 /* Set chipkill only if ECC is enabled: */
3151 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3154 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3158 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
3159 struct amd64_family_type *fam)
3161 struct amd64_pvt *pvt = mci->pvt_info;
3163 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3164 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3167 f17h_determine_edac_ctl_cap(mci, pvt);
3169 if (pvt->nbcap & NBCAP_SECDED)
3170 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3172 if (pvt->nbcap & NBCAP_CHIPKILL)
3173 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3176 mci->edac_cap = determine_edac_cap(pvt);
3177 mci->mod_name = EDAC_MOD_STR;
3178 mci->ctl_name = fam->ctl_name;
3179 mci->dev_name = pci_name(pvt->F3);
3180 mci->ctl_page_to_phys = NULL;
3182 /* memory scrubber interface */
3183 mci->set_sdram_scrub_rate = set_scrub_rate;
3184 mci->get_sdram_scrub_rate = get_scrub_rate;
3188 * returns a pointer to the family descriptor on success, NULL otherwise.
3190 static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3192 struct amd64_family_type *fam_type = NULL;
3194 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3195 pvt->stepping = boot_cpu_data.x86_stepping;
3196 pvt->model = boot_cpu_data.x86_model;
3197 pvt->fam = boot_cpu_data.x86;
3201 fam_type = &family_types[K8_CPUS];
3202 pvt->ops = &family_types[K8_CPUS].ops;
3206 fam_type = &family_types[F10_CPUS];
3207 pvt->ops = &family_types[F10_CPUS].ops;
3211 if (pvt->model == 0x30) {
3212 fam_type = &family_types[F15_M30H_CPUS];
3213 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3215 } else if (pvt->model == 0x60) {
3216 fam_type = &family_types[F15_M60H_CPUS];
3217 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3221 fam_type = &family_types[F15_CPUS];
3222 pvt->ops = &family_types[F15_CPUS].ops;
3226 if (pvt->model == 0x30) {
3227 fam_type = &family_types[F16_M30H_CPUS];
3228 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3231 fam_type = &family_types[F16_CPUS];
3232 pvt->ops = &family_types[F16_CPUS].ops;
3236 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3237 fam_type = &family_types[F17_M10H_CPUS];
3238 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3240 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3241 fam_type = &family_types[F17_M30H_CPUS];
3242 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3247 fam_type = &family_types[F17_CPUS];
3248 pvt->ops = &family_types[F17_CPUS].ops;
3250 if (pvt->fam == 0x18)
3251 family_types[F17_CPUS].ctl_name = "F18h";
3255 amd64_err("Unsupported family!\n");
3259 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3261 (pvt->ext_model >= K8_REV_F ? "revF or later "
3262 : "revE or earlier ")
3263 : ""), pvt->mc_node_id);
3267 static const struct attribute_group *amd64_edac_attr_groups[] = {
3268 #ifdef CONFIG_EDAC_DEBUG
3269 &amd64_edac_dbg_group,
3271 #ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
3272 &amd64_edac_inj_group,
3277 /* Set the number of Unified Memory Controllers in the system. */
3278 static void compute_num_umcs(void)
3280 u8 model = boot_cpu_data.x86_model;
3282 if (boot_cpu_data.x86 < 0x17)
3285 if (model >= 0x30 && model <= 0x3f)
3290 edac_dbg(1, "Number of UMCs: %x", num_umcs);
3293 static int init_one_instance(unsigned int nid)
3295 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3296 struct amd64_family_type *fam_type = NULL;
3297 struct mem_ctl_info *mci = NULL;
3298 struct edac_mc_layer layers[2];
3299 struct amd64_pvt *pvt = NULL;
3300 u16 pci_id1, pci_id2;
3304 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3308 pvt->mc_node_id = nid;
3312 fam_type = per_family_init(pvt);
3316 if (pvt->fam >= 0x17) {
3317 pvt->umc = kcalloc(num_umcs, sizeof(struct amd64_umc), GFP_KERNEL);
3323 pci_id1 = fam_type->f0_id;
3324 pci_id2 = fam_type->f6_id;
3326 pci_id1 = fam_type->f1_id;
3327 pci_id2 = fam_type->f2_id;
3330 err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3337 * We need to determine how many memory channels there are. Then use
3338 * that information for calculating the size of the dynamic instance
3339 * tables in the 'mci' structure.
3342 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3343 if (pvt->channel_count < 0)
3347 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3348 layers[0].size = pvt->csels[0].b_cnt;
3349 layers[0].is_virt_csrow = true;
3350 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3353 * Always allocate two channels since we can have setups with DIMMs on
3354 * only one channel. Also, this simplifies handling later for the price
3355 * of a couple of KBs tops.
3357 * On Fam17h+, the number of controllers may be greater than two. So set
3358 * the size equal to the maximum number of UMCs.
3360 if (pvt->fam >= 0x17)
3361 layers[1].size = num_umcs;
3364 layers[1].is_virt_csrow = false;
3366 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
3370 mci->pvt_info = pvt;
3371 mci->pdev = &pvt->F3->dev;
3373 setup_mci_misc_attrs(mci, fam_type);
3375 if (init_csrows(mci))
3376 mci->edac_cap = EDAC_FLAG_NONE;
3379 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3380 edac_dbg(1, "failed edac_mc_add_mc()\n");
3390 free_mc_sibling_devs(pvt);
3393 if (pvt->fam >= 0x17)
3403 static int probe_one_instance(unsigned int nid)
3405 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3406 struct ecc_settings *s;
3410 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3416 if (!ecc_enabled(F3, nid)) {
3419 if (!ecc_enable_override)
3422 if (boot_cpu_data.x86 >= 0x17) {
3423 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3426 amd64_warn("Forcing ECC on!\n");
3428 if (!enable_ecc_error_reporting(s, nid, F3))
3432 ret = init_one_instance(nid);
3434 amd64_err("Error probing instance: %d\n", nid);
3436 if (boot_cpu_data.x86 < 0x17)
3437 restore_ecc_error_reporting(s, nid, F3);
3446 ecc_stngs[nid] = NULL;
3452 static void remove_one_instance(unsigned int nid)
3454 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3455 struct ecc_settings *s = ecc_stngs[nid];
3456 struct mem_ctl_info *mci;
3457 struct amd64_pvt *pvt;
3459 mci = find_mci_by_dev(&F3->dev);
3462 /* Remove from EDAC CORE tracking list */
3463 mci = edac_mc_del_mc(&F3->dev);
3467 pvt = mci->pvt_info;
3469 restore_ecc_error_reporting(s, nid, F3);
3471 free_mc_sibling_devs(pvt);
3473 kfree(ecc_stngs[nid]);
3474 ecc_stngs[nid] = NULL;
3476 /* Free the EDAC CORE resources */
3477 mci->pvt_info = NULL;
3483 static void setup_pci_device(void)
3485 struct mem_ctl_info *mci;
3486 struct amd64_pvt *pvt;
3491 mci = edac_mc_find(0);
3495 pvt = mci->pvt_info;
3497 pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
3499 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
3501 pr_warn("%s(): Unable to create PCI control\n", __func__);
3502 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3506 static const struct x86_cpu_id amd64_cpuids[] = {
3507 { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3508 { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3509 { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3510 { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3511 { X86_VENDOR_AMD, 0x17, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3512 { X86_VENDOR_HYGON, 0x18, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3515 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3517 static int __init amd64_edac_init(void)
3523 owner = edac_get_owner();
3524 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3527 if (!x86_match_cpu(amd64_cpuids))
3530 if (amd_cache_northbridges() < 0)
3536 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
3540 msrs = msrs_alloc();
3546 for (i = 0; i < amd_nb_num(); i++) {
3547 err = probe_one_instance(i);
3549 /* unwind properly */
3551 remove_one_instance(i);
3557 if (!edac_has_mcs()) {
3562 /* register stuff with EDAC MCE */
3563 if (report_gart_errors)
3564 amd_report_gart_errors(true);
3566 if (boot_cpu_data.x86 >= 0x17)
3567 amd_register_ecc_decoder(decode_umc_error);
3569 amd_register_ecc_decoder(decode_bus_error);
3573 #ifdef CONFIG_X86_32
3574 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3577 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3592 static void __exit amd64_edac_exit(void)
3597 edac_pci_release_generic_ctl(pci_ctl);
3599 /* unregister from EDAC MCE */
3600 amd_report_gart_errors(false);
3602 if (boot_cpu_data.x86 >= 0x17)
3603 amd_unregister_ecc_decoder(decode_umc_error);
3605 amd_unregister_ecc_decoder(decode_bus_error);
3607 for (i = 0; i < amd_nb_num(); i++)
3608 remove_one_instance(i);
3617 module_init(amd64_edac_init);
3618 module_exit(amd64_edac_exit);
3620 MODULE_LICENSE("GPL");
3621 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3622 "Dave Peterson, Thayne Harbaugh");
3623 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3624 EDAC_AMD64_VERSION);
3626 module_param(edac_op_state, int, 0444);
3627 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");