summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MAINTAINERS6
-rw-r--r--drivers/edac/Kconfig9
-rw-r--r--drivers/edac/Makefile1
-rw-r--r--drivers/edac/pnd2_edac.c1542
-rw-r--r--drivers/edac/pnd2_edac.h301
5 files changed, 1859 insertions, 0 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index c265a5fe4848..572fe4252ac4 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -4776,6 +4776,12 @@ L: linux-edac@vger.kernel.org
S: Maintained
F: drivers/edac/mpc85xx_edac.[ch]
+EDAC-PND2
+M: Tony Luck <tony.luck@intel.com>
+L: linux-edac@vger.kernel.org
+S: Maintained
+F: drivers/edac/pnd2_edac.[ch]
+
EDAC-PASEMI
M: Egor Martovetsky <egor@pasemi.com>
L: linux-edac@vger.kernel.org
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 82d85cce81f8..be3eac6ad54d 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -259,6 +259,15 @@ config EDAC_SKX
Support for error detection and correction the Intel
Skylake server Integrated Memory Controllers.
+config EDAC_PND2
+ tristate "Intel Pondicherry2"
+ depends on EDAC_MM_EDAC && PCI && X86_64 && X86_MCE_INTEL
+ help
+ Support for error detection and correction on the Intel
+ Pondicherry2 Integrated Memory Controller. This SoC IP is
+ first used on the Apollo Lake platform and Denverton
+ micro-server but may appear on others in the future.
+
config EDAC_MPC85XX
tristate "Freescale MPC83xx / MPC85xx"
depends on EDAC_MM_EDAC && FSL_SOC
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 88e472e8b9a9..587107e90996 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -32,6 +32,7 @@ obj-$(CONFIG_EDAC_I7300) += i7300_edac.o
obj-$(CONFIG_EDAC_I7CORE) += i7core_edac.o
obj-$(CONFIG_EDAC_SBRIDGE) += sb_edac.o
obj-$(CONFIG_EDAC_SKX) += skx_edac.o
+obj-$(CONFIG_EDAC_PND2) += pnd2_edac.o
obj-$(CONFIG_EDAC_E7XXX) += e7xxx_edac.o
obj-$(CONFIG_EDAC_E752X) += e752x_edac.o
obj-$(CONFIG_EDAC_I82443BXGX) += i82443bxgx_edac.o
diff --git a/drivers/edac/pnd2_edac.c b/drivers/edac/pnd2_edac.c
new file mode 100644
index 000000000000..14d39f05226e
--- /dev/null
+++ b/drivers/edac/pnd2_edac.c
@@ -0,0 +1,1542 @@
+/*
+ * Driver for Pondicherry2 memory controller.
+ *
+ * Copyright (c) 2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * [Derived from sb_edac.c]
+ *
+ * Translation of system physical addresses to DIMM addresses
+ * is a two stage process:
+ *
+ * First the Pondicherry 2 memory controller handles slice and channel interleaving
+ * in "sys2pmi()". This is (almost) completley common between platforms.
+ *
+ * Then a platform specific dunit (DIMM unit) completes the process to provide DIMM,
+ * rank, bank, row and column using the appropriate "dunit_ops" functions/parameters.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/edac.h>
+#include <linux/mmzone.h>
+#include <linux/smp.h>
+#include <linux/bitmap.h>
+#include <linux/math64.h>
+#include <linux/mod_devicetable.h>
+#include <asm/cpu_device_id.h>
+#include <asm/intel-family.h>
+#include <asm/processor.h>
+#include <asm/mce.h>
+
+#include "edac_mc.h"
+#include "edac_module.h"
+#include "pnd2_edac.h"
+
+#define APL_NUM_CHANNELS 4
+#define DNV_NUM_CHANNELS 2
+#define DNV_MAX_DIMMS 2 /* Max DIMMs per channel */
+
+enum type {
+ APL,
+ DNV, /* All requests go to PMI CH0 on each slice (CH1 disabled) */
+};
+
+struct dram_addr {
+ int chan;
+ int dimm;
+ int rank;
+ int bank;
+ int row;
+ int col;
+};
+
+struct pnd2_pvt {
+ int dimm_geom[APL_NUM_CHANNELS];
+ u64 tolm, tohm;
+};
+
+/*
+ * System address space is divided into multiple regions with
+ * different interleave rules in each. The as0/as1 regions
+ * have no interleaving at all. The as2 region is interleaved
+ * between two channels. The mot region is magic and may overlap
+ * other regions, with its interleave rules taking precedence.
+ * Addresses not in any of these regions are interleaved across
+ * all four channels.
+ */
+static struct region {
+ u64 base;
+ u64 limit;
+ u8 enabled;
+} mot, as0, as1, as2;
+
+static struct dunit_ops {
+ char *name;
+ enum type type;
+ int pmiaddr_shift;
+ int pmiidx_shift;
+ int channels;
+ int dimms_per_channel;
+ int (*rd_reg)(int port, int off, int op, void *data, size_t sz, char *name);
+ int (*get_registers)(void);
+ int (*check_ecc)(void);
+ void (*mk_region)(char *name, struct region *rp, void *asym);
+ void (*get_dimm_config)(struct mem_ctl_info *mci);
+ int (*pmi2mem)(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
+ struct dram_addr *daddr, char *msg);
+} *ops;
+
+static struct mem_ctl_info *pnd2_mci;
+
+#define PND2_MSG_SIZE 256
+
+/* Debug macros */
+#define pnd2_printk(level, fmt, arg...) \
+ edac_printk(level, "pnd2", fmt, ##arg)
+
+#define pnd2_mc_printk(mci, level, fmt, arg...) \
+ edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg)
+
+#define MOT_CHAN_INTLV_BIT_1SLC_2CH 12
+#define MOT_CHAN_INTLV_BIT_2SLC_2CH 13
+#define SELECTOR_DISABLED (-1)
+#define _4GB (1ul << 32)
+
+#define PMI_ADDRESS_WIDTH 31
+#define PND_MAX_PHYS_BIT 39
+
+#define APL_ASYMSHIFT 28
+#define DNV_ASYMSHIFT 31
+#define CH_HASH_MASK_LSB 6
+#define SLICE_HASH_MASK_LSB 6
+#define MOT_SLC_INTLV_BIT 12
+#define LOG2_PMI_ADDR_GRANULARITY 5
+#define MOT_SHIFT 24
+
+#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
+#define U64_LSHIFT(val, s) ((u64)(val) << (s))
+
+#ifdef CONFIG_X86_INTEL_SBI_APL
+#include "linux/platform_data/sbi_apl.h"
+int sbi_send(int port, int off, int op, u32 *data)
+{
+ struct sbi_apl_message sbi_arg;
+ int ret, read = 0;
+
+ memset(&sbi_arg, 0, sizeof(sbi_arg));
+
+ if (op == 0 || op == 4 || op == 6)
+ read = 1;
+ else
+ sbi_arg.data = *data;
+
+ sbi_arg.opcode = op;
+ sbi_arg.port_address = port;
+ sbi_arg.register_offset = off;
+ ret = sbi_apl_commit(&sbi_arg);
+ if (ret || sbi_arg.status)
+ edac_dbg(2, "sbi_send status=%d ret=%d data=%x\n",
+ sbi_arg.status, ret, sbi_arg.data);
+
+ if (ret == 0)
+ ret = sbi_arg.status;
+
+ if (ret == 0 && read)
+ *data = sbi_arg.data;
+
+ return ret;
+}
+#else
+int sbi_send(int port, int off, int op, u32 *data)
+{
+ return -EUNATCH;
+}
+#endif
+
+static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
+{
+ int ret = 0;
+
+ edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op);
+ switch (sz) {
+ case 8:
+ ret = sbi_send(port, off + 4, op, (u32 *)(data + 4));
+ case 4:
+ ret = sbi_send(port, off, op, (u32 *)data);
+ pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name,
+ sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret);
+ break;
+ }
+
+ return ret;
+}
+
+static u64 get_mem_ctrl_hub_base_addr(void)
+{
+ struct b_cr_mchbar_lo_pci lo;
+ struct b_cr_mchbar_hi_pci hi;
+ struct pci_dev *pdev;
+
+ pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL);
+ if (pdev) {
+ pci_read_config_dword(pdev, 0x48, (u32 *)&lo);
+ pci_read_config_dword(pdev, 0x4c, (u32 *)&hi);
+ pci_dev_put(pdev);
+ } else {
+ return 0;
+ }
+
+ if (!lo.enable) {
+ edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n");
+ return 0;
+ }
+
+ return U64_LSHIFT(hi.base, 32) | U64_LSHIFT(lo.base, 15);
+}
+
+static u64 get_sideband_reg_base_addr(void)
+{
+ struct pci_dev *pdev;
+ u32 hi, lo;
+
+ pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x19dd, NULL);
+ if (pdev) {
+ pci_read_config_dword(pdev, 0x10, &lo);
+ pci_read_config_dword(pdev, 0x14, &hi);
+ pci_dev_put(pdev);
+ return (U64_LSHIFT(hi, 32) | U64_LSHIFT(lo, 0));
+ } else {
+ return 0xfd000000;
+ }
+}
+
+static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
+{
+ struct pci_dev *pdev;
+ char *base;
+ u64 addr;
+
+ if (op == 4) {
+ pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL);
+ if (!pdev)
+ return -ENODEV;
+
+ pci_read_config_dword(pdev, off, data);
+ pci_dev_put(pdev);
+ } else {
+ /* MMIO via memory controller hub base address */
+ if (op == 0 && port == 0x4c) {
+ addr = get_mem_ctrl_hub_base_addr();
+ if (!addr)
+ return -ENODEV;
+ } else {
+ /* MMIO via sideband register base address */
+ addr = get_sideband_reg_base_addr();
+ if (!addr)
+ return -ENODEV;
+ addr += (port << 16);
+ }
+
+ base = ioremap((resource_size_t)addr, 0x10000);
+ if (!base)
+ return -ENODEV;
+
+ if (sz == 8)
+ *(u32 *)(data + 4) = *(u32 *)(base + off + 4);
+ *(u32 *)data = *(u32 *)(base + off);
+
+ iounmap(base);
+ }
+
+ edac_dbg(2, "Read %s=%.8x_%.8x\n", name,
+ (sz == 8) ? *(u32 *)(data + 4) : 0, *(u32 *)data);
+
+ return 0;
+}
+
+#define RD_REGP(regp, regname, port) \
+ ops->rd_reg(port, \
+ regname##_offset, \
+ regname##_r_opcode, \
+ regp, sizeof(struct regname), \
+ #regname)
+
+#define RD_REG(regp, regname) \
+ ops->rd_reg(regname ## _port, \
+ regname##_offset, \
+ regname##_r_opcode, \
+ regp, sizeof(struct regname), \
+ #regname)
+
+static u64 top_lm, top_hm;
+static bool two_slices;
+static bool two_channels; /* Both PMI channels in one slice enabled */
+
+static u8 sym_chan_mask;
+static u8 asym_chan_mask;
+static u8 chan_mask;
+
+static int slice_selector = -1;
+static int chan_selector = -1;
+static u64 slice_hash_mask;
+static u64 chan_hash_mask;
+
+static void mk_region(char *name, struct region *rp, u64 base, u64 limit)
+{
+ rp->enabled = 1;
+ rp->base = base;
+ rp->limit = limit;
+ edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, limit);
+}
+
+static void mk_region_mask(char *name, struct region *rp, u64 base, u64 mask)
+{
+ if (mask == 0) {
+ pr_info(FW_BUG "MOT mask cannot be zero\n");
+ return;
+ }
+ if (mask != GENMASK_ULL(PND_MAX_PHYS_BIT, __ffs(mask))) {
+ pr_info(FW_BUG "MOT mask not power of two\n");
+ return;
+ }
+ if (base & ~mask) {
+ pr_info(FW_BUG "MOT region base/mask alignment error\n");
+ return;
+ }
+ rp->base = base;
+ rp->limit = (base | ~mask) & GENMASK_ULL(PND_MAX_PHYS_BIT, 0);
+ rp->enabled = 1;
+ edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, rp->limit);
+}
+
+static bool in_region(struct region *rp, u64 addr)
+{
+ if (!rp->enabled)
+ return false;
+
+ return rp->base <= addr && addr <= rp->limit;
+}
+
+static int gen_sym_mask(struct b_cr_slice_channel_hash *p)
+{
+ int mask = 0;
+
+ if (!p->slice_0_mem_disabled)
+ mask |= p->sym_slice0_channel_enabled;
+
+ if (!p->slice_1_disabled)
+ mask |= p->sym_slice1_channel_enabled << 2;
+
+ if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
+ mask &= 0x5;
+
+ return mask;
+}
+
+static int gen_asym_mask(struct b_cr_slice_channel_hash *p,
+ struct b_cr_asym_mem_region0_mchbar *as0,
+ struct b_cr_asym_mem_region1_mchbar *as1,
+ struct b_cr_asym_2way_mem_region_mchbar *as2way)
+{
+ const int intlv[] = { 0x5, 0xA, 0x3, 0xC };
+ int mask = 0;
+
+ if (as2way->asym_2way_interleave_enable)
+ mask = intlv[as2way->asym_2way_intlv_mode];
+ if (as0->slice0_asym_enable)
+ mask |= (1 << as0->slice0_asym_channel_select);
+ if (as1->slice1_asym_enable)
+ mask |= (4 << as1->slice1_asym_channel_select);
+ if (p->slice_0_mem_disabled)
+ mask &= 0xc;
+ if (p->slice_1_disabled)
+ mask &= 0x3;
+ if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
+ mask &= 0x5;
+
+ return mask;
+}
+
+static struct b_cr_tolud_pci tolud;
+static struct b_cr_touud_lo_pci touud_lo;
+static struct b_cr_touud_hi_pci touud_hi;
+static struct b_cr_asym_mem_region0_mchbar asym0;
+static struct b_cr_asym_mem_region1_mchbar asym1;
+static struct b_cr_asym_2way_mem_region_mchbar asym_2way;
+static struct b_cr_mot_out_base_mchbar mot_base;
+static struct b_cr_mot_out_mask_mchbar mot_mask;
+static struct b_cr_slice_channel_hash chash;
+
+/* Apollo Lake dunit */
+/*
+ * Validated on board with just two DIMMs in the [0] and [2] positions
+ * in this array. Other port number matches documentation, but caution
+ * advised.
+ */
+static const int apl_dports[APL_NUM_CHANNELS] = { 0x18, 0x10, 0x11, 0x19 };
+static struct d_cr_drp0 drp0[APL_NUM_CHANNELS];
+
+/* Denverton dunit */
+static const int dnv_dports[DNV_NUM_CHANNELS] = { 0x10, 0x12 };
+static struct d_cr_dsch dsch;
+static struct d_cr_ecc_ctrl ecc_ctrl[DNV_NUM_CHANNELS];
+static struct d_cr_drp drp[DNV_NUM_CHANNELS];
+static struct d_cr_dmap dmap[DNV_NUM_CHANNELS];
+static struct d_cr_dmap1 dmap1[DNV_NUM_CHANNELS];
+static struct d_cr_dmap2 dmap2[DNV_NUM_CHANNELS];
+static struct d_cr_dmap3 dmap3[DNV_NUM_CHANNELS];
+static struct d_cr_dmap4 dmap4[DNV_NUM_CHANNELS];
+static struct d_cr_dmap5 dmap5[DNV_NUM_CHANNELS];
+
+static void apl_mk_region(char *name, struct region *rp, void *asym)
+{
+ struct b_cr_asym_mem_region0_mchbar *a = asym;
+
+ mk_region(name, rp,
+ U64_LSHIFT(a->slice0_asym_base, APL_ASYMSHIFT),
+ U64_LSHIFT(a->slice0_asym_limit, APL_ASYMSHIFT) +
+ GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
+}
+
+static void dnv_mk_region(char *name, struct region *rp, void *asym)
+{
+ struct b_cr_asym_mem_region_denverton *a = asym;
+
+ mk_region(name, rp,
+ U64_LSHIFT(a->slice_asym_base, DNV_ASYMSHIFT),
+ U64_LSHIFT(a->slice_asym_limit, DNV_ASYMSHIFT) +
+ GENMASK_ULL(DNV_ASYMSHIFT - 1, 0));
+}
+
+static int apl_get_registers(void)
+{
+ int i;
+
+ if (RD_REG(&asym_2way, b_cr_asym_2way_mem_region_mchbar))
+ return -ENODEV;
+
+ for (i = 0; i < APL_NUM_CHANNELS; i++)
+ if (RD_REGP(&drp0[i], d_cr_drp0, apl_dports[i]))
+ return -ENODEV;
+
+ return 0;
+}
+
+static int dnv_get_registers(void)
+{
+ int i;
+
+ if (RD_REG(&dsch, d_cr_dsch))
+ return -ENODEV;
+
+ for (i = 0; i < DNV_NUM_CHANNELS; i++)
+ if (RD_REGP(&ecc_ctrl[i], d_cr_ecc_ctrl, dnv_dports[i]) ||
+ RD_REGP(&drp[i], d_cr_drp, dnv_dports[i]) ||
+ RD_REGP(&dmap[i], d_cr_dmap, dnv_dports[i]) ||
+ RD_REGP(&dmap1[i], d_cr_dmap1, dnv_dports[i]) ||
+ RD_REGP(&dmap2[i], d_cr_dmap2, dnv_dports[i]) ||
+ RD_REGP(&dmap3[i], d_cr_dmap3, dnv_dports[i]) ||
+ RD_REGP(&dmap4[i], d_cr_dmap4, dnv_dports[i]) ||
+ RD_REGP(&dmap5[i], d_cr_dmap5, dnv_dports[i]))
+ return -ENODEV;
+
+ return 0;
+}
+
+/*
+ * Read all the h/w config registers once here (they don't
+ * change at run time. Figure out which address ranges have
+ * which interleave characteristics.
+ */
+static int get_registers(void)
+{
+ const int intlv[] = { 10, 11, 12, 12 };
+
+ if (RD_REG(&tolud, b_cr_tolud_pci) ||
+ RD_REG(&touud_lo, b_cr_touud_lo_pci) ||
+ RD_REG(&touud_hi, b_cr_touud_hi_pci) ||
+ RD_REG(&asym0, b_cr_asym_mem_region0_mchbar) ||
+ RD_REG(&asym1, b_cr_asym_mem_region1_mchbar) ||
+ RD_REG(&mot_base, b_cr_mot_out_base_mchbar) ||
+ RD_REG(&mot_mask, b_cr_mot_out_mask_mchbar) ||
+ RD_REG(&chash, b_cr_slice_channel_hash))
+ return -ENODEV;
+
+ if (ops->get_registers())
+ return -ENODEV;
+
+ if (ops->type == DNV) {
+ /* PMI channel idx (always 0) for asymmetric region */
+ asym0.slice0_asym_channel_select = 0;
+ asym1.slice1_asym_channel_select = 0;
+ /* PMI channel bitmap (always 1) for symmetric region */
+ chash.sym_slice0_channel_enabled = 0x1;
+ chash.sym_slice1_channel_enabled = 0x1;
+ }
+
+ if (asym0.slice0_asym_enable)
+ ops->mk_region("as0", &as0, &asym0);
+
+ if (asym1.slice1_asym_enable)
+ ops->mk_region("as1", &as1, &asym1);
+
+ if (asym_2way.asym_2way_interleave_enable) {
+ mk_region("as2way", &as2,
+ U64_LSHIFT(asym_2way.asym_2way_base, APL_ASYMSHIFT),
+ U64_LSHIFT(asym_2way.asym_2way_limit, APL_ASYMSHIFT) +
+ GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
+ }
+
+ if (mot_base.imr_en) {
+ mk_region_mask("mot", &mot,
+ U64_LSHIFT(mot_base.mot_out_base, MOT_SHIFT),
+ U64_LSHIFT(mot_mask.mot_out_mask, MOT_SHIFT));
+ }
+
+ top_lm = U64_LSHIFT(tolud.tolud, 20);
+ top_hm = U64_LSHIFT(touud_hi.touud, 32) | U64_LSHIFT(touud_lo.touud, 20);
+
+ two_slices = !chash.slice_1_disabled &&
+ !chash.slice_0_mem_disabled &&
+ (chash.sym_slice0_channel_enabled != 0) &&
+ (chash.sym_slice1_channel_enabled != 0);
+ two_channels = !chash.ch_1_disabled &&
+ !chash.enable_pmi_dual_data_mode &&
+ ((chash.sym_slice0_channel_enabled == 3) ||
+ (chash.sym_slice1_channel_enabled == 3));
+
+ sym_chan_mask = gen_sym_mask(&chash);
+ asym_chan_mask = gen_asym_mask(&chash, &asym0, &asym1, &asym_2way);
+ chan_mask = sym_chan_mask | asym_chan_mask;
+
+ if (two_slices && !two_channels) {
+ if (chash.hvm_mode)
+ slice_selector = 29;
+ else
+ slice_selector = intlv[chash.interleave_mode];
+ } else if (!two_slices && two_channels) {
+ if (chash.hvm_mode)
+ chan_selector = 29;
+ else
+ chan_selector = intlv[chash.interleave_mode];
+ } else if (two_slices && two_channels) {
+ if (chash.hvm_mode) {
+ slice_selector = 29;
+ chan_selector = 30;
+ } else {
+ slice_selector = intlv[chash.interleave_mode];
+ chan_selector = intlv[chash.interleave_mode] + 1;
+ }
+ }
+
+ if (two_slices) {
+ if (!chash.hvm_mode)
+ slice_hash_mask = chash.slice_hash_mask << SLICE_HASH_MASK_LSB;
+ if (!two_channels)
+ slice_hash_mask |= BIT_ULL(slice_selector);
+ }
+
+ if (two_channels) {
+ if (!chash.hvm_mode)
+ chan_hash_mask = chash.ch_hash_mask << CH_HASH_MASK_LSB;
+ if (!two_slices)
+ chan_hash_mask |= BIT_ULL(chan_selector);
+ }
+
+ return 0;
+}
+
+/* Get a contiguous memory address (remove the MMIO gap) */
+static u64 remove_mmio_gap(u64 sys)
+{
+ return (sys < _4GB) ? sys : sys - (_4GB - top_lm);
+}
+
+/* Squeeze out one address bit, shift upper part down to fill gap */
+static void remove_addr_bit(u64 *addr, int bitidx)
+{
+ u64 mask;
+
+ if (bitidx == -1)
+ return;
+
+ mask = (1ull << bitidx) - 1;
+ *addr = ((*addr >> 1) & ~mask) | (*addr & mask);
+}
+
+/* XOR all the bits from addr specified in mask */
+static int hash_by_mask(u64 addr, u64 mask)
+{
+ u64 result = addr & mask;
+
+ result = (result >> 32) ^ result;
+ result = (result >> 16) ^ result;
+ result = (result >> 8) ^ result;
+ result = (result >> 4) ^ result;
+ result = (result >> 2) ^ result;
+ result = (result >> 1) ^ result;
+
+ return (int)result & 1;
+}
+
+/*
+ * First stage decode. Take the system address and figure out which
+ * second stage will deal with it based on interleave modes.
+ */
+static int sys2pmi(const u64 addr, u32 *pmiidx, u64 *pmiaddr, char *msg)
+{
+ u64 contig_addr, contig_base, contig_offset, contig_base_adj;
+ int mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH :
+ MOT_CHAN_INTLV_BIT_1SLC_2CH;
+ int slice_intlv_bit_rm = SELECTOR_DISABLED;
+ int chan_intlv_bit_rm = SELECTOR_DISABLED;
+ /* Determine if address is in the MOT region. */
+ bool mot_hit = in_region(&mot, addr);
+ /* Calculate the number of symmetric regions enabled. */
+ int sym_channels = hweight8(sym_chan_mask);
+
+ /*
+ * The amount we need to shift the asym base can be determined by the
+ * number of enabled symmetric channels.
+ * NOTE: This can only work because symmetric memory is not supposed
+ * to do a 3-way interleave.
+ */
+ int sym_chan_shift = sym_channels >> 1;
+
+ /* Give up if address is out of range, or in MMIO gap */
+ if (addr >= (1ul << PND_MAX_PHYS_BIT) ||
+ (addr >= top_lm && addr < _4GB) || addr >= top_hm) {
+ snprintf(msg, PND2_MSG_SIZE, "Error address 0x%llx is not DRAM", addr);
+ return -EINVAL;
+ }
+
+ /* Get a contiguous memory address (remove the MMIO gap) */
+ contig_addr = remove_mmio_gap(addr);
+
+ if (in_region(&as0, addr)) {
+ *pmiidx = asym0.slice0_asym_channel_select;
+
+ contig_base = remove_mmio_gap(as0.base);
+ contig_offset = contig_addr - contig_base;
+ contig_base_adj = (contig_base >> sym_chan_shift) *
+ ((chash.sym_slice0_channel_enabled >> (*pmiidx & 1)) & 1);
+ contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull);
+ } else if (in_region(&as1, addr)) {
+ *pmiidx = 2u + asym1.slice1_asym_channel_select;
+
+ contig_base = remove_mmio_gap(as1.base);
+ contig_offset = contig_addr - contig_base;
+ contig_base_adj = (contig_base >> sym_chan_shift) *
+ ((chash.sym_slice1_channel_enabled >> (*pmiidx & 1)) & 1);
+ contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull);
+ } else if (in_region(&as2, addr) && (asym_2way.asym_2way_intlv_mode == 0x3ul)) {
+ bool channel1;
+
+ mot_intlv_bit = MOT_CHAN_INTLV_BIT_1SLC_2CH;
+ *pmiidx = (asym_2way.asym_2way_intlv_mode & 1) << 1;
+ channel1 = mot_hit ? ((bool)((addr >> mot_intlv_bit) & 1)) :
+ hash_by_mask(contig_addr, chan_hash_mask);
+ *pmiidx |= (u32)channel1;
+
+ contig_base = remove_mmio_gap(as2.base);
+ chan_intlv_bit_rm = mot_hit ? mot_intlv_bit : chan_selector;
+ contig_offset = contig_addr - contig_base;
+ remove_addr_bit(&contig_offset, chan_intlv_bit_rm);
+ contig_addr = (contig_base >> sym_chan_shift) + contig_offset;
+ } else {
+ /* Otherwise we're in normal, boring symmetric mode. */
+ *pmiidx = 0u;
+
+ if (two_slices) {
+ bool slice1;
+
+ if (mot_hit) {
+ slice_intlv_bit_rm = MOT_SLC_INTLV_BIT;
+ slice1 = (addr >> MOT_SLC_INTLV_BIT) & 1;
+ } else {
+ slice_intlv_bit_rm = slice_selector;
+ slice1 = hash_by_mask(addr, slice_hash_mask);
+ }
+
+ *pmiidx = (u32)slice1 << 1;
+ }
+
+ if (two_channels) {
+ bool channel1;
+
+ mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH :
+ MOT_CHAN_INTLV_BIT_1SLC_2CH;
+
+ if (mot_hit) {
+ chan_intlv_bit_rm = mot_intlv_bit;
+ channel1 = (addr >> mot_intlv_bit) & 1;
+ } else {
+ chan_intlv_bit_rm = chan_selector;
+ channel1 = hash_by_mask(contig_addr, chan_hash_mask);
+ }
+
+ *pmiidx |= (u32)channel1;
+ }
+ }
+
+ /* Remove the chan_selector bit first */
+ remove_addr_bit(&contig_addr, chan_intlv_bit_rm);
+ /* Remove the slice bit (we remove it second because it must be lower */
+ remove_addr_bit(&contig_addr, slice_intlv_bit_rm);
+ *pmiaddr = contig_addr;
+
+ return 0;
+}
+
+/* Translate PMI address to memory (rank, row, bank, column) */
+#define C(n) (0x10 | (n)) /* column */
+#define B(n) (0x20 | (n)) /* bank */
+#define R(n) (0x40 | (n)) /* row */
+#define RS (0x80) /* rank */
+
+/* addrdec values */
+#define AMAP_1KB 0
+#define AMAP_2KB 1
+#define AMAP_4KB 2
+#define AMAP_RSVD 3
+
+/* dden values */
+#define DEN_4Gb 0
+#define DEN_8Gb 2
+
+/* dwid values */
+#define X8 0
+#define X16 1
+
+static struct dimm_geometry {
+ u8 addrdec;
+ u8 dden;
+ u8 dwid;
+ u8 rowbits, colbits;
+ u16 bits[PMI_ADDRESS_WIDTH];
+} dimms[] = {
+ {
+ .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X16,
+ .rowbits = 15, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
+ R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
+ R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ 0, 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
+ R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
+ R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X16,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
+ R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
+ R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 11,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
+ R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
+ R(10), C(7), C(8), C(9), R(11), RS, C(11), R(12), R(13),
+ R(14), R(15), 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X16,
+ .rowbits = 15, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
+ R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
+ R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ 0, 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
+ R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
+ R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X16,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
+ R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
+ R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 11,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
+ R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
+ R(9), R(10), C(8), C(9), R(11), RS, C(11), R(12), R(13),
+ R(14), R(15), 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X16,
+ .rowbits = 15, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
+ B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
+ R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
+ 0, 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
+ B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
+ R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X16,
+ .rowbits = 16, .colbits = 10,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
+ B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
+ R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
+ R(15), 0, 0, 0
+ }
+ },
+ {
+ .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X8,
+ .rowbits = 16, .colbits = 11,
+ .bits = {
+ C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
+ B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
+ R(8), R(9), R(10), C(9), R(11), RS, C(11), R(12), R(13),
+ R(14), R(15), 0, 0
+ }
+ }
+};
+
+static int bank_hash(u64 pmiaddr, int idx, int shft)
+{
+ int bhash = 0;
+
+ switch (idx) {
+ case 0:
+ bhash ^= ((pmiaddr >> (12 + shft)) ^ (pmiaddr >> (9 + shft))) & 1;
+ break;
+ case 1:
+ bhash ^= (((pmiaddr >> (10 + shft)) ^ (pmiaddr >> (8 + shft))) & 1) << 1;
+ bhash ^= ((pmiaddr >> 22) & 1) << 1;
+ break;
+ case 2:
+ bhash ^= (((pmiaddr >> (13 + shft)) ^ (pmiaddr >> (11 + shft))) & 1) << 2;
+ break;
+ }
+
+ return bhash;
+}
+
+static int rank_hash(u64 pmiaddr)
+{
+ return ((pmiaddr >> 16) ^ (pmiaddr >> 10)) & 1;
+}
+
+/* Second stage decode. Compute rank, bank, row & column. */
+static int apl_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
+ struct dram_addr *daddr, char *msg)
+{
+ struct d_cr_drp0 *cr_drp0 = &drp0[pmiidx];
+ struct pnd2_pvt *pvt = mci->pvt_info;
+ int g = pvt->dimm_geom[pmiidx];
+ struct dimm_geometry *d = &dimms[g];
+ int column = 0, bank = 0, row = 0, rank = 0;
+ int i, idx, type, skiprs = 0;
+
+ for (i = 0; i < PMI_ADDRESS_WIDTH; i++) {
+ int bit = (pmiaddr >> i) & 1;
+
+ if (i + skiprs >= PMI_ADDRESS_WIDTH) {
+ snprintf(msg, PND2_MSG_SIZE, "Bad dimm_geometry[] table\n");
+ return -EINVAL;
+ }
+
+ type = d->bits[i + skiprs] & ~0xf;
+ idx = d->bits[i + skiprs] & 0xf;
+
+ /*
+ * On single rank DIMMs ignore the rank select bit
+ * and shift remainder of "bits[]" down one place.
+ */
+ if (type == RS && (cr_drp0->rken0 + cr_drp0->rken1) == 1) {
+ skiprs = 1;
+ type = d->bits[i + skiprs] & ~0xf;
+ idx = d->bits[i + skiprs] & 0xf;
+ }
+
+ switch (type) {
+ case C(0):
+ column |= (bit << idx);
+ break;
+ case B(0):
+ bank |= (bit << idx);
+ if (cr_drp0->bahen)
+ bank ^= bank_hash(pmiaddr, idx, d->addrdec);
+ break;
+ case R(0):
+ row |= (bit << idx);
+ break;
+ case RS:
+ rank = bit;
+ if (cr_drp0->rsien)
+ rank ^= rank_hash(pmiaddr);
+ break;
+ default:
+ if (bit) {
+ snprintf(msg, PND2_MSG_SIZE, "Bad translation\n");
+ return -EINVAL;
+ }
+ goto done;
+ }
+ }
+
+done:
+ daddr->col = column;
+ daddr->bank = bank;
+ daddr->row = row;
+ daddr->rank = rank;
+ daddr->dimm = 0;
+
+ return 0;
+}
+
+/* Pluck bit "in" from pmiaddr and return value shifted to bit "out" */
+#define dnv_get_bit(pmi, in, out) ((int)(((pmi) >> (in)) & 1u) << (out))
+
+static int dnv_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
+ struct dram_addr *daddr, char *msg)
+{
+ /* Rank 0 or 1 */
+ daddr->rank = dnv_get_bit(pmiaddr, dmap[pmiidx].rs0 + 13, 0);
+ /* Rank 2 or 3 */
+ daddr->rank |= dnv_get_bit(pmiaddr, dmap[pmiidx].rs1 + 13, 1);
+
+ /*
+ * Normally ranks 0,1 are DIMM0, and 2,3 are DIMM1, but we
+ * flip them if DIMM1 is larger than DIMM0.
+ */
+ daddr->dimm = (daddr->rank >= 2) ^ drp[pmiidx].dimmflip;
+
+ daddr->bank = dnv_get_bit(pmiaddr, dmap[pmiidx].ba0 + 6, 0);
+ daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].ba1 + 6, 1);
+ daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg0 + 6, 2);
+ if (dsch.ddr4en)
+ daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg1 + 6, 3);
+ if (dmap1[pmiidx].bxor) {
+ if (dsch.ddr4en) {
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 0);
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 1);
+ if (dsch.chan_width == 0)
+ /* 64/72 bit dram channel width */
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2);
+ else
+ /* 32/40 bit dram channel width */
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2);
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 3);
+ } else {
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 0);
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 1);
+ if (dsch.chan_width == 0)
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2);
+ else
+ daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2);
+ }
+ }
+
+ daddr->row = dnv_get_bit(pmiaddr, dmap2[pmiidx].row0 + 6, 0);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row1 + 6, 1);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 2);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row3 + 6, 3);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row4 + 6, 4);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row5 + 6, 5);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 6);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 7);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row8 + 6, 8);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row9 + 6, 9);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row10 + 6, 10);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row11 + 6, 11);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row12 + 6, 12);
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row13 + 6, 13);
+ if (dmap4[pmiidx].row14 != 31)
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row14 + 6, 14);
+ if (dmap4[pmiidx].row15 != 31)
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row15 + 6, 15);
+ if (dmap4[pmiidx].row16 != 31)
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row16 + 6, 16);
+ if (dmap4[pmiidx].row17 != 31)
+ daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row17 + 6, 17);
+
+ daddr->col = dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 3);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 4);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca5 + 6, 5);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca6 + 6, 6);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca7 + 6, 7);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca8 + 6, 8);
+ daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca9 + 6, 9);
+ if (!dsch.ddr4en && dmap1[pmiidx].ca11 != 0x3f)
+ daddr->col |= dnv_get_bit(pmiaddr, dmap1[pmiidx].ca11 + 13, 11);
+
+ return 0;
+}
+
+static int check_channel(int ch)
+{
+ if (drp0[ch].dramtype != 0) {
+ pnd2_printk(KERN_INFO, "Unsupported DIMM in channel %d\n", ch);
+ return 1;
+ } else if (