diff options
60 files changed, 3788 insertions, 740 deletions
diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index 8494ef27e8d2..3acf2f17a73f 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -90,6 +90,21 @@ Description: capability. +What: /sys/bus/cxl/devices/{port,endpoint}X/parent_dport +Date: January, 2023 +KernelVersion: v6.3 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) CXL port objects are instantiated for each upstream port in + a CXL/PCIe switch, and for each endpoint to map the + corresponding memory device into the CXL port hierarchy. When a + descendant CXL port (switch or endpoint) is enumerated it is + useful to know which 'dport' object in the parent CXL port + routes to this descendant. The 'parent_dport' symlink points to + the device representing the downstream port of a CXL switch that + routes to {port,endpoint}X. + + What: /sys/bus/cxl/devices/portX/dportY Date: June, 2021 KernelVersion: v5.14 @@ -183,7 +198,7 @@ Description: What: /sys/bus/cxl/devices/endpointX/CDAT Date: July, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RO) If this sysfs entry is not present no DOE mailbox was @@ -194,7 +209,7 @@ Description: What: /sys/bus/cxl/devices/decoderX.Y/mode Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it @@ -214,7 +229,7 @@ Description: What: /sys/bus/cxl/devices/decoderX.Y/dpa_resource Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RO) When a CXL decoder is of devtype "cxl_decoder_endpoint", @@ -225,7 +240,7 @@ Description: What: /sys/bus/cxl/devices/decoderX.Y/dpa_size Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it @@ -245,7 +260,7 @@ Description: What: /sys/bus/cxl/devices/decoderX.Y/interleave_ways Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RO) The number of targets across which this decoder's host @@ -260,7 +275,7 @@ Description: What: /sys/bus/cxl/devices/decoderX.Y/interleave_granularity Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RO) The number of consecutive bytes of host physical address @@ -270,25 +285,25 @@ Description: interleave_granularity). -What: /sys/bus/cxl/devices/decoderX.Y/create_pmem_region -Date: May, 2022 -KernelVersion: v5.20 +What: /sys/bus/cxl/devices/decoderX.Y/create_{pmem,ram}_region +Date: May, 2022, January, 2023 +KernelVersion: v6.0 (pmem), v6.3 (ram) Contact: linux-cxl@vger.kernel.org Description: (RW) Write a string in the form 'regionZ' to start the process - of defining a new persistent memory region (interleave-set) - within the decode range bounded by root decoder 'decoderX.Y'. - The value written must match the current value returned from - reading this attribute. An atomic compare exchange operation is - done on write to assign the requested id to a region and - allocate the region-id for the next creation attempt. EBUSY is - returned if the region name written does not match the current - cached value. + of defining a new persistent, or volatile memory region + (interleave-set) within the decode range bounded by root decoder + 'decoderX.Y'. The value written must match the current value + returned from reading this attribute. An atomic compare exchange + operation is done on write to assign the requested id to a + region and allocate the region-id for the next creation attempt. + EBUSY is returned if the region name written does not match the + current cached value. What: /sys/bus/cxl/devices/decoderX.Y/delete_region Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (WO) Write a string in the form 'regionZ' to delete that region, @@ -297,17 +312,18 @@ Description: What: /sys/bus/cxl/devices/regionZ/uuid Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) Write a unique identifier for the region. This field must be set for persistent regions and it must not conflict with the - UUID of another region. + UUID of another region. For volatile ram regions this + attribute is a read-only empty string. What: /sys/bus/cxl/devices/regionZ/interleave_granularity Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) Set the number of consecutive bytes each device in the @@ -318,7 +334,7 @@ Description: What: /sys/bus/cxl/devices/regionZ/interleave_ways Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) Configures the number of devices participating in the @@ -328,7 +344,7 @@ Description: What: /sys/bus/cxl/devices/regionZ/size Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) System physical address space to be consumed by the region. @@ -343,9 +359,20 @@ Description: results in the same address being allocated. +What: /sys/bus/cxl/devices/regionZ/mode +Date: January, 2023 +KernelVersion: v6.3 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) The mode of a region is established at region creation time + and dictates the mode of the endpoint decoder that comprise the + region. For more details on the possible modes see + /sys/bus/cxl/devices/decoderX.Y/mode + + What: /sys/bus/cxl/devices/regionZ/resource Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RO) A region is a contiguous partition of a CXL root decoder @@ -357,7 +384,7 @@ Description: What: /sys/bus/cxl/devices/regionZ/target[0..N] Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) Write an endpoint decoder object name to 'targetX' where X @@ -376,7 +403,7 @@ Description: What: /sys/bus/cxl/devices/regionZ/commit Date: May, 2022 -KernelVersion: v5.20 +KernelVersion: v6.0 Contact: linux-cxl@vger.kernel.org Description: (RW) Write a boolean 'true' string value to this attribute to diff --git a/MAINTAINERS b/MAINTAINERS index 0120ae58d2e7..324a857e0784 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5912,6 +5912,7 @@ M: Dan Williams <dan.j.williams@intel.com> M: Vishal Verma <vishal.l.verma@intel.com> M: Dave Jiang <dave.jiang@intel.com> L: nvdimm@lists.linux.dev +L: linux-cxl@vger.kernel.org S: Supported F: drivers/dax/ diff --git a/drivers/Makefile b/drivers/Makefile index b1a714686050..7d01fa2b2b7e 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -71,7 +71,7 @@ obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ obj-$(CONFIG_PARPORT) += parport/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ -obj-$(CONFIG_DAX) += dax/ +obj-y += dax/ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += cxl/ diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 605a0c7053be..bba268ecd802 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -718,7 +718,7 @@ static void hmat_register_target_devices(struct memory_target *target) for (res = target->memregions.child; res; res = res->sibling) { int target_nid = pxm_to_node(target->memory_pxm); - hmem_register_device(target_nid, res); + hmem_register_resource(target_nid, res); } } @@ -869,4 +869,4 @@ out_put: acpi_put_table(tbl); return 0; } -device_initcall(hmat_init); +subsys_initcall(hmat_init); diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index b3c202d2a433..84030804a763 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -1047,6 +1047,9 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root, if (!(root->osc_control_set & OSC_PCI_EXPRESS_DPC_CONTROL)) host_bridge->native_dpc = 0; + if (!(root->osc_ext_control_set & OSC_CXL_ERROR_REPORTING_CONTROL)) + host_bridge->native_cxl_error = 0; + /* * Evaluate the "PCI Boot Configuration" _DSM Function. If it * exists and returns 0, we must preserve any PCI resource diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 0ac53c422c31..ff4e78117b31 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -104,19 +104,29 @@ config CXL_SUSPEND depends on SUSPEND && CXL_MEM config CXL_REGION - bool + bool "CXL: Region Support" default CXL_BUS # For MAX_PHYSMEM_BITS depends on SPARSEMEM select MEMREGION select GET_FREE_REGION + help + Enable the CXL core to enumerate and provision CXL regions. A CXL + region is defined by one or more CXL expanders that decode a given + system-physical address range. For CXL regions established by + platform-firmware this option enables memory error handling to + identify the devices participating in a given interleaved memory + range. Otherwise, platform-firmware managed CXL is enabled by being + placed in the system address map and does not need a driver. + + If unsure say 'y' config CXL_REGION_INVALIDATION_TEST bool "CXL: Region Cache Management Bypass (TEST)" depends on CXL_REGION help CXL Region management and security operations potentially invalidate - the content of CPU caches without notifiying those caches to + the content of CPU caches without notifying those caches to invalidate the affected cachelines. The CXL Region driver attempts to invalidate caches when those events occur. If that invalidation fails the region will fail to enable. Reasons for cache diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 13cde44c6086..7e1765b09e04 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -19,7 +19,7 @@ struct cxl_cxims_data { /* * Find a targets entry (n) in the host bridge interleave list. - * CXL Specfication 3.0 Table 9-22 + * CXL Specification 3.0 Table 9-22 */ static int cxl_xor_calc_n(u64 hpa, struct cxl_cxims_data *cximsd, int iw, int ig) @@ -731,7 +731,8 @@ static void __exit cxl_acpi_exit(void) cxl_bus_drain(); } -module_init(cxl_acpi_init); +/* load before dax_hmem sees 'Soft Reserved' CXL ranges */ +subsys_initcall(cxl_acpi_init); module_exit(cxl_acpi_exit); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index 79c7257f4107..ca4ae31d8f57 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -3,6 +3,8 @@ obj-$(CONFIG_CXL_BUS) += cxl_core.o obj-$(CONFIG_CXL_SUSPEND) += suspend.o ccflags-y += -I$(srctree)/drivers/cxl +CFLAGS_trace.o = -DTRACE_INCLUDE_PATH=. -I$(src) + cxl_core-y := port.o cxl_core-y += pmem.o cxl_core-y += regs.o @@ -10,4 +12,5 @@ cxl_core-y += memdev.o cxl_core-y += mbox.o cxl_core-y += pci.o cxl_core-y += hdm.o +cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 8c04672dca56..cde475e13216 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -11,15 +11,18 @@ extern struct attribute_group cxl_base_attribute_group; #ifdef CONFIG_CXL_REGION extern struct device_attribute dev_attr_create_pmem_region; +extern struct device_attribute dev_attr_create_ram_region; extern struct device_attribute dev_attr_delete_region; extern struct device_attribute dev_attr_region; extern const struct device_type cxl_pmem_region_type; +extern const struct device_type cxl_dax_region_type; extern const struct device_type cxl_region_type; void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled); #define CXL_REGION_ATTR(x) (&dev_attr_##x.attr) #define CXL_REGION_TYPE(x) (&cxl_region_type) #define SET_CXL_REGION_ATTR(x) (&dev_attr_##x.attr), #define CXL_PMEM_REGION_TYPE(x) (&cxl_pmem_region_type) +#define CXL_DAX_REGION_TYPE(x) (&cxl_dax_region_type) int cxl_region_init(void); void cxl_region_exit(void); #else @@ -37,6 +40,7 @@ static inline void cxl_region_exit(void) #define CXL_REGION_TYPE(x) NULL #define SET_CXL_REGION_ATTR(x) #define CXL_PMEM_REGION_TYPE(x) NULL +#define CXL_DAX_REGION_TYPE(x) NULL #endif struct cxl_send_command; @@ -56,9 +60,6 @@ resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled); extern struct rw_semaphore cxl_dpa_rwsem; -bool is_switch_decoder(struct device *dev); -struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev); - int cxl_memdev_init(void); void cxl_memdev_exit(void); void cxl_mbox_init(void); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index dcc16d7cb8f3..45deda18ed32 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -101,11 +101,34 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, BIT(CXL_CM_CAP_CAP_ID_HDM)); } +static struct cxl_hdm *devm_cxl_setup_emulated_hdm(struct cxl_port *port, + struct cxl_endpoint_dvsec_info *info) +{ + struct device *dev = &port->dev; + struct cxl_hdm *cxlhdm; + + if (!info->mem_enabled) + return ERR_PTR(-ENODEV); + + cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); + if (!cxlhdm) + return ERR_PTR(-ENOMEM); + + cxlhdm->port = port; + cxlhdm->decoder_count = info->ranges; + cxlhdm->target_count = info->ranges; + dev_set_drvdata(&port->dev, cxlhdm); + + return cxlhdm; +} + /** * devm_cxl_setup_hdm - map HDM decoder component registers * @port: cxl_port to map + * @info: cached DVSEC range register info */ -struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port) +struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, + struct cxl_endpoint_dvsec_info *info) { struct device *dev = &port->dev; struct cxl_hdm *cxlhdm; @@ -119,6 +142,9 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port) cxlhdm->port = port; crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE); if (!crb) { + if (info && info->mem_enabled) + return devm_cxl_setup_emulated_hdm(port, info); + dev_err(dev, "No component registers mapped\n"); return ERR_PTR(-ENXIO); } @@ -279,7 +305,7 @@ success: return 0; } -static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, +int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, resource_size_t base, resource_size_t len, resource_size_t skipped) { @@ -295,6 +321,7 @@ static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); } +EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) { @@ -676,12 +703,71 @@ static int cxl_decoder_reset(struct cxl_decoder *cxld) port->commit_end--; cxld->flags &= ~CXL_DECODER_F_ENABLE; + /* Userspace is now responsible for reconfiguring this decoder */ + if (is_endpoint_decoder(&cxld->dev)) { + struct cxl_endpoint_decoder *cxled; + + cxled = to_cxl_endpoint_decoder(&cxld->dev); + cxled->state = CXL_DECODER_STATE_MANUAL; + } + + return 0; +} + +static int cxl_setup_hdm_decoder_from_dvsec(struct cxl_port *port, + struct cxl_decoder *cxld, int which, + struct cxl_endpoint_dvsec_info *info) +{ + if (!is_cxl_endpoint(port)) + return -EOPNOTSUPP; + + if (!range_len(&info->dvsec_range[which])) + return -ENOENT; + + cxld->target_type = CXL_DECODER_EXPANDER; + cxld->commit = NULL; + cxld->reset = NULL; + cxld->hpa_range = info->dvsec_range[which]; + + /* + * Set the emulated decoder as locked pending additional support to + * change the range registers at run time. + */ + cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; + port->commit_end = cxld->id; + return 0; } +static bool should_emulate_decoders(struct cxl_port *port) +{ + struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); + void __iomem *hdm = cxlhdm->regs.hdm_decoder; + u32 ctrl; + int i; + + if (!is_cxl_endpoint(cxlhdm->port)) + return false; + + if (!hdm) + return true; + + /* + * If any decoders are committed already, there should not be any + * emulated DVSEC decoders. + */ + for (i = 0; i < cxlhdm->decoder_count; i++) { + ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); + if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) + return false; + } + + return true; +} + static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, int *target_map, void __iomem *hdm, int which, - u64 *dpa_base) + u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) { struct cxl_endpoint_decoder *cxled = NULL; u64 size, base, skip, dpa_size; @@ -694,6 +780,9 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, unsigned char target_id[8]; } target_list; + if (should_emulate_decoders(port)) + return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info); + if (is_endpoint_decoder(&cxld->dev)) cxled = to_cxl_endpoint_decoder(&cxld->dev); @@ -717,6 +806,9 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, .end = base + size - 1, }; + if (cxled && !committed && range_len(&info->dvsec_range[which])) + return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info); + /* decoders are enabled if committed */ if (committed) { cxld->flags |= CXL_DECODER_F_ENABLE; @@ -783,21 +875,21 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, return rc; } *dpa_base += dpa_size + skip; + + cxled->state = CXL_DECODER_STATE_AUTO; + return 0; } -/** - * devm_cxl_enumerate_decoders - add decoder objects per HDM register set - * @cxlhdm: Structure to populate with HDM capabilities - */ -int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) +static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) { void __iomem *hdm = cxlhdm->regs.hdm_decoder; - struct cxl_port *port = cxlhdm->port; - int i, committed; - u64 dpa_base = 0; + int committed, i; u32 ctrl; + if (!hdm) + return; + /* * Since the register resource was recently claimed via request_region() * be careful about trusting the "not-committed" status until the commit @@ -814,6 +906,22 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) /* ensure that future checks of committed can be trusted */ if (committed != cxlhdm->decoder_count) msleep(20); +} + +/** + * devm_cxl_enumerate_decoders - add decoder objects per HDM register set + * @cxlhdm: Structure to populate with HDM capabilities + * @info: cached DVSEC range register info + */ +int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, + struct cxl_endpoint_dvsec_info *info) +{ + void __iomem *hdm = cxlhdm->regs.hdm_decoder; + struct cxl_port *port = cxlhdm->port; + int i; + u64 dpa_base = 0; + + cxl_settle_decoders(cxlhdm); for (i = 0; i < cxlhdm->decoder_count; i++) { int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; @@ -826,7 +934,8 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) cxled = cxl_endpoint_decoder_alloc(port); if (IS_ERR(cxled)) { dev_warn(&port->dev, - "Failed to allocate the decoder\n"); + "Failed to allocate decoder%d.%d\n", + port->id, i); return PTR_ERR(cxled); } cxld = &cxled->cxld; @@ -836,21 +945,26 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) cxlsd = cxl_switch_decoder_alloc(port, target_count); if (IS_ERR(cxlsd)) { dev_warn(&port->dev, - "Failed to allocate the decoder\n"); + "Failed to allocate decoder%d.%d\n", + port->id, i); return PTR_ERR(cxlsd); } cxld = &cxlsd->cxld; } - rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base); + rc = init_hdm_decoder(port, cxld, target_map, hdm, i, + &dpa_base, info); if (rc) { + dev_warn(&port->dev, |